From 13d693b58a252b5ee16affc996057e43e1a27dac Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Fri, 16 Oct 2026 23:06:34 +1100 Subject: [PATCH 01/13] Add ansible Galaxy remote type - add ansible package type proxying the Galaxy v3 API - rewrite href/links/download_url back to the remote so tarballs flow through it - forward the query string for providers that opt in, keeping Galaxy pagination working - add a UI usage snippet and an ansible e2e caching case --- README.md | 3 +- e2e-docker/README.md | 2 +- e2e-docker/caching_test.go | 1 + .../galaxy/artifacts/ns-col-1.0.0.tar.gz | Bin 0 -> 51 bytes internal/api/v1/proxy.go | 4 + internal/provider/ansible/ansible.go | 65 ++++++++++++++ internal/provider/ansible/ansible_test.go | 85 ++++++++++++++++++ internal/provider/provider.go | 7 ++ internal/server/server.go | 1 + pkg/models/package_type.go | 2 + pkg/models/package_type_test.go | 1 + ui/src/components/UsageInstructions.tsx | 9 ++ ui/src/pages/Remotes.tsx | 1 + 13 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 e2e-docker/fixtures/galaxy/artifacts/ns-col-1.0.0.tar.gz create mode 100644 internal/provider/ansible/ansible.go create mode 100644 internal/provider/ansible/ansible_test.go diff --git a/README.md b/README.md index 660a658..bba296a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ArtifactAPI -Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform. +Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform. ## Quick Start @@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` | `terraform` | `*/versions` | `*/download/*/*` | | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | +| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` | | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | Providers classify paths automatically. Users only configure what to proxy and TTLs. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index 3125d23..a33e0c2 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -23,7 +23,7 @@ already-running stack. - **Repository lifecycle** — add / change / delete for remote, local and virtual repos. - **Caching** — one immutable artifact per remote package type (generic, docker, - helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the + helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, ansible) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index d04ed6f..74ef86f 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) { {"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}, {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, + {"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"}, {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, } diff --git a/e2e-docker/fixtures/galaxy/artifacts/ns-col-1.0.0.tar.gz b/e2e-docker/fixtures/galaxy/artifacts/ns-col-1.0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..8245b996dfad42ac6ee368f73285008472d63c92 GIT binary patch literal 51 zcmb2|=3oE=W^YrkP>(a(=X_3ihI*Vkf6~)e@8p^D!5W@AUS4NTK4fC>HQjC?4^#^P Do175v literal 0 HcmV?d00001 diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index e77f43e..e5f7f09 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -92,6 +92,10 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { } } + if _, ok := prov.(provider.QueryForwarder); ok && r.URL.RawQuery != "" { + path += "?" + r.URL.RawQuery + } + result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header) if err != nil { var proxyErr *proxy.ProxyError diff --git a/internal/provider/ansible/ansible.go b/internal/provider/ansible/ansible.go new file mode 100644 index 0000000..8f9cdb2 --- /dev/null +++ b/internal/provider/ansible/ansible.go @@ -0,0 +1,65 @@ +// Package ansible proxies an Ansible Galaxy server (v3 collections API): +// mutable JSON metadata whose links are rewritten back to the remote, plus +// immutable collection tarballs. +package ansible + +import ( + "context" + "encoding/json" + "net/http" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/auth" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(&Provider{}) +} + +type Provider struct{} + +func (p *Provider) Type() models.PackageType { return models.PackageAnsible } + +func (p *Provider) Classify(path string) provider.Mutability { + if isArtifact(path) { + return provider.Immutable + } + return provider.Mutable +} + +func (p *Provider) ContentType(path string) string { + if isArtifact(path) { + return "application/gzip" + } + return "application/json" +} + +func (p *Provider) UpstreamURL(remote models.Remote, path string) string { + return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") +} + +// ForwardQuery keeps limit/offset pagination in the proxied path. +func (p *Provider) ForwardQuery() {} + +// RewriteResponse points root-relative hrefs/links and absolute download_url +// values at the remote. Links stay root-relative, which ansible-galaxy joins +// onto the server URL, so the cached body is independent of the request host. +func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) { + if !json.Valid(body) { + return nil, nil + } + prefix := `"/api/v1/remote/` + remote.Name + `/` + content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`) + content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix) + return []byte(content), nil +} + +func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { + return auth.BasicHeaders(remote), nil +} + +func isArtifact(path string) bool { + return strings.HasSuffix(path, ".tar.gz") +} diff --git a/internal/provider/ansible/ansible_test.go b/internal/provider/ansible/ansible_test.go new file mode 100644 index 0000000..680eb02 --- /dev/null +++ b/internal/provider/ansible/ansible_test.go @@ -0,0 +1,85 @@ +package ansible + +import ( + "encoding/json" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestClassify(t *testing.T) { + p := &Provider{} + tests := []struct { + path string + want provider.Mutability + }{ + {"api/", provider.Mutable}, + {"api/v3/collections/community/general/versions/?limit=100", provider.Mutable}, + {"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable}, + {"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable}, + } + for _, tt := range tests { + if got := p.Classify(tt.path); got != tt.want { + t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestUpstreamURLKeepsQuery(t *testing.T) { + p := &Provider{} + got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100") + want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100" + if got != want { + t.Errorf("UpstreamURL = %q, want %q", got, want) + } +} + +func TestRewriteResponse(t *testing.T) { + p := &Provider{} + remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"} + body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` + + `"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` + + `"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` + + `"metadata":{"repository":"https://github.com/a/b"}}` + + out, err := p.RewriteResponse([]byte(body), remote, "") + if err != nil { + t.Fatal(err) + } + var got struct { + Links map[string]any `json:"links"` + Href string `json:"href"` + DownloadURL string `json:"download_url"` + Metadata map[string]any `json:"metadata"` + } + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("rewritten body is not JSON: %v", err) + } + pre := "/api/v1/remote/galaxy/" + if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want { + t.Errorf("next = %v, want %q", got.Links["next"], want) + } + if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want { + t.Errorf("href = %q, want %q", got.Href, want) + } + if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want { + t.Errorf("download_url = %q, want %q", got.DownloadURL, want) + } + if got.Metadata["repository"] != "https://github.com/a/b" { + t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"]) + } +} + +func TestRewriteResponseSkipsNonJSON(t *testing.T) { + out, err := (&Provider{}).RewriteResponse([]byte(""), models.Remote{Name: "g"}, "") + if err != nil || out != nil { + t.Errorf("got %q, %v; want nil, nil", out, err) + } +} + +func TestForwardsQuery(t *testing.T) { + if _, ok := any(&Provider{}).(provider.QueryForwarder); !ok { + t.Error("ansible provider must forward the query string for pagination") + } +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 8de9201..5e98232 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -65,6 +65,13 @@ type MetadataStore interface { InsertDebMetadata(ctx context.Context, meta *DebMetadata) error } +// QueryForwarder marks a remote provider whose upstream answers depend on the +// query string (e.g. paginated JSON APIs); the query is kept as part of the +// proxied and cached path. +type QueryForwarder interface { + ForwardQuery() +} + // RemoteServer lets a remote provider fully answer a request itself instead of // going through the byte-proxy engine. It is the remote-side analog of // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize diff --git a/internal/server/server.go b/internal/server/server.go index 88d12ac..4082632 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,6 +22,7 @@ import ( "git.unkin.net/unkin/artifactapi/internal/gc" "git.unkin.net/unkin/artifactapi/internal/githubauth" "git.unkin.net/unkin/artifactapi/internal/provider/alpine" + _ "git.unkin.net/unkin/artifactapi/internal/provider/ansible" _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo" "git.unkin.net/unkin/artifactapi/internal/provider/deb" _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" diff --git a/pkg/models/package_type.go b/pkg/models/package_type.go index 52c4cc2..c06f3e0 100644 --- a/pkg/models/package_type.go +++ b/pkg/models/package_type.go @@ -17,6 +17,7 @@ const ( PackageTerraform PackageType = "terraform" PackageGoProxy PackageType = "goproxy" PackageCargo PackageType = "cargo" + PackageAnsible PackageType = "ansible" PackageGitHubRPM PackageType = "github_rpm" PackageGitHubDeb PackageType = "github_deb" PackageGitHubAlpine PackageType = "github_alpine" @@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{ PackageTerraform: true, PackageGoProxy: true, PackageCargo: true, + PackageAnsible: true, PackageGitHubRPM: true, PackageGitHubDeb: true, PackageGitHubAlpine: true, diff --git a/pkg/models/package_type_test.go b/pkg/models/package_type_test.go index 5143487..482575c 100644 --- a/pkg/models/package_type_test.go +++ b/pkg/models/package_type_test.go @@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) { models.PackageTerraform, models.PackageGoProxy, models.PackageCargo, + models.PackageAnsible, models.PackageGitHubRPM, models.PackageGitHubDeb, models.PackageGitHubAlpine, diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index 228a924..a078be1 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -243,6 +243,15 @@ index = "sparse+${proxy}/"`, }, ]; + case 'ansible': + return [ + { + title: 'Install collections through this Galaxy proxy', + language: 'bash', + code: `ansible-galaxy collection install community.general -s ${proxy}/api/`, + }, + ]; + case 'puppet': return [ { diff --git a/ui/src/pages/Remotes.tsx b/ui/src/pages/Remotes.tsx index 31bfcf6..c8f48ee 100644 --- a/ui/src/pages/Remotes.tsx +++ b/ui/src/pages/Remotes.tsx @@ -18,6 +18,7 @@ const typeColors: Record Date: Fri, 16 Oct 2026 23:09:20 +1100 Subject: [PATCH 02/13] Retrigger CI -- 2.47.3 From 586b07bab6504495418b1fc2dc4b02acbeb335ec Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:31:30 +1100 Subject: [PATCH 03/13] Forward allowlisted query params for opt-in remote providers --- internal/api/v1/forwardquery_test.go | 52 ++++++++++++++++++++++++++++ internal/api/v1/proxy.go | 24 +++++++++++++ internal/provider/provider.go | 7 ++++ 3 files changed, 83 insertions(+) create mode 100644 internal/api/v1/forwardquery_test.go diff --git a/internal/api/v1/forwardquery_test.go b/internal/api/v1/forwardquery_test.go new file mode 100644 index 0000000..be758e2 --- /dev/null +++ b/internal/api/v1/forwardquery_test.go @@ -0,0 +1,52 @@ +package v1 + +import ( + "net/url" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +type plainProvider struct{ provider.Provider } + +type forwardingProvider struct{ plainProvider } + +func (forwardingProvider) ForwardedQuery(path string) []string { + if strings.HasSuffix(path, ".tar.gz") { + return nil + } + return []string{"limit", "offset"} +} + +func TestForwardQuery(t *testing.T) { + long := strings.Repeat("9", maxForwardedValueLen+1) + tests := []struct { + name string + prov provider.Provider + path string + query string + want string + }{ + {"non-forwarder keeps path", plainProvider{}, "a/versions/", "limit=10&offset=5", "a/versions/"}, + {"no query", forwardingProvider{}, "a/versions/", "", "a/versions/"}, + {"allowlisted params sorted", forwardingProvider{}, "a/versions/", "offset=5&limit=10", "a/versions/?limit=10&offset=5"}, + {"unknown params dropped", forwardingProvider{}, "a/versions/", "x=1&limit=10&cb=zz", "a/versions/?limit=10"}, + {"only unknown params", forwardingProvider{}, "a/versions/", "x=1", "a/versions/"}, + {"first value wins", forwardingProvider{}, "a/versions/", "limit=10&limit=20", "a/versions/?limit=10"}, + {"values re-encoded", forwardingProvider{}, "a/versions/", "limit=1%202", "a/versions/?limit=1+2"}, + {"overlong value dropped", forwardingProvider{}, "a/versions/", "limit=" + long + "&offset=5", "a/versions/?offset=5"}, + {"path without allowlist", forwardingProvider{}, "a/b-1.0.0.tar.gz", "limit=10", "a/b-1.0.0.tar.gz"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + q, err := url.ParseQuery(tt.query) + if err != nil { + t.Fatal(err) + } + if got := forwardQuery(tt.prov, tt.path, q); got != tt.want { + t.Errorf("forwardQuery = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 7429007..c64c139 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -7,6 +7,7 @@ import ( "io" "log/slog" "net/http" + "net/url" "github.com/go-chi/chi/v5" @@ -93,6 +94,8 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { } } + path = forwardQuery(prov, path, r.URL.Query()) + result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header) if err != nil { var proxyErr *proxy.ProxyError @@ -258,3 +261,24 @@ func scheme(r *http.Request) string { } return "http" } + +const maxForwardedValueLen = 32 + +// forwardQuery appends the provider's allowlisted query params to path in a +// canonical (sorted, single-valued) form, since the result is the cache key. +func forwardQuery(prov provider.Provider, path string, query url.Values) string { + f, ok := prov.(provider.QueryForwarder) + if !ok { + return path + } + kept := url.Values{} + for _, name := range f.ForwardedQuery(path) { + if v := query.Get(name); v != "" && len(v) <= maxForwardedValueLen { + kept.Set(name, v) + } + } + if len(kept) == 0 { + return path + } + return path + "?" + kept.Encode() +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 4c005f3..37afb2d 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -65,6 +65,13 @@ type MetadataStore interface { InsertDebMetadata(ctx context.Context, meta *DebMetadata) error } +// QueryForwarder is a remote provider whose upstream answers depend on the +// query string (e.g. paginated JSON APIs). Only the named params for path are +// forwarded upstream and kept in the cache key; nil forwards nothing. +type QueryForwarder interface { + ForwardedQuery(path string) []string +} + // RemoteServer lets a remote provider fully answer a request itself instead of // going through the byte-proxy engine. It is the remote-side analog of // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize -- 2.47.3 From af6a088ef4d6cd9f6ef326eba86173dfbd3bff33 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:35:03 +1100 Subject: [PATCH 04/13] Forward ansible pagination params and rewrite only Galaxy link fields --- internal/provider/ansible/ansible.go | 82 ++++++++++++++--- internal/provider/ansible/ansible_test.go | 104 +++++++++++++++------- internal/provider/provider.go | 3 + 3 files changed, 147 insertions(+), 42 deletions(-) diff --git a/internal/provider/ansible/ansible.go b/internal/provider/ansible/ansible.go index 668155c..ef1e969 100644 --- a/internal/provider/ansible/ansible.go +++ b/internal/provider/ansible/ansible.go @@ -4,9 +4,11 @@ package ansible import ( + "bytes" "context" "encoding/json" "net/http" + "net/url" "strings" "git.unkin.net/unkin/artifactapi/internal/auth" @@ -40,26 +42,82 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string { return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") } -// ForwardQuery keeps limit/offset pagination in the proxied path. -func (p *Provider) ForwardedQuery(string) []string { return nil } +// ForwardedQuery keeps limit/offset pagination on metadata paths only. +func (p *Provider) ForwardedQuery(path string) []string { + if isArtifact(path) { + return nil + } + return []string{"limit", "offset"} +} -// RewriteResponse points root-relative hrefs/links and absolute download_url -// values at the remote. Links stay root-relative, which ansible-galaxy joins -// onto the server URL, so the cached body is independent of the request host. +// RewriteResponse points href, download_url and links.* at the remote. Links +// stay root-relative, which ansible-galaxy joins onto the server URL, so the +// cached body is independent of the request host. func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) { - if !json.Valid(body) { + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + var doc any + if err := dec.Decode(&doc); err != nil { return nil, nil } - prefix := `"/api/v1/remote/` + remote.Name + `/` - content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`) - content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix) - return []byte(content), nil + r := rewriter{ + prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/", + base: strings.TrimRight(remote.BaseURL, "/") + "/", + } + r.walk(doc) + var out bytes.Buffer + enc := json.NewEncoder(&out) + enc.SetEscapeHTML(false) + if err := enc.Encode(doc); err != nil { + return nil, err + } + return out.Bytes(), nil } func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { return auth.BasicHeaders(remote), nil } -func isArtifact(path string) bool { - return strings.HasSuffix(path, ".tar.gz") +type rewriter struct{ prefix, base string } + +func (r rewriter) walk(v any) { + switch v := v.(type) { + case []any: + for _, e := range v { + r.walk(e) + } + case map[string]any: + for k, e := range v { + switch k { + case "href", "download_url": + v[k] = r.link(e) + case "links": + if links, ok := e.(map[string]any); ok { + for lk, l := range links { + links[lk] = r.link(l) + } + continue + } + } + r.walk(v[k]) + } + } +} + +func (r rewriter) link(v any) any { + s, ok := v.(string) + switch { + case !ok: + return v + case strings.HasPrefix(s, "/api/"): + return r.prefix + s[1:] + case strings.HasPrefix(s, r.base): + return r.prefix + strings.TrimPrefix(s, r.base) + } + return s +} + +func isArtifact(path string) bool { + path, _, _ = strings.Cut(path, "?") + return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz") } diff --git a/internal/provider/ansible/ansible_test.go b/internal/provider/ansible/ansible_test.go index d979d1c..8c96db7 100644 --- a/internal/provider/ansible/ansible_test.go +++ b/internal/provider/ansible/ansible_test.go @@ -2,12 +2,15 @@ package ansible import ( "encoding/json" + "reflect" "testing" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/pkg/models" ) +const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz" + func TestClassify(t *testing.T) { p := &Provider{} tests := []struct { @@ -16,8 +19,11 @@ func TestClassify(t *testing.T) { }{ {"api/", provider.Mutable}, {"api/v3/collections/community/general/versions/?limit=100", provider.Mutable}, + {"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable}, {"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable}, - {"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable}, + {"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable}, + {tarball, provider.Immutable}, + {tarball + "?limit=1", provider.Immutable}, } for _, tt := range tests { if got := p.Classify(tt.path); got != tt.want { @@ -26,48 +32,87 @@ func TestClassify(t *testing.T) { } } -func TestUpstreamURLKeepsQuery(t *testing.T) { - p := &Provider{} - got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100") - want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100" - if got != want { - t.Errorf("UpstreamURL = %q, want %q", got, want) +func TestContentTypeIgnoresQuery(t *testing.T) { + if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" { + t.Errorf("ContentType = %q, want application/json", got) } } -func TestRewriteResponse(t *testing.T) { +func TestForwardedQuery(t *testing.T) { p := &Provider{} - remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"} - body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` + - `"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` + - `"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` + - `"metadata":{"repository":"https://github.com/a/b"}}` + if got := p.ForwardedQuery("api/v3/collections/a/b/versions/"); !reflect.DeepEqual(got, []string{"limit", "offset"}) { + t.Errorf("metadata ForwardedQuery = %v", got) + } + if got := p.ForwardedQuery(tarball); got != nil { + t.Errorf("tarball ForwardedQuery = %v, want nil", got) + } +} - out, err := p.RewriteResponse([]byte(body), remote, "") +func rewrite(t *testing.T, remote models.Remote, body string) map[string]any { + t.Helper() + out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "") if err != nil { t.Fatal(err) } - var got struct { - Links map[string]any `json:"links"` - Href string `json:"href"` - DownloadURL string `json:"download_url"` - Metadata map[string]any `json:"metadata"` - } + var got map[string]any if err := json.Unmarshal(out, &got); err != nil { - t.Fatalf("rewritten body is not JSON: %v", err) + t.Fatalf("rewritten body is not JSON: %v: %s", err, out) } + return got +} + +func TestRewriteResponse(t *testing.T) { + remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"} + got := rewrite(t, remote, `{ + "links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null}, + "data": [{ + "href": "/api/v3/collections/a/b/versions/1.0.0/", + "download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz", + "description": "/api/ is where it lives", + "docs": "/api/v3/not-a-link/", + "metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"} + }] + }`) pre := "/api/v1/remote/galaxy/" - if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want { - t.Errorf("next = %v, want %q", got.Links["next"], want) + links := got["links"].(map[string]any) + if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want { + t.Errorf("links.next = %v, want %q", links["next"], want) } - if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want { - t.Errorf("href = %q, want %q", got.Href, want) + if links["previous"] != nil { + t.Errorf("links.previous = %v, want null", links["previous"]) } - if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want { - t.Errorf("download_url = %q, want %q", got.DownloadURL, want) + item := got["data"].([]any)[0].(map[string]any) + if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want { + t.Errorf("href = %v, want %q", item["href"], want) } - if got.Metadata["repository"] != "https://github.com/a/b" { - t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"]) + if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want { + t.Errorf("download_url = %v, want %q", item["download_url"], want) + } + for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} { + if item[k] != want { + t.Errorf("%s rewritten to %v", k, item[k]) + } + } + meta := item["metadata"].(map[string]any) + if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" { + t.Errorf("unrelated URLs rewritten: %v", meta) + } +} + +func TestRewriteResponseEscapesRemoteName(t *testing.T) { + got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`) + if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want { + t.Errorf("href = %v, want %q", got["href"], want) + } +} + +func TestRewriteResponseKeepsLargeNumbers(t *testing.T) { + out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "") + if err != nil { + t.Fatal(err) + } + if want := "{\"n\":12345678901234567890}\n"; string(out) != want { + t.Errorf("got %q, want %q", out, want) } } @@ -77,4 +122,3 @@ func TestRewriteResponseSkipsNonJSON(t *testing.T) { t.Errorf("got %q, %v; want nil, nil", out, err) } } - diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 37afb2d..20ff597 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -10,6 +10,9 @@ import ( "git.unkin.net/unkin/artifactapi/pkg/models" ) +// RemoteMountPath is the URL path remote proxies are served under. +const RemoteMountPath = "/api/v1/remote/" + type Mutability int const ( -- 2.47.3 From 9935691c6b3848033ad468f77f911fb4d96a79e6 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:40:47 +1100 Subject: [PATCH 05/13] Document ansible pagination and artifact paths --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 936c879..46a9081 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` | `terraform` | `*/versions` | `*/download/*/*` | | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | -| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` | +| `ansible` | Galaxy `api/` JSON (links rewritten, `limit`/`offset` forwarded) | `artifacts/*.tar.gz` | | `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | -- 2.47.3 From 769d673a4a406cc5b8874faf6d47f9968def0ab1 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:52:49 +1100 Subject: [PATCH 06/13] Classify bare paths, validate forwarded query values, evict query variants --- internal/api/v1/forwardquery_test.go | 16 +++++++++++----- internal/api/v1/proxy.go | 12 ++++-------- internal/provider/provider.go | 8 ++++++-- internal/proxy/classifier.go | 3 +++ internal/proxy/classifier_extra_test.go | 18 ++++++++++++++++++ internal/proxy/engine.go | 17 +++++++++++++++-- internal/proxy/evict_test.go | 25 +++++++++++++++++++++++++ 7 files changed, 82 insertions(+), 17 deletions(-) diff --git a/internal/api/v1/forwardquery_test.go b/internal/api/v1/forwardquery_test.go index be758e2..6121838 100644 --- a/internal/api/v1/forwardquery_test.go +++ b/internal/api/v1/forwardquery_test.go @@ -12,15 +12,21 @@ type plainProvider struct{ provider.Provider } type forwardingProvider struct{ plainProvider } -func (forwardingProvider) ForwardedQuery(path string) []string { +func digits(v string) (string, bool) { + if v == "" || strings.Trim(v, "0123456789") != "" { + return "", false + } + return strings.TrimLeft(v, "0"), true +} + +func (forwardingProvider) ForwardedQuery(path string) map[string]provider.QueryValidator { if strings.HasSuffix(path, ".tar.gz") { return nil } - return []string{"limit", "offset"} + return map[string]provider.QueryValidator{"limit": digits, "offset": digits} } func TestForwardQuery(t *testing.T) { - long := strings.Repeat("9", maxForwardedValueLen+1) tests := []struct { name string prov provider.Provider @@ -34,8 +40,8 @@ func TestForwardQuery(t *testing.T) { {"unknown params dropped", forwardingProvider{}, "a/versions/", "x=1&limit=10&cb=zz", "a/versions/?limit=10"}, {"only unknown params", forwardingProvider{}, "a/versions/", "x=1", "a/versions/"}, {"first value wins", forwardingProvider{}, "a/versions/", "limit=10&limit=20", "a/versions/?limit=10"}, - {"values re-encoded", forwardingProvider{}, "a/versions/", "limit=1%202", "a/versions/?limit=1+2"}, - {"overlong value dropped", forwardingProvider{}, "a/versions/", "limit=" + long + "&offset=5", "a/versions/?offset=5"}, + {"invalid value dropped", forwardingProvider{}, "a/versions/", "limit=1%202&offset=5", "a/versions/?offset=5"}, + {"value canonicalised", forwardingProvider{}, "a/versions/", "limit=0010", "a/versions/?limit=10"}, {"path without allowlist", forwardingProvider{}, "a/b-1.0.0.tar.gz", "limit=10", "a/b-1.0.0.tar.gz"}, } for _, tt := range tests { diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index c64c139..8100658 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -94,9 +94,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { } } - path = forwardQuery(prov, path, r.URL.Query()) - - result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header) + result, err := h.engine.Fetch(r.Context(), *remote, forwardQuery(prov, path, r.URL.Query()), prov, r.Header) if err != nil { var proxyErr *proxy.ProxyError if errors.As(err, &proxyErr) { @@ -262,9 +260,7 @@ func scheme(r *http.Request) string { return "http" } -const maxForwardedValueLen = 32 - -// forwardQuery appends the provider's allowlisted query params to path in a +// forwardQuery appends the provider's validated query params to path in a // canonical (sorted, single-valued) form, since the result is the cache key. func forwardQuery(prov provider.Provider, path string, query url.Values) string { f, ok := prov.(provider.QueryForwarder) @@ -272,8 +268,8 @@ func forwardQuery(prov provider.Provider, path string, query url.Values) string return path } kept := url.Values{} - for _, name := range f.ForwardedQuery(path) { - if v := query.Get(name); v != "" && len(v) <= maxForwardedValueLen { + for name, valid := range f.ForwardedQuery(path) { + if v, ok := valid(query.Get(name)); ok { kept.Set(name, v) } } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 37afb2d..66cf2b5 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -67,11 +67,15 @@ type MetadataStore interface { // QueryForwarder is a remote provider whose upstream answers depend on the // query string (e.g. paginated JSON APIs). Only the named params for path are -// forwarded upstream and kept in the cache key; nil forwards nothing. +// forwarded upstream and kept in the cache key, each as the canonical value +// its validator returns; a value the validator rejects is dropped. type QueryForwarder interface { - ForwardedQuery(path string) []string + ForwardedQuery(path string) map[string]QueryValidator } +// QueryValidator canonicalises a raw query value, reporting false to drop it. +type QueryValidator func(v string) (string, bool) + // RemoteServer lets a remote provider fully answer a request itself instead of // going through the byte-proxy engine. It is the remote-side analog of // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize diff --git a/internal/proxy/classifier.go b/internal/proxy/classifier.go index 82b889c..28aee30 100644 --- a/internal/proxy/classifier.go +++ b/internal/proxy/classifier.go @@ -2,6 +2,7 @@ package proxy import ( "regexp" + "strings" "sync" "git.unkin.net/unkin/artifactapi/internal/provider" @@ -37,7 +38,9 @@ func NewClassifier(p provider.Provider) *Classifier { return &Classifier{provider: p} } +// Classify judges the bare path; a forwarded query must not change the verdict. func (c *Classifier) Classify(remote models.Remote, path string) Classification { + path, _, _ = strings.Cut(path, "?") if matchesAny(path, compilePatterns(remote.Blocklist)) { return ClassDenied } diff --git a/internal/proxy/classifier_extra_test.go b/internal/proxy/classifier_extra_test.go index dd5403a..7eaf152 100644 --- a/internal/proxy/classifier_extra_test.go +++ b/internal/proxy/classifier_extra_test.go @@ -38,6 +38,24 @@ func TestClassifierBranches(t *testing.T) { } } +func TestClassifierIgnoresForwardedQuery(t *testing.T) { + gp, err := provider.Get(models.PackageGeneric) + if err != nil { + t.Fatal(err) + } + c := NewClassifier(gp) + if got := c.Classify(models.Remote{Blocklist: []string{`\.exe$`}}, "x.exe?limit=1"); got != ClassDenied { + t.Errorf("blocklist bypassed by query: %v", got) + } + allow := models.Remote{Patterns: []string{`/versions/$`}} + if got := c.Classify(allow, "a/versions/?limit=1&offset=100"); got != ClassImmutable { + t.Errorf("allowlisted paginated path = %v, want immutable", got) + } + if got := c.Classify(models.Remote{MutablePatterns: []string{`/versions/$`}}, "a/versions/?offset=100"); got != ClassMutable { + t.Errorf("mutable pattern on paginated path = %v, want mutable", got) + } +} + func TestClassificationString(t *testing.T) { for c, want := range map[Classification]string{ ClassImmutable: "immutable", diff --git a/internal/proxy/engine.go b/internal/proxy/engine.go index 45a1b87..a2e82e3 100644 --- a/internal/proxy/engine.go +++ b/internal/proxy/engine.go @@ -216,7 +216,8 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p // Evict drops path from every cache layer (artifact row, index object, Redis // freshness and ETag keys) so the next request refetches from upstream. A -// trailing "/*" evicts every path under that directory. +// trailing "/*" evicts every path under that directory; a plain path also +// drops its forwarded-query variants ("path?..."). func (e *Engine) Evict(ctx context.Context, remoteName, path string) error { prefix, wildcard := strings.CutSuffix(path, "*") if wildcard && !strings.HasSuffix(prefix, "/") { @@ -238,11 +239,23 @@ func (e *Engine) Evict(ctx context.Context, remoteName, path string) error { if err := e.store.Delete(ctx, storage.IndexKey(remoteName, path)); err != nil { return fmt.Errorf("delete index: %w", err) } - return e.cache.ForgetPath(ctx, remoteName, path) + if err := e.cache.ForgetPath(ctx, remoteName, path); err != nil { + return err + } + if strings.Contains(path, "?") { + return nil + } + return e.evictPrefix(ctx, remoteName, path+"?") } // ponytail: no lock for wildcards; a Fetch already in flight under the // prefix can re-cache its path after the evict. Per-path locks over a // directory would close it if that ever matters. + return e.evictPrefix(ctx, remoteName, prefix) +} + +// evictPrefix drops every cached path starting with prefix, including the +// forwarded-query variants ("path?...") of a single path. +func (e *Engine) evictPrefix(ctx context.Context, remoteName, prefix string) error { if err := e.db.DeleteArtifactsByPrefix(ctx, remoteName, prefix); err != nil { return fmt.Errorf("delete artifacts: %w", err) } diff --git a/internal/proxy/evict_test.go b/internal/proxy/evict_test.go index 15b8def..347de74 100644 --- a/internal/proxy/evict_test.go +++ b/internal/proxy/evict_test.go @@ -269,3 +269,28 @@ func TestEvictWildcardClearsPrefixOnly(t *testing.T) { t.Errorf("path outside prefix = %q, want cached rev1", got) } } + +func TestEvictPathDropsQueryVariants(t *testing.T) { + requireStack(t) + srv, _, _ := changingUpstream(t) + r := rpmRemote(t, "evict-query", srv.URL) + const ( + path = "8/Everything/x86_64/repodata/repomd.xml" + variant = path + "?limit=10" + sibling = "8/Everything/x86_64/repodata/repomd.xml.asc" + ) + for _, p := range []string{path, variant, sibling} { + fetchBody(t, r, p) + } + if err := testEngine.Evict(context.Background(), r.Name, path); err != nil { + t.Fatalf("evict: %v", err) + } + for _, p := range []string{path, variant} { + if c := layers(t, r.Name, p); c != (cached{}) { + t.Errorf("%s after evict = %+v, want every layer gone", p, c) + } + } + if c := layers(t, r.Name, sibling); c == (cached{}) { + t.Errorf("%s sharing the name prefix was evicted", sibling) + } +} -- 2.47.3 From 285e8674ca94ad20420ea57c63c09d1570395f8b Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 12:56:16 +1100 Subject: [PATCH 07/13] Validate and cap ansible pagination params --- internal/provider/ansible/ansible.go | 20 +++++++++++-- internal/provider/ansible/ansible_test.go | 34 ++++++++++++++++++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/internal/provider/ansible/ansible.go b/internal/provider/ansible/ansible.go index ef1e969..464591f 100644 --- a/internal/provider/ansible/ansible.go +++ b/internal/provider/ansible/ansible.go @@ -9,6 +9,7 @@ import ( "encoding/json" "net/http" "net/url" + "strconv" "strings" "git.unkin.net/unkin/artifactapi/internal/auth" @@ -42,12 +43,27 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string { return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") } +const maxPageLimit = 100 + // ForwardedQuery keeps limit/offset pagination on metadata paths only. -func (p *Provider) ForwardedQuery(path string) []string { +func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator { if isArtifact(path) { return nil } - return []string{"limit", "offset"} + return map[string]provider.QueryValidator{"limit": pageLimit, "offset": pageOffset} +} + +func pageOffset(v string) (string, bool) { + n, err := strconv.ParseUint(v, 10, 32) + return strconv.FormatUint(n, 10), err == nil +} + +func pageLimit(v string) (string, bool) { + n, err := strconv.ParseUint(v, 10, 32) + if err != nil || n == 0 { + return "", false + } + return strconv.FormatUint(min(n, maxPageLimit), 10), true } // RewriteResponse points href, download_url and links.* at the remote. Links diff --git a/internal/provider/ansible/ansible_test.go b/internal/provider/ansible/ansible_test.go index 8c96db7..978eacc 100644 --- a/internal/provider/ansible/ansible_test.go +++ b/internal/provider/ansible/ansible_test.go @@ -2,7 +2,6 @@ package ansible import ( "encoding/json" - "reflect" "testing" "git.unkin.net/unkin/artifactapi/internal/provider" @@ -40,12 +39,39 @@ func TestContentTypeIgnoresQuery(t *testing.T) { func TestForwardedQuery(t *testing.T) { p := &Provider{} - if got := p.ForwardedQuery("api/v3/collections/a/b/versions/"); !reflect.DeepEqual(got, []string{"limit", "offset"}) { - t.Errorf("metadata ForwardedQuery = %v", got) - } if got := p.ForwardedQuery(tarball); got != nil { t.Errorf("tarball ForwardedQuery = %v, want nil", got) } + q := p.ForwardedQuery("api/v3/collections/a/b/versions/") + if len(q) != 2 || q["limit"] == nil || q["offset"] == nil { + t.Fatalf("metadata ForwardedQuery keys = %v, want limit and offset", q) + } + tests := []struct { + param, in, want string + ok bool + }{ + {"limit", "10", "10", true}, + {"limit", "007", "7", true}, + {"limit", "100", "100", true}, + {"limit", "5000", "100", true}, + {"limit", "0", "", false}, + {"limit", "", "", false}, + {"limit", "-1", "", false}, + {"limit", "+5", "", false}, + {"limit", "1e3", "", false}, + {"limit", "10 ", "", false}, + {"offset", "0", "0", true}, + {"offset", "000", "0", true}, + {"offset", "0100", "100", true}, + {"offset", "abc", "", false}, + {"offset", "99999999999999999999", "", false}, + } + for _, tt := range tests { + got, ok := q[tt.param](tt.in) + if ok != tt.ok || (ok && got != tt.want) { + t.Errorf("%s(%q) = %q, %v, want %q, %v", tt.param, tt.in, got, ok, tt.want, tt.ok) + } + } } func rewrite(t *testing.T, remote models.Remote, body string) map[string]any { -- 2.47.3 From ba8d3a3c39e92623b435341a428e5fa9b0ff8619 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 03:37:52 +1100 Subject: [PATCH 08/13] Add local ansible Galaxy repositories --- README.md | 2 +- docs/ansible/local.md | 26 ++ e2e-docker/README.md | 3 +- e2e-docker/ansible_local_test.go | 66 +++++ internal/api/v1/ansible.go | 289 +++++++++++++++++++++ internal/api/v1/ansible_handler_test.go | 212 +++++++++++++++ internal/api/v1/gems.go | 4 + internal/api/v1/proxy.go | 12 + internal/provider/ansible/local.go | 328 ++++++++++++++++++++++++ internal/provider/ansible/local_test.go | 196 ++++++++++++++ ui/src/packagetypes/defs/ansible.ts | 14 +- 11 files changed, 1149 insertions(+), 3 deletions(-) create mode 100644 docs/ansible/local.md create mode 100644 e2e-docker/ansible_local_test.go create mode 100644 internal/api/v1/ansible.go create mode 100644 internal/api/v1/ansible_handler_test.go create mode 100644 internal/provider/ansible/local.go create mode 100644 internal/provider/ansible/local_test.go diff --git a/README.md b/README.md index 21e173f..4305324 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Classification precedence, TTLs and eviction for every remote: [docs/common/cach Each type's paths, Terraform examples and local/virtual behaviour are in `docs//`: - `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md) -- `ansible`: [remote](docs/ansible/remote.md) +- `ansible`: [remote](docs/ansible/remote.md), [local](docs/ansible/local.md) - `cargo`: [remote](docs/cargo/remote.md) - `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) - `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) diff --git a/docs/ansible/local.md b/docs/ansible/local.md new file mode 100644 index 0000000..33bd7ad --- /dev/null +++ b/docs/ansible/local.md @@ -0,0 +1,26 @@ +# Local ansible repository + +A local `ansible` repo is a Galaxy v3 server: `ansible-galaxy collection publish` +and `ansible-galaxy collection install` work against `/api/v1/local/`. +`publish` needs an API key; any value works. + +```bash +ansible-galaxy collection build +ansible-galaxy collection publish unkin-hello-1.0.0.tar.gz \ + --server https://artifactapi.example.com/api/v1/local// --api-key unused +ansible-galaxy collection install unkin.hello \ + --server https://artifactapi.example.com/api/v1/local// +``` + +Publishing reads `collection_info` from the tarball's `MANIFEST.json` and +finishes the import synchronously, so the import task is already `completed`. +Uploads over 256 MiB, tarballs with members or symlinks escaping the collection root, or +with an invalid name, version or dependency map are rejected with 400. + +The repo serves `api/v3/collections///`, the paginated +`versions/` list (`limit` up to 100, `offset`), `versions//` with +`download_url`, `artifact.sha256` and `metadata.dependencies`, and +`artifacts/--.tar.gz`. + +Versions are immutable: publishing an existing `namespace.name:version` is +rejected with 409. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index 4cc17ef..71cf793 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -31,7 +31,8 @@ already-running stack. `node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`, second version, republish rejected), gems (stock `ruby:3.3`: `gem push`, `gem install`, `bundle install` + `require`, republish rejected, `gem yank`, - yanked re-push rejected). + yanked re-push rejected), ansible (stock `ansible-galaxy`: `collection build`, + `publish`, republish rejected, `install` + module run). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members; npm packument merge over two locals via a stock `node:22` client (merged versions, install through the member redirect); gems compact diff --git a/e2e-docker/ansible_local_test.go b/e2e-docker/ansible_local_test.go new file mode 100644 index 0000000..feb1b13 --- /dev/null +++ b/e2e-docker/ansible_local_test.go @@ -0,0 +1,66 @@ +//go:build dockere2e + +package e2edocker + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// ansibleHelloScript builds unkin.e2e_hello 1.0.0 with one module into /w/dist. +const ansibleHelloScript = `pip install -q ansible-core >/dev/null 2>&1 +mkdir -p /w && cd /w && ansible-galaxy collection init unkin.e2e_hello >/dev/null +cd unkin/e2e_hello && sed -i 's/^version: .*/version: 1.0.0/' galaxy.yml && mkdir -p plugins/modules +printf '%s\n' 'DOCUMENTATION = r"""' 'module: hello' 'short_description: e2e greeting' 'description: Returns a greeting.' 'options:' ' name: {type: str, default: world, description: Who to greet.}' '"""' \ + 'from ansible.module_utils.basic import AnsibleModule' \ + 'm = AnsibleModule(argument_spec=dict(name=dict(type="str", default="world")))' \ + 'm.exit_json(changed=False, msg="hello " + m.params["name"])' > plugins/modules/hello.py +ansible-galaxy collection build --output-path /w/dist >/dev/null +` + +// ansibleHelloPlay runs unkin.e2e_hello.hello from collections in /w/coll. +const ansibleHelloPlay = `export ANSIBLE_COLLECTIONS_PATH=/w/coll +ansible-doc unkin.e2e_hello.hello | grep -q 'Returns a greeting' +printf -- '- hosts: localhost\n gather_facts: false\n tasks:\n - unkin.e2e_hello.hello: {name: e2e}\n register: r\n - debug: {msg: "PLAY {{ r.msg }}"}\n' > /w/play.yml +ansible-playbook -i localhost, -c local /w/play.yml | grep -o 'PLAY hello e2e' | head -1 +` + +// TestLocalAnsiblePublishInstall builds a collection with a stock +// ansible-galaxy client, publishes it to a local ansible repo, rejects the +// republish, installs it back and runs its module. +func TestLocalAnsiblePublishInstall(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := os.Getenv("ARTIFACTAPI_INTERNAL") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + name := fmt.Sprintf("e2e-ansible-local-%d", time.Now().UnixNano()) + createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","repo_type":"local"}`, name)) + defer deleteRepo(t, name) + + script := "set -euo pipefail\n" + ansibleHelloScript + ` +ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused +if ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused; then echo "republish accepted"; exit 1; fi +ansible-galaxy collection install unkin.e2e_hello:==1.0.0 --server "$SRV" -p /w/coll +` + ansibleHelloPlay + + srv := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/" + out, err := exec.Command("docker", "run", "--rm", "--network", network, + "-e", "SRV="+srv, "python:3.12-slim", "bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("ansible-galaxy client run failed: %v\n%s", err, out) + } + for _, want := range []string{"successfully published and imported", "HTTP Code: 409", "unkin.e2e_hello:1.0.0 was installed successfully", "PLAY hello e2e"} { + if !strings.Contains(string(out), want) { + t.Fatalf("missing %q in client output:\n%s", want, out) + } + } +} diff --git a/internal/api/v1/ansible.go b/internal/api/v1/ansible.go new file mode 100644 index 0000000..da45a3c --- /dev/null +++ b/internal/api/v1/ansible.go @@ -0,0 +1,289 @@ +package v1 + +import ( + "bytes" + "cmp" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/provider/ansible" + "git.unkin.net/unkin/artifactapi/internal/virtual" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +var ansibleMaxPublishBytes int64 = 256 << 20 + +// ansibleError writes a Galaxy v3 error document, which ansible-galaxy prints. +func ansibleError(w http.ResponseWriter, status int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{"errors": []map[string]string{{ + "status": strconv.Itoa(status), "code": strings.ToLower(strings.ReplaceAll(http.StatusText(status), " ", "_")), + "title": http.StatusText(status), "detail": msg, + }}}) +} + +func writeAnsibleJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Vary", "Host, X-Forwarded-Proto") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func isAnsiblePublish(path string) bool { + return strings.TrimSuffix(path, "/")+"/" == ansible.PublishPath +} + +func (h *ProxyHandler) handleAnsiblePublish(w http.ResponseWriter, r *http.Request) { + remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName")) + if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageAnsible { + ansibleError(w, http.StatusMethodNotAllowed, "publish is only supported on local ansible repositories") + return + } + base, ok := requestBaseURL(w, r) + if !ok { + return + } + r.Body = http.MaxBytesReader(w, r.Body, ansibleMaxPublishBytes) + file, sum, err := readAnsibleUpload(r) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + ansibleError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("upload exceeds %d bytes", ansibleMaxPublishBytes)) + return + } + ansibleError(w, http.StatusBadRequest, err.Error()) + return + } + digest := sha256.Sum256(file) + if got := hex.EncodeToString(digest[:]); sum != "" && !strings.EqualFold(sum, got) { + ansibleError(w, http.StatusBadRequest, "sha256 does not match the uploaded file") + return + } + e, err := ansible.ParseCollection(bytes.NewReader(file)) + if err != nil { + ansibleError(w, http.StatusBadRequest, err.Error()) + return + } + e.SHA256, e.Size, e.CreatedAt = hex.EncodeToString(digest[:]), int64(len(file)), time.Now().UTC() + + id := e.Namespace + "." + e.Name + ":" + e.Version + unlock, err := h.engine.Lock(r.Context(), remote.Name, "_ansiblelock/"+e.TaskID(), gemsLockWait, 5*time.Minute) + if err != nil { + ansibleError(w, http.StatusServiceUnavailable, id+" is being published, retry") + return + } + defer unlock() + conflict := func() { + ansibleError(w, http.StatusConflict, id+" already exists; overwrites are not allowed") + } + if f, err := h.db.GetLocalFile(r.Context(), remote.Name, e.IndexPath()); err != nil { + ansibleError(w, http.StatusInternalServerError, err.Error()) + return + } else if f != nil { + conflict() + return + } + // The version is not live, so a tarball left by a failed publish is safe to drop. + artifact := ansible.ArtifactPrefix + e.Filename() + fail := func(status int, msg string) { + if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, artifact); err != nil { + slog.Error("ansible publish cleanup failed", "repo", remote.Name, "path", artifact, "error", err) + } + ansibleError(w, status, msg) + } + if ok, err := h.npmStoreImmutable(r.Context(), remote.Name, artifact, file, "application/gzip"); err != nil { + fail(http.StatusInternalServerError, err.Error()) + return + } else if !ok { + fail(http.StatusConflict, id+" already exists; overwrites are not allowed") + return + } + entry, err := json.Marshal(e) + if err != nil { + fail(http.StatusInternalServerError, err.Error()) + return + } + // The index entry gates visibility, so it is written last. + if _, err := h.npmStore(r.Context(), remote.Name, e.IndexPath(), entry, "application/json", false); err != nil { + if errors.Is(err, database.ErrAlreadyExists) { + conflict() + return + } + fail(http.StatusInternalServerError, err.Error()) + return + } + slog.Info("local ansible collection published", "repo", remote.Name, "collection", id) + repoBase := base + "/api/v1/local/" + url.PathEscape(remote.Name) + writeAnsibleJSON(w, http.StatusAccepted, map[string]string{"task": repoBase + "/api/v3/imports/collections/" + e.TaskID() + "/"}) +} + +// readAnsibleUpload returns the multipart "file" part, which ansible-galaxy +// base64-encodes, and the optional "sha256" field sent with it. +func readAnsibleUpload(r *http.Request) ([]byte, string, error) { + mr, err := r.MultipartReader() + if err != nil { + return nil, "", errors.New("expected a multipart/form-data upload") + } + var file []byte + var sum string + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return nil, "", err + } + switch part.FormName() { + case "file": + if file != nil { + return nil, "", errors.New("multiple file parts") + } + var src io.Reader = part + switch enc := strings.ToLower(part.Header.Get("Content-Transfer-Encoding")); enc { + case "base64": + src = base64.NewDecoder(base64.StdEncoding, part) + case "", "binary", "7bit", "8bit": + default: + return nil, "", fmt.Errorf("unsupported Content-Transfer-Encoding %q", enc) + } + if file, err = io.ReadAll(src); err != nil { + return nil, "", err + } + case "sha256": + b, err := io.ReadAll(io.LimitReader(part, 128)) + if err != nil { + return nil, "", err + } + sum = strings.TrimSpace(string(b)) + } + } + if file == nil { + return nil, "", errors.New("missing file part") + } + return file, sum, nil +} + +// serveLocalAnsible answers the Galaxy v3 API of a local ansible repo. +func (h *ProxyHandler) serveLocalAnsible(w http.ResponseWriter, r *http.Request, remote *models.Remote, ap ansible.APIPath) { + base, ok := requestBaseURL(w, r) + if !ok { + return + } + body, err := h.localAnsibleDoc(r.Context(), remote.Name, ap, r.URL.Query(), base+"/api/v1/local/"+url.PathEscape(remote.Name)) + if errors.Is(err, virtual.ErrNotFound) { + ansibleError(w, http.StatusNotFound, "not found") + return + } + if err != nil { + ansibleError(w, http.StatusInternalServerError, err.Error()) + return + } + w.Header().Set("X-Artifact-Source", "local") + writeAnsibleJSON(w, http.StatusOK, body) +} + +// LocalAnsible serves a local ansible repo's collection, full versions list +// or version detail to ansible virtuals, with root-relative URLs. +func (h *ProxyHandler) LocalAnsible(ctx context.Context, repo, path string) ([]byte, error) { + doc, err := h.localAnsibleDoc(ctx, repo, ansible.ParseAPIPath(path), nil, "/api/v1/local/"+url.PathEscape(repo)) + if err != nil { + return nil, err + } + return json.Marshal(doc) +} + +// localAnsibleDoc builds the document for ap; a nil q lists every version. +func (h *ProxyHandler) localAnsibleDoc(ctx context.Context, repo string, ap ansible.APIPath, q url.Values, base string) (any, error) { + switch ap.Kind { + case ansible.APIRoot: + return ansible.Root(), nil + case ansible.APIImport: + ns, rest, _ := strings.Cut(ap.TaskID, "-") + name, version, _ := strings.Cut(rest, "-") + entries, err := h.localAnsibleEntries(ctx, repo, ns, name) + if err != nil { + return nil, err + } + for _, e := range entries { + if e.Version == version { + return ansible.ImportTask(e), nil + } + } + case ansible.APICollection, ansible.APIVersions, ansible.APIVersion: + entries, err := h.localAnsibleEntries(ctx, repo, ap.Namespace, ap.Name) + if err != nil || len(entries) == 0 { + return nil, cmp.Or(err, virtual.ErrNotFound) + } + refs := make([]ansible.VersionRef, len(entries)) + for i, e := range entries { + refs[i] = e.Ref() + } + switch ap.Kind { + case ansible.APICollection: + return ansible.CollectionDoc(base, ap.Namespace, ap.Name, refs), nil + case ansible.APIVersions: + limit, offset := len(refs), 0 + if q != nil { + limit, offset = ansible.Page(q) + } + return ansible.VersionsPage(base, ap.Namespace, ap.Name, refs, limit, offset), nil + } + for _, e := range entries { + if e.Version == ap.Version { + return ansible.VersionDoc(base, e), nil + } + } + } + return nil, virtual.ErrNotFound +} + +// localAnsibleEntries returns a collection's versions, newest publish first. +func (h *ProxyHandler) localAnsibleEntries(ctx context.Context, repo, ns, name string) ([]ansible.Entry, error) { + if !ansible.NameRe.MatchString(ns) || !ansible.NameRe.MatchString(name) { + return nil, nil + } + prefix := ansible.IndexPrefix + ns + "/" + name + "/" + files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix) + if err != nil { + return nil, err + } + slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(b.ID, a.ID) }) + var out []ansible.Entry + for _, f := range files { + // LIKE treats '_' as a wildcard, so the prefix match can over-select. + if !strings.HasPrefix(f.FilePath, prefix) { + continue + } + raw, err := h.readLocalBlob(ctx, f.ContentHash) + if err != nil { + return nil, err + } + var e ansible.Entry + if err := json.Unmarshal(raw, &e); err != nil { + return nil, fmt.Errorf("decode %s: %w", f.FilePath, err) + } + out = append(out, e) + } + return out, nil +} diff --git a/internal/api/v1/ansible_handler_test.go b/internal/api/v1/ansible_handler_test.go new file mode 100644 index 0000000..b3b48e5 --- /dev/null +++ b/internal/api/v1/ansible_handler_test.go @@ -0,0 +1,212 @@ +package v1 + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func testCollection(t *testing.T, ns, name, version string) []byte { + t.Helper() + manifest := `{"collection_info":{"namespace":"` + ns + `","name":"` + name + `","version":"` + version + `","dependencies":{"ansible.posix":">=1.0.0"}}}` + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + _ = tw.WriteHeader(&tar.Header{Name: "MANIFEST.json", Mode: 0o644, Size: int64(len(manifest))}) + _, _ = tw.Write([]byte(manifest)) + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +// ansiblePublish uploads like ansible-galaxy: the file part base64-encoded +// whenever a sha256 field is sent, raw otherwise. +func ansiblePublish(t *testing.T, url string, tarball []byte, sum string) (int, string) { + t.Helper() + var body bytes.Buffer + mw := multipart.NewWriter(&body) + hdr := textproto.MIMEHeader{"Content-Disposition": {`form-data; name="file"; filename="c.tar.gz"`}} + data := tarball + if sum != "" { + _ = mw.WriteField("sha256", sum) + hdr.Set("Content-Transfer-Encoding", "base64") + data = []byte(base64.StdEncoding.EncodeToString(tarball)) + } + fw, _ := mw.CreatePart(hdr) + _, _ = fw.Write(data) + _ = mw.Close() + req, _ := http.NewRequest(http.MethodPost, url, &body) + req.Header.Set("Content-Type", mw.FormDataContentType()) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + var b bytes.Buffer + _, _ = b.ReadFrom(resp.Body) + return resp.StatusCode, b.String() +} + +func getJSON(t *testing.T, url string, v any) { + t.Helper() + code, body := npmDo(t, http.MethodGet, url, "") + if code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", url, code, body) + } + if err := json.Unmarshal([]byte(body), v); err != nil { + t.Fatalf("GET %s: %v: %s", url, err, body) + } +} + +func TestAnsiblePublishAndServe(t *testing.T) { + h, srv := npmTestServer(t) + repo := npmRepo(t, h, models.PackageAnsible) + base := srv.URL + "/api/v1/local/" + repo + "/" + publish := base + "api/v3/artifacts/collections/" + + v1 := testCollection(t, "unkin", "e2e_hello", "1.0.0") + code, body := ansiblePublish(t, publish, v1, sha256Hex(v1)) + if code != http.StatusAccepted { + t.Fatalf("publish = %d: %s", code, body) + } + var task struct{ Task string } + _ = json.Unmarshal([]byte(body), &task) + if task.Task != base+"api/v3/imports/collections/unkin-e2e_hello-1.0.0/" { + t.Fatalf("task = %q", task.Task) + } + var st map[string]any + getJSON(t, task.Task, &st) + if st["state"] != "completed" || st["finished_at"] == "" { + t.Fatalf("task = %v", st) + } + if code, _ := npmDo(t, http.MethodGet, base+"api/v3/imports/collections/unkin-e2e_hello-9.9.9/", ""); code != http.StatusNotFound { + t.Errorf("unknown task = %d, want 404", code) + } + + for _, v := range []string{"1.1.0", "2.0.0"} { + tb := testCollection(t, "unkin", "e2e_hello", v) + if code, body := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted { + t.Fatalf("publish %s = %d: %s", v, code, body) + } + } + if code, body := ansiblePublish(t, publish, v1, ""); code != http.StatusConflict || !strings.Contains(body, "already exists") { + t.Errorf("identical republish = %d: %s, want 409", code, body) + } + if code, _ := ansiblePublish(t, publish, testCollection(t, "unkin", "e2e_hello", "1.0.0")[:20], ""); code != http.StatusBadRequest { + t.Errorf("truncated tarball = %d, want 400", code) + } + + var root map[string]map[string]string + getJSON(t, base+"api/", &root) + if root["available_versions"]["v3"] != "v3/" { + t.Fatalf("root = %v", root) + } + + var page struct { + Meta struct{ Count int } + Links struct{ Next, Previous *string } + Data []struct{ Version, Href string } + } + getJSON(t, base+"api/v3/collections/unkin/e2e_hello/versions/?limit=2", &page) + if page.Meta.Count != 3 || len(page.Data) != 2 || page.Data[0].Version != "2.0.0" || page.Links.Next == nil || page.Links.Previous != nil { + t.Fatalf("page 1 = %+v", page) + } + next := *page.Links.Next + page.Links.Next = nil + getJSON(t, next, &page) + if len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next != nil { + t.Fatalf("page 2 (%s) = %+v", next, page) + } + + var detail struct { + DownloadURL string `json:"download_url"` + Artifact struct{ Sha256 string } + Metadata struct{ Dependencies map[string]string } + Namespace struct{ Name string } + Collection struct{ Name string } + } + getJSON(t, page.Data[0].Href, &detail) + if detail.DownloadURL != base+"artifacts/unkin-e2e_hello-1.0.0.tar.gz" || detail.Artifact.Sha256 != sha256Hex(v1) || + detail.Metadata.Dependencies["ansible.posix"] != ">=1.0.0" || detail.Namespace.Name != "unkin" || detail.Collection.Name != "e2e_hello" { + t.Fatalf("detail = %+v", detail) + } + if code, got := npmDo(t, http.MethodGet, detail.DownloadURL, ""); code != http.StatusOK || got != string(v1) { + t.Fatalf("download = %d (%d bytes)", code, len(got)) + } + + var coll map[string]any + getJSON(t, base+"api/v3/collections/unkin/e2e_hello/", &coll) + if coll["versions_url"] != base+"api/v3/collections/unkin/e2e_hello/versions/" || coll["updated_at"] == "" { + t.Fatalf("collection = %v", coll) + } + for _, p := range []string{"api/v3/collections/unkin/nope/", "api/v3/collections/unkin/e2e_hello/versions/9.0.0/", "_index/unkin/e2e_hello/1.0.0", "other"} { + if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound { + t.Errorf("GET %s = %d, want 404", p, code) + } + } +} + +func TestAnsiblePublishRejects(t *testing.T) { + h, srv := npmTestServer(t) + repo := npmRepo(t, h, models.PackageAnsible) + publish := srv.URL + "/api/v1/local/" + repo + "/api/v3/artifacts/collections/" + tb := testCollection(t, "unkin", "rej", "1.0.0") + + if code, _ := ansiblePublish(t, publish, tb, strings.Repeat("0", 64)); code != http.StatusBadRequest { + t.Errorf("sha256 mismatch = %d, want 400", code) + } + if code, _ := npmDo(t, http.MethodPost, publish, "raw"); code != http.StatusBadRequest { + t.Errorf("non-multipart = %d, want 400", code) + } + old := ansibleMaxPublishBytes + ansibleMaxPublishBytes = 64 + if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusRequestEntityTooLarge { + t.Errorf("oversize = %d, want 413", code) + } + ansibleMaxPublishBytes = old + generic := npmRepo(t, h, models.PackageGeneric) + if code, _ := ansiblePublish(t, srv.URL+"/api/v1/local/"+generic+"/api/v3/artifacts/collections/", tb, ""); code != http.StatusMethodNotAllowed { + t.Errorf("publish to generic local = %d, want 405", code) + } + if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted { + t.Errorf("publish after rejects = %d, want 202", code) + } +} + +func TestAnsibleLocalPerHost(t *testing.T) { + h, srv := npmTestServer(t) + repo := npmRepo(t, h, models.PackageAnsible) + if code, b := ansiblePublish(t, srv.URL+"/api/v1/local/"+repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "host", "1.0.0"), ""); code != http.StatusAccepted { + t.Fatalf("publish = %d: %s", code, b) + } + routes := h.Routes() + path := "/local/" + repo + "/api/v3/collections/unkin/host/versions/1.0.0/" + r := httptest.NewRequest(http.MethodGet, path, nil) + r.Host = "x@evil.com" + w := httptest.NewRecorder() + routes.ServeHTTP(w, r) + if w.Code != http.StatusBadRequest { + t.Errorf("bad Host = %d, want 400", w.Code) + } + r = httptest.NewRequest(http.MethodGet, path, nil) + r.Host = "aa.example" + r.Header.Set("X-Forwarded-Proto", "https") + w = httptest.NewRecorder() + routes.ServeHTTP(w, r) + if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"download_url":"https://aa.example/api/v1/local/`+repo+`/artifacts/unkin-host-1.0.0.tar.gz"`) { + t.Fatalf("detail = %d: %s", w.Code, w.Body) + } + if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto" { + t.Errorf("Cache-Control %q Vary %q", cc, vary) + } +} diff --git a/internal/api/v1/gems.go b/internal/api/v1/gems.go index 912777f..db9e447 100644 --- a/internal/api/v1/gems.go +++ b/internal/api/v1/gems.go @@ -64,6 +64,10 @@ func (h *ProxyHandler) lockGemVersion(w http.ResponseWriter, r *http.Request, re } func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) { + if isAnsiblePublish(chi.URLParam(r, "*")) { + h.handleAnsiblePublish(w, r) + return + } if chi.URLParam(r, "*") != "api/v1/gems" { gemsText(w, http.StatusNotFound, "not found") return diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 486394b..dded049 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -17,6 +17,7 @@ import ( v2 "git.unkin.net/unkin/artifactapi/internal/api/v2" "git.unkin.net/unkin/artifactapi/internal/database" "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/provider/ansible" "git.unkin.net/unkin/artifactapi/internal/proxy" "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/virtual" @@ -275,6 +276,17 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) { } } + if remote.PackageType == models.PackageAnsible { + if ap := ansible.ParseAPIPath(path); ap.Kind != ansible.APIUnknown { + h.serveLocalAnsible(w, r, remote, ap) + return + } + if !strings.HasPrefix(path, ansible.ArtifactPrefix) { + http.Error(w, "not found", http.StatusNotFound) + return + } + } + prov, _ := provider.Get(remote.PackageType) if indexer, ok := prov.(provider.LocalIndexer); ok { if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) { diff --git a/internal/provider/ansible/local.go b/internal/provider/ansible/local.go new file mode 100644 index 0000000..db50164 --- /dev/null +++ b/internal/provider/ansible/local.go @@ -0,0 +1,328 @@ +package ansible + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "path" + "regexp" + "strconv" + "strings" + "time" +) + +// Local ansible repos speak the Galaxy v3 publish and install API. Per version: +// artifacts/--.tar.gz collection tarball (immutable) +// _index/// Entry JSON; gates visibility + +const ( + IndexPrefix = "_index/" + ArtifactPrefix = "artifacts/" + PublishPath = "api/v3/artifacts/collections/" +) + +var ( + NameRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`) + VersionRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`) +) + +var ( + MaxManifestBytes int64 = 1 << 20 + MaxScanBytes int64 = 512 << 20 +) + +// Entry is one published collection version. +type Entry struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + Version string `json:"version"` + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + Metadata json.RawMessage `json:"metadata"` + CreatedAt time.Time `json:"created_at"` +} + +func (e Entry) Filename() string { return e.Namespace + "-" + e.Name + "-" + e.Version + ".tar.gz" } + +// IndexPath is the entry's hidden index file. +func (e Entry) IndexPath() string { return IndexPrefix + e.Namespace + "/" + e.Name + "/" + e.Version } + +// TaskID names the import task returned on publish. +func (e Entry) TaskID() string { return e.Namespace + "-" + e.Name + "-" + e.Version } + +// ParseCollection reads collection_info from a collection tarball's top-level +// MANIFEST.json. Every member is checked, so a tarball whose paths or links +// escape the collection root is rejected even after the manifest. +func ParseCollection(r io.Reader) (*Entry, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, fmt.Errorf("not a gzip tarball: %w", err) + } + scan := &io.LimitedReader{R: zr, N: MaxScanBytes} + tr := tar.NewReader(scan) + var e *Entry + for { + hdr, err := tr.Next() + if scan.N <= 0 { + return nil, errors.New("collection exceeds decompressed size limit") + } + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("read tarball: %w", err) + } + name, err := memberPath(hdr) + if err != nil { + return nil, err + } + if name != "MANIFEST.json" || hdr.Typeflag != tar.TypeReg { + continue + } + if e != nil { + return nil, errors.New("duplicate MANIFEST.json") + } + if e, err = parseManifest(io.LimitReader(tr, MaxManifestBytes+1)); err != nil { + return nil, err + } + } + if e == nil { + return nil, errors.New("no MANIFEST.json in collection tarball") + } + return e, nil +} + +// memberPath returns hdr's cleaned name, rejecting absolute or escaping +// paths and links, and anything but files, directories and symlinks. +func memberPath(hdr *tar.Header) (string, error) { + name := path.Clean(strings.TrimPrefix(hdr.Name, "./")) + if escapes(name) { + return "", fmt.Errorf("tarball member %q escapes the collection root", hdr.Name) + } + switch hdr.Typeflag { + case tar.TypeReg, tar.TypeDir: + case tar.TypeSymlink: + if path.IsAbs(hdr.Linkname) || escapes(path.Join(path.Dir(name), hdr.Linkname)) { + return "", fmt.Errorf("tarball symlink %q escapes the collection root", hdr.Name) + } + default: + return "", fmt.Errorf("tarball member %q has unsupported type %q", hdr.Name, hdr.Typeflag) + } + return name, nil +} + +func escapes(p string) bool { + return path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") || strings.Contains(p, "\\") +} + +func parseManifest(r io.Reader) (*Entry, error) { + raw, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("read MANIFEST.json: %w", err) + } + if int64(len(raw)) > MaxManifestBytes { + return nil, fmt.Errorf("MANIFEST.json exceeds %d bytes", MaxManifestBytes) + } + var doc struct { + CollectionInfo map[string]json.RawMessage `json:"collection_info"` + } + if err := json.Unmarshal(raw, &doc); err != nil || doc.CollectionInfo == nil { + return nil, errors.New("MANIFEST.json has no collection_info") + } + info := doc.CollectionInfo + e := &Entry{} + for k, dst := range map[string]*string{"namespace": &e.Namespace, "name": &e.Name, "version": &e.Version} { + if err := json.Unmarshal(info[k], dst); err != nil { + return nil, fmt.Errorf("collection_info.%s is not a string", k) + } + } + if !NameRe.MatchString(e.Namespace) || !NameRe.MatchString(e.Name) || !VersionRe.MatchString(e.Version) { + return nil, fmt.Errorf("invalid collection %q.%q version %q", e.Namespace, e.Name, e.Version) + } + deps := map[string]string{} + if d := info["dependencies"]; len(d) > 0 && string(d) != "null" { + if err := json.Unmarshal(d, &deps); err != nil { + return nil, errors.New("collection_info.dependencies must map collection names to version ranges") + } + } + for k := range deps { + ns, n, ok := strings.Cut(k, ".") + if !ok || !NameRe.MatchString(ns) || !NameRe.MatchString(n) { + return nil, fmt.Errorf("invalid dependency %q", k) + } + } + info["dependencies"], _ = json.Marshal(deps) + if e.Metadata, err = json.Marshal(info); err != nil { + return nil, err + } + return e, nil +} + +// APIPath is a parsed Galaxy v3 request path below a repo. +type APIPath struct { + Kind APIKind + Namespace, Name, Version string + TaskID string +} + +type APIKind int + +const ( + APIUnknown APIKind = iota + APIRoot + APICollection + APIVersions + APIVersion + APIImport +) + +// ParseAPIPath parses api/, api/v3/collections///[versions/[/]] +// and api/v3/imports/collections//, with or without trailing slashes. +func ParseAPIPath(p string) APIPath { + p = strings.TrimSuffix(p, "/") + if p == "api" { + return APIPath{Kind: APIRoot} + } + if id, ok := strings.CutPrefix(p, "api/v3/imports/collections/"); ok && id != "" && !strings.Contains(id, "/") { + return APIPath{Kind: APIImport, TaskID: id} + } + rest, ok := strings.CutPrefix(p, "api/v3/collections/") + if !ok { + return APIPath{} + } + segs := strings.Split(rest, "/") + if len(segs) < 2 || !NameRe.MatchString(segs[0]) || !NameRe.MatchString(segs[1]) { + return APIPath{} + } + ap := APIPath{Namespace: segs[0], Name: segs[1]} + switch { + case len(segs) == 2: + ap.Kind = APICollection + case len(segs) == 3 && segs[2] == "versions": + ap.Kind = APIVersions + case len(segs) == 4 && segs[2] == "versions" && VersionRe.MatchString(segs[3]): + ap.Kind, ap.Version = APIVersion, segs[3] + default: + return APIPath{} + } + return ap +} + +// Root is the API root ansible-galaxy discovers. +func Root() map[string]any { + return map[string]any{"available_versions": map[string]string{"v3": "v3/"}} +} + +func CollectionPath(ns, name string) string { return "/api/v3/collections/" + ns + "/" + name + "/" } + +func VersionPath(ns, name, version string) string { + return CollectionPath(ns, name) + "versions/" + url.PathEscape(version) + "/" +} + +// VersionRef is one entry of a versions list. +type VersionRef struct { + Version string `json:"version"` + CreatedAt string `json:"created_at,omitempty"` + UpdatedAt string `json:"updated_at,omitempty"` +} + +func (e Entry) Ref() VersionRef { + t := e.CreatedAt.UTC().Format(time.RFC3339Nano) + return VersionRef{Version: e.Version, CreatedAt: t, UpdatedAt: t} +} + +// CollectionDoc describes a collection from its versions; base is the repo +// URL the hrefs are built on. +func CollectionDoc(base, ns, name string, refs []VersionRef) map[string]any { + created, updated := "", "" + for _, r := range refs { + if created == "" || r.CreatedAt < created { + created = r.CreatedAt + } + updated = max(updated, r.UpdatedAt) + } + return map[string]any{ + "href": base + CollectionPath(ns, name), + "namespace": ns, + "name": name, + "deprecated": false, + "versions_url": base + CollectionPath(ns, name) + "versions/", + "created_at": created, + "updated_at": updated, + } +} + +// Page reads limit/offset as the remote forwards them, defaulting to 100/0. +func Page(q url.Values) (limit, offset int) { + limit, offset = maxPageLimit, 0 + if v, ok := pageLimit(q.Get("limit")); ok { + limit, _ = strconv.Atoi(v) + } + if v, ok := pageOffset(q.Get("offset")); ok { + offset, _ = strconv.Atoi(v) + } + return limit, offset +} + +// VersionsPage is one page of refs in Galaxy v3 list form. +func VersionsPage(base, ns, name string, refs []VersionRef, limit, offset int) map[string]any { + list := base + CollectionPath(ns, name) + "versions/" + link := func(off int) any { + return list + "?" + url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}.Encode() + } + offset = min(offset, len(refs)) + var previous, next any + if offset > 0 { + previous = link(max(offset-limit, 0)) + } + if offset+limit < len(refs) { + next = link(offset + limit) + } + data := []map[string]any{} + for _, r := range refs[offset:min(offset+limit, len(refs))] { + data = append(data, map[string]any{ + "version": r.Version, + "href": base + VersionPath(ns, name, r.Version), + "created_at": r.CreatedAt, + "updated_at": r.UpdatedAt, + "marks": []string{}, + }) + } + return map[string]any{ + "meta": map[string]any{"count": len(refs)}, + "links": map[string]any{"first": link(0), "previous": previous, "next": next, "last": link(max(len(refs)-limit, 0))}, + "data": data, + } +} + +// VersionDoc is a version's detail; the tarball is served from the repo at +// artifacts/. +func VersionDoc(base string, e Entry) map[string]any { + t := e.CreatedAt.UTC().Format(time.RFC3339Nano) + return map[string]any{ + "href": base + VersionPath(e.Namespace, e.Name, e.Version), + "namespace": map[string]string{"name": e.Namespace}, + "collection": map[string]string{"name": e.Name}, + "name": e.Name, + "version": e.Version, + "download_url": base + "/" + ArtifactPrefix + e.Filename(), + "artifact": map[string]any{"filename": e.Filename(), "sha256": e.SHA256, "size": e.Size}, + "metadata": e.Metadata, + "signatures": []string{}, + "created_at": t, + "updated_at": t, + } +} + +// ImportTask is a finished import: publishing completes synchronously. +func ImportTask(e Entry) map[string]any { + t := e.CreatedAt.UTC().Format(time.RFC3339Nano) + return map[string]any{ + "id": e.TaskID(), "state": "completed", "created_at": t, "started_at": t, "finished_at": t, + "messages": []any{}, "error": nil, + } +} diff --git a/internal/provider/ansible/local_test.go b/internal/provider/ansible/local_test.go new file mode 100644 index 0000000..db25a7d --- /dev/null +++ b/internal/provider/ansible/local_test.go @@ -0,0 +1,196 @@ +package ansible + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "net/url" + "strings" + "testing" +) + +type member struct { + hdr tar.Header + body string +} + +func collectionTar(t *testing.T, members ...member) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for _, m := range members { + h := m.hdr + if h.Typeflag == 0 { + h.Typeflag = tar.TypeReg + } + if h.Typeflag == tar.TypeReg { + h.Size = int64(len(m.body)) + } + h.Mode = 0o644 + if err := tw.WriteHeader(&h); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(m.body)) + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +func manifest(ns, name, version, deps string) member { + return member{tar.Header{Name: "MANIFEST.json"}, `{"collection_info":{"namespace":"` + ns + `","name":"` + name + + `","version":"` + version + `","authors":["a"],"dependencies":` + deps + `}}`} +} + +func TestParseCollection(t *testing.T) { + e, err := ParseCollection(bytes.NewReader(collectionTar(t, + member{tar.Header{Name: "plugins/", Typeflag: tar.TypeDir}, ""}, + member{tar.Header{Name: "plugins/modules/hello.py"}, "x"}, + member{tar.Header{Name: "docs/link", Typeflag: tar.TypeSymlink, Linkname: "../plugins/modules/hello.py"}, ""}, + manifest("unkin", "e2e_hello", "1.0.0-rc.1+b1", `{"ansible.posix":">=1.0.0"}`), + ))) + if err != nil { + t.Fatal(err) + } + if e.Namespace != "unkin" || e.Name != "e2e_hello" || e.Version != "1.0.0-rc.1+b1" || e.Filename() != "unkin-e2e_hello-1.0.0-rc.1+b1.tar.gz" { + t.Fatalf("entry = %+v", e) + } + var md struct { + Authors []string `json:"authors"` + Dependencies map[string]string `json:"dependencies"` + } + if err := json.Unmarshal(e.Metadata, &md); err != nil || md.Dependencies["ansible.posix"] != ">=1.0.0" || len(md.Authors) != 1 { + t.Fatalf("metadata = %s (%v)", e.Metadata, err) + } + + e, err = ParseCollection(bytes.NewReader(collectionTar(t, manifest("a", "b", "1.0.0", "null")))) + if err != nil || !strings.Contains(string(e.Metadata), `"dependencies":{}`) { + t.Fatalf("null dependencies = %s (%v), want {}", e.Metadata, err) + } +} + +func TestParseCollectionRejects(t *testing.T) { + ok := manifest("a", "b", "1.0.0", "{}") + tests := []struct { + name string + members []member + want string + }{ + {"no manifest", []member{{tar.Header{Name: "README.md"}, "x"}}, "no MANIFEST.json"}, + {"nested manifest only", []member{{tar.Header{Name: "x/MANIFEST.json"}, ok.body}}, "no MANIFEST.json"}, + {"dotdot", []member{ok, {tar.Header{Name: "../evil"}, "x"}}, "escapes"}, + {"dotdot after clean", []member{ok, {tar.Header{Name: "a/../../evil"}, "x"}}, "escapes"}, + {"absolute", []member{ok, {tar.Header{Name: "/etc/passwd"}, "x"}}, "escapes"}, + {"symlink escape", []member{ok, {tar.Header{Name: "a/l", Typeflag: tar.TypeSymlink, Linkname: "../../x"}, ""}}, "symlink"}, + {"absolute symlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeSymlink, Linkname: "/etc"}, ""}}, "symlink"}, + {"hardlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeLink, Linkname: "MANIFEST.json"}, ""}}, "unsupported type"}, + {"duplicate manifest", []member{ok, ok}, "duplicate"}, + {"bad namespace", []member{manifest("Bad", "b", "1.0.0", "{}")}, "invalid collection"}, + {"bad version", []member{manifest("a", "b", "1.0", "{}")}, "invalid collection"}, + {"bad dependency", []member{manifest("a", "b", "1.0.0", `{"nodot":"*"}`)}, "invalid dependency"}, + {"list dependencies", []member{manifest("a", "b", "1.0.0", `["x.y"]`)}, "dependencies must map"}, + {"huge manifest", []member{{tar.Header{Name: "MANIFEST.json"}, strings.Repeat(" ", int(MaxManifestBytes)+1)}}, "exceeds"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ParseCollection(bytes.NewReader(collectionTar(t, tt.members...))) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("err = %v, want %q", err, tt.want) + } + }) + } + if _, err := ParseCollection(strings.NewReader("not gzip")); err == nil { + t.Error("non-gzip input accepted") + } +} + +func TestParseCollectionScanLimit(t *testing.T) { + old := MaxScanBytes + MaxScanBytes = 4096 + defer func() { MaxScanBytes = old }() + _, err := ParseCollection(bytes.NewReader(collectionTar(t, + member{tar.Header{Name: "big"}, strings.Repeat("x", 8192)}, + manifest("a", "b", "1.0.0", "{}"), + ))) + if err == nil || !strings.Contains(err.Error(), "decompressed size limit") { + t.Fatalf("err = %v, want decompressed size limit", err) + } +} + +func TestParseAPIPath(t *testing.T) { + tests := []struct { + in string + want APIPath + }{ + {"api", APIPath{Kind: APIRoot}}, + {"api/", APIPath{Kind: APIRoot}}, + {"api/v3/collections/unkin/e2e_hello/", APIPath{Kind: APICollection, Namespace: "unkin", Name: "e2e_hello"}}, + {"api/v3/collections/unkin/e2e_hello/versions", APIPath{Kind: APIVersions, Namespace: "unkin", Name: "e2e_hello"}}, + {"api/v3/collections/unkin/e2e_hello/versions/1.0.0/", APIPath{Kind: APIVersion, Namespace: "unkin", Name: "e2e_hello", Version: "1.0.0"}}, + {"api/v3/imports/collections/unkin-e2e_hello-1.0.0/", APIPath{Kind: APIImport, TaskID: "unkin-e2e_hello-1.0.0"}}, + {"api/v3/collections/unkin/e2e_hello/versions/1.0/", APIPath{}}, + {"api/v3/collections/../x/", APIPath{}}, + {"api/v3/collections/unkin/e2e_hello/versions/1.0.0/extra/", APIPath{}}, + {"artifacts/unkin-e2e_hello-1.0.0.tar.gz", APIPath{}}, + {"api/v3/imports/collections/a/b/", APIPath{}}, + } + for _, tt := range tests { + if got := ParseAPIPath(tt.in); got != tt.want { + t.Errorf("ParseAPIPath(%q) = %+v, want %+v", tt.in, got, tt.want) + } + } +} + +func TestVersionsPage(t *testing.T) { + refs := []VersionRef{{Version: "3.0.0"}, {Version: "2.0.0"}, {Version: "1.0.0"}} + list := "https://h/r/api/v3/collections/a/b/versions/" + links := func(p map[string]any) map[string]any { return p["links"].(map[string]any) } + versions := func(p map[string]any) (out []string) { + for _, d := range p["data"].([]map[string]any) { + out = append(out, d["version"].(string)) + } + return out + } + + p := VersionsPage("https://h/r", "a", "b", refs, 2, 0) + if got := versions(p); strings.Join(got, ",") != "3.0.0,2.0.0" { + t.Fatalf("page 1 = %v", got) + } + if l := links(p); l["previous"] != nil || l["next"] != list+"?limit=2&offset=2" || l["last"] != list+"?limit=2&offset=1" { + t.Fatalf("page 1 links = %v", l) + } + if p["meta"].(map[string]any)["count"] != 3 { + t.Fatalf("count = %v", p["meta"]) + } + + p = VersionsPage("https://h/r", "a", "b", refs, 2, 2) + if got := versions(p); strings.Join(got, ",") != "1.0.0" { + t.Fatalf("page 2 = %v", got) + } + if l := links(p); l["next"] != nil || l["previous"] != list+"?limit=2&offset=0" { + t.Fatalf("page 2 links = %v", l) + } + + if got := versions(VersionsPage("", "a", "b", refs, 2, 99)); len(got) != 0 { + t.Fatalf("offset past end = %v, want empty", got) + } +} + +func TestPage(t *testing.T) { + for _, tt := range []struct { + q string + limit, offset int + }{ + {"", 100, 0}, + {"limit=5&offset=10", 5, 10}, + {"limit=5000", 100, 0}, + {"limit=0&offset=-1", 100, 0}, + } { + q, _ := url.ParseQuery(tt.q) + if l, o := Page(q); l != tt.limit || o != tt.offset { + t.Errorf("Page(%q) = %d,%d want %d,%d", tt.q, l, o, tt.limit, tt.offset) + } + } +} diff --git a/ui/src/packagetypes/defs/ansible.ts b/ui/src/packagetypes/defs/ansible.ts index 653b9ed..61708b2 100644 --- a/ui/src/packagetypes/defs/ansible.ts +++ b/ui/src/packagetypes/defs/ansible.ts @@ -2,7 +2,19 @@ import type { PackageTypeUI } from '../types'; const def: PackageTypeUI = { color: 'red', - snippets: ({ proxy }) => { + snippets: ({ name, url, proxy, isLocal }) => { + if (isLocal) { + const src = `${url}/api/v1/local/${name}/`; + return [ + { + title: 'Publish and install', + language: 'bash', + code: `ansible-galaxy collection publish --.tar.gz --server ${src} --api-key unused +ansible-galaxy collection install . --server ${src}`, + note: 'publish needs an API key; any value works. Publishing an existing version is rejected.', + }, + ]; + } return [ { title: 'Install collections through this Galaxy proxy', -- 2.47.3 From 4b202b1852be5a256995264f952df6b634e101ba Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 03:45:30 +1100 Subject: [PATCH 09/13] Add ansible virtual repositories --- README.md | 2 +- docs/ansible/virtual.md | 28 ++++ e2e-docker/README.md | 3 +- e2e-docker/ansible_virtual_test.go | 64 ++++++++ internal/api/v1/ansible_handler_test.go | 56 +++++++ internal/api/v1/ansible_virtual.go | 64 ++++++++ internal/api/v1/proxy.go | 4 + internal/server/server.go | 1 + internal/virtual/ansible_merger.go | 179 ++++++++++++++++++++++ internal/virtual/ansible_merger_test.go | 194 ++++++++++++++++++++++++ internal/virtual/engine.go | 15 +- 11 files changed, 604 insertions(+), 6 deletions(-) create mode 100644 docs/ansible/virtual.md create mode 100644 e2e-docker/ansible_virtual_test.go create mode 100644 internal/api/v1/ansible_virtual.go create mode 100644 internal/virtual/ansible_merger.go create mode 100644 internal/virtual/ansible_merger_test.go diff --git a/README.md b/README.md index 4305324..0411434 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Classification precedence, TTLs and eviction for every remote: [docs/common/cach Each type's paths, Terraform examples and local/virtual behaviour are in `docs//`: - `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md) -- `ansible`: [remote](docs/ansible/remote.md), [local](docs/ansible/local.md) +- `ansible`: [remote](docs/ansible/remote.md), [local](docs/ansible/local.md), [virtual](docs/ansible/virtual.md) - `cargo`: [remote](docs/cargo/remote.md) - `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) - `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) diff --git a/docs/ansible/virtual.md b/docs/ansible/virtual.md new file mode 100644 index 0000000..95297a9 --- /dev/null +++ b/docs/ansible/virtual.md @@ -0,0 +1,28 @@ +# `ansible` virtual + +An ansible virtual serves one Galaxy v3 API over local and remote ansible +members: `ansible-galaxy collection install` works against +`/api/v1/virtual//`. + +- `versions/` concatenates member version lists in member order; on a + duplicate version the earlier member wins. Pagination (`limit` up to 100, + `offset`) runs over the merged list. +- `versions//` is the owning member's detail with `href` and + `download_url` pointing at the virtual; the download 302s to that member. +- A member error fails the request with 502; a member without the collection + is skipped. Merges are cached for 60s. + +```hcl +resource "artifactapi_virtual" "ansible" { + name = "ansible" + package_type = "ansible" + members = [artifactapi_local_ansible.internal.name, artifactapi_remote_ansible.galaxy.name] +} +``` + +```bash +ansible-galaxy collection install unkin.hello community.general \ + --server https://artifactapi.example.com/api/v1/virtual/ansible/ +``` + +Publish to the local member directly; the virtual is read-only. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index 71cf793..53658ca 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -37,7 +37,8 @@ already-running stack. across two members; npm packument merge over two locals via a stock `node:22` client (merged versions, install through the member redirect); gems compact index merge over two locals via stock `ruby:3.3` bundler (member order wins, - downloads 302 to the owning member). + downloads 302 to the owning member); ansible over two locals via stock + `ansible-galaxy` (install through the virtual, download 302 to the owner). - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors (pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors (constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary, diff --git a/e2e-docker/ansible_virtual_test.go b/e2e-docker/ansible_virtual_test.go new file mode 100644 index 0000000..2c54bcc --- /dev/null +++ b/e2e-docker/ansible_virtual_test.go @@ -0,0 +1,64 @@ +//go:build dockere2e + +package e2edocker + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// TestVirtualAnsibleInstall publishes a collection to the second of two +// ansible locals and installs it with stock ansible-galaxy through a virtual +// over both, the download 302ing to the owning member. +func TestVirtualAnsibleInstall(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := os.Getenv("ARTIFACTAPI_INTERNAL") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + id := time.Now().UnixNano() + a, b, virt := fmt.Sprintf("e2e-ansible-a-%d", id), fmt.Sprintf("e2e-ansible-b-%d", id), fmt.Sprintf("e2e-ansible-v-%d", id) + for _, name := range []string{a, b} { + createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","repo_type":"local"}`, name)) + defer deleteRepo(t, name) + } + createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","members":[%q,%q]}`, virt, a, b)) + defer deleteVirtual(t, virt) + + base := strings.TrimRight(internal, "/") + script := "set -euo pipefail\n" + ansibleHelloScript + ` +ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$BASE/api/v1/local/$B/" --api-key unused +V="$BASE/api/v1/virtual/$VIRT" +ansible-galaxy collection install unkin.e2e_hello --server "$V/" -p /w/coll +DL=$(python3 -c 'import json,sys,urllib.request as u;print(json.load(u.urlopen(sys.argv[1]))["download_url"])' "$V/api/v3/collections/unkin/e2e_hello/versions/1.0.0/") +echo "REDIRECT $(python3 -c 'import sys,urllib.request as u +class N(u.HTTPRedirectHandler): + def redirect_request(self,*a): return None +try: u.build_opener(N).open(sys.argv[1]) +except u.HTTPError as e: print(e.code, e.headers["Location"])' "$DL")" +` + ansibleHelloPlay + + out, err := exec.Command("docker", "run", "--rm", "--network", network, + "-e", "BASE="+base, "-e", "B="+b, "-e", "VIRT="+virt, + "python:3.12-slim", "bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("ansible-galaxy client run failed: %v\n%s", err, out) + } + for _, want := range []string{ + "unkin.e2e_hello:1.0.0 was installed successfully", + "REDIRECT 302 " + base + "/api/v1/local/" + b + "/artifacts/unkin-e2e_hello-1.0.0.tar.gz", + "PLAY hello e2e", + } { + if !strings.Contains(string(out), want) { + t.Fatalf("missing %q in client output:\n%s", want, out) + } + } +} diff --git a/internal/api/v1/ansible_handler_test.go b/internal/api/v1/ansible_handler_test.go index b3b48e5..d094683 100644 --- a/internal/api/v1/ansible_handler_test.go +++ b/internal/api/v1/ansible_handler_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "context" "encoding/base64" "encoding/json" "mime/multipart" @@ -13,6 +14,7 @@ import ( "strings" "testing" + "git.unkin.net/unkin/artifactapi/internal/virtual" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -210,3 +212,57 @@ func TestAnsibleLocalPerHost(t *testing.T) { t.Errorf("Cache-Control %q Vary %q", cc, vary) } } + +func TestAnsibleVirtualOverLocals(t *testing.T) { + h, srv := npmTestServer(t) + h.virtualEngine = virtual.NewEngine(h.db, nil) + h.virtualEngine.AnsibleLocal = h.LocalAnsible + a, b := npmRepo(t, h, models.PackageAnsible), npmRepo(t, h, models.PackageAnsible) + for _, p := range []struct{ repo, version string }{{a, "1.0.0"}, {b, "1.0.0"}, {b, "2.0.0"}} { + if code, body := ansiblePublish(t, srv.URL+"/api/v1/local/"+p.repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "virt", p.version), ""); code != http.StatusAccepted { + t.Fatalf("publish %s to %s = %d: %s", p.version, p.repo, code, body) + } + } + virt := "v" + a + if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageAnsible, Members: []string{a, b}}); err != nil { + t.Fatal(err) + } + base := srv.URL + "/api/v1/virtual/" + virt + "/" + + var page struct { + Meta struct{ Count int } + Links struct{ Next *string } + Data []struct{ Version, Href string } + } + getJSON(t, base+"api/v3/collections/unkin/virt/versions/?limit=1", &page) + if page.Meta.Count != 2 || len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next == nil { + t.Fatalf("page 1 = %+v", page) + } + href := page.Data[0].Href + getJSON(t, *page.Links.Next, &page) + if len(page.Data) != 1 || page.Data[0].Version != "2.0.0" || page.Links.Next != nil { + t.Fatalf("page 2 = %+v", page) + } + + var detail struct { + DownloadURL string `json:"download_url"` + } + getJSON(t, href, &detail) + if detail.DownloadURL != base+a+"/artifacts/unkin-virt-1.0.0.tar.gz" { + t.Fatalf("download_url = %q, want the first member via the virtual", detail.DownloadURL) + } + noFollow := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + resp, err := noFollow.Get(detail.DownloadURL) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != srv.URL+"/api/v1/local/"+a+"/artifacts/unkin-virt-1.0.0.tar.gz" { + t.Fatalf("download = %d -> %q", resp.StatusCode, resp.Header.Get("Location")) + } + for _, p := range []string{"api/v3/collections/unkin/nope/", "notamember/x.tar.gz"} { + if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound { + t.Errorf("GET %s = %d, want 404", p, code) + } + } +} diff --git a/internal/api/v1/ansible_virtual.go b/internal/api/v1/ansible_virtual.go new file mode 100644 index 0000000..919fea9 --- /dev/null +++ b/internal/api/v1/ansible_virtual.go @@ -0,0 +1,64 @@ +package v1 + +import ( + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/url" + + "git.unkin.net/unkin/artifactapi/internal/provider/ansible" + "git.unkin.net/unkin/artifactapi/internal/virtual" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// serveVirtualAnsible serves the merged Galaxy v3 API and 302s downloads to +// the member owning that version. +func (h *ProxyHandler) serveVirtualAnsible(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) { + ap := ansible.ParseAPIPath(path) + base := proxyBaseURL + "/api/v1/virtual/" + url.PathEscape(virt.Name) + var doc any + var err error + switch ap.Kind { + case ansible.APIRoot: + doc = ansible.Root() + case ansible.APIVersion: + var body []byte + if body, err = h.virtualEngine.AnsibleVersion(r.Context(), virt, ap.Namespace, ap.Name, ap.Version, proxyBaseURL); err == nil { + doc = json.RawMessage(body) + } + case ansible.APICollection, ansible.APIVersions: + var d *virtual.AnsibleDoc + if d, err = h.virtualEngine.AnsibleVersions(r.Context(), virt, ap.Namespace, ap.Name); err == nil { + if ap.Kind == ansible.APICollection { + doc = ansible.CollectionDoc(base, ap.Namespace, ap.Name, d.Refs) + } else { + limit, offset := ansible.Page(r.URL.Query()) + doc = ansible.VersionsPage(base, ap.Namespace, ap.Name, d.Refs, limit, offset) + } + } + default: + loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL) + switch { + case err != nil: + ansibleError(w, http.StatusBadRequest, err.Error()) + case !ok: + ansibleError(w, http.StatusNotFound, "not found") + default: + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Vary", "Host, X-Forwarded-Proto") + http.Redirect(w, r, loc, http.StatusFound) + } + return + } + switch { + case errors.Is(err, virtual.ErrNotFound): + ansibleError(w, http.StatusNotFound, "not found") + case err != nil: + slog.Error("virtual ansible fetch failed", "virtual", virt.Name, "path", path, "error", err) + ansibleError(w, http.StatusBadGateway, "bad gateway") + default: + w.Header().Set("X-Artifact-Source", "virtual") + writeAnsibleJSON(w, http.StatusOK, doc) + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index dded049..5ec1741 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -211,6 +211,10 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) { h.serveVirtualGems(w, r, *virt, path, proxyBaseURL) return } + if virt.PackageType == models.PackageAnsible { + h.serveVirtualAnsible(w, r, *virt, path, proxyBaseURL) + return + } loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL) if err != nil { diff --git a/internal/server/server.go b/internal/server/server.go index 26a4953..5cdacc0 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -168,6 +168,7 @@ func (s *Server) routes() chi.Router { proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler) s.virtEngine.NPMLocal = proxyHandler.LocalNPMPackument s.virtEngine.GemsLocal = proxyHandler.LocalGemsIndex + s.virtEngine.AnsibleLocal = proxyHandler.LocalAnsible r.Mount("/api/v1", proxyHandler.Routes()) r.Mount("/v2", proxyHandler.DockerV2Routes()) diff --git a/internal/virtual/ansible_merger.go b/internal/virtual/ansible_merger.go new file mode 100644 index 0000000..181387d --- /dev/null +++ b/internal/virtual/ansible_merger.go @@ -0,0 +1,179 @@ +package virtual + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider/ansible" + "git.unkin.net/unkin/artifactapi/internal/proxy" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +const ( + maxAnsibleCollections = 1024 + ansibleMemberPage = 100 + maxAnsibleMemberPages = 100 +) + +// AnsibleMember is one member's versions of a collection; Found is false when +// the member does not have it. +type AnsibleMember struct { + RemoteName string + Found bool + Refs []ansible.VersionRef +} + +// AnsibleDoc is a collection's merged versions; Owners maps each version to +// the member serving it. +type AnsibleDoc struct { + Refs []ansible.VersionRef + Owners map[string]string + body []byte +} + +func (d *AnsibleDoc) index() []byte { return d.body } +func (d *AnsibleDoc) files() map[string][]byte { return nil } + +// MergeAnsibleVersions concatenates member versions in member order; the +// first member wins a duplicate version. ErrNotFound means no member has the +// collection. +func MergeAnsibleVersions(members []AnsibleMember) (*AnsibleDoc, error) { + d := &AnsibleDoc{Refs: []ansible.VersionRef{}, Owners: map[string]string{}} + found := false + for _, m := range members { + found = found || m.Found + for _, r := range m.Refs { + if _, dup := d.Owners[r.Version]; dup { + continue + } + d.Owners[r.Version] = m.RemoteName + d.Refs = append(d.Refs, r) + } + } + if !found { + return nil, ErrNotFound + } + var err error + d.body, err = json.Marshal(d.Refs) + return d, err +} + +// AnsibleVersions returns the merged versions of ns.name across members. +func (e *Engine) AnsibleVersions(ctx context.Context, virt models.Virtual, ns, name string) (*AnsibleDoc, error) { + prefix := "ansible\x00" + virt.Name + "\x00" + inserted := false + repo, err := e.merged(ctx, prefix+ns+"/"+name, func(ctx context.Context) (mergedRepo, error) { + inserted = true + members, err := fetchMembers(virt, func(m string) (*AnsibleMember, error) { return e.ansibleMember(ctx, m, ns, name) }) + if err != nil { + return nil, err + } + return MergeAnsibleVersions(members) + }) + if err != nil { + return nil, err + } + if inserted { + e.evictOldest(prefix, maxAnsibleCollections) + } + return repo.(*AnsibleDoc), nil +} + +// AnsibleVersion serves the owning member's version detail with href and +// download_url pointing at the virtual; the download 302s to the member. +// Member documents carry root-relative URLs, so anything else is rejected. +func (e *Engine) AnsibleVersion(ctx context.Context, virt models.Virtual, ns, name, version, proxyBaseURL string) ([]byte, error) { + d, err := e.AnsibleVersions(ctx, virt, ns, name) + if err != nil { + return nil, err + } + owner, ok := d.Owners[version] + if !ok { + return nil, ErrNotFound + } + remote, err := e.getRemote(ctx, owner) + if err != nil { + return nil, fmt.Errorf("remote %q: %w", owner, err) + } + body, err := e.ansibleDoc(ctx, *remote, ansible.VersionPath(ns, name, version)[1:]) + if err != nil { + return nil, fmt.Errorf("member %q: %v", owner, err) + } + var detail map[string]any + if err := json.Unmarshal(body, &detail); err != nil { + return nil, fmt.Errorf("member %q version detail: %v", owner, err) + } + dl, _ := detail["download_url"].(string) + u, err := url.Parse(dl) + if err != nil { + return nil, fmt.Errorf("member %q download_url %q: %v", owner, dl, err) + } + rest, ok := strings.CutPrefix(u.EscapedPath(), "/api/v1/"+string(remote.RepoType)+"/"+url.PathEscape(owner)+"/") + if !ok || u.Scheme != "" || u.Host != "" { + return nil, fmt.Errorf("member %q download_url %q is not served by the member", owner, dl) + } + base := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/virtual/" + url.PathEscape(virt.Name) + detail["download_url"] = base + "/" + url.PathEscape(owner) + "/" + rest + detail["href"] = base + ansible.VersionPath(ns, name, version) + return json.Marshal(detail) +} + +// ansibleMember reads every version a member lists for ns.name, following +// the member's pagination for remotes. +func (e *Engine) ansibleMember(ctx context.Context, member, ns, name string) (*AnsibleMember, error) { + remote, err := e.getRemote(ctx, member) + if err != nil { + return nil, fmt.Errorf("remote %q: %w", member, err) + } + m := &AnsibleMember{RemoteName: member} + list := ansible.CollectionPath(ns, name)[1:] + "versions/" + for page := 0; page < maxAnsibleMemberPages; page++ { + // Locals list every version at once; remote pages match the keys the + // remote caches client requests under. + path := list + if remote.RepoType != models.RepoTypeLocal { + path += "?limit=" + strconv.Itoa(ansibleMemberPage) + if page > 0 { + path += "&offset=" + strconv.Itoa(len(m.Refs)) + } + } + body, err := e.ansibleDoc(ctx, *remote, path) + var pe *proxy.ProxyError + if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound { + return m, nil + } + if err != nil { + return nil, err + } + var doc struct { + Meta struct{ Count int } + Data []ansible.VersionRef + } + if err := json.Unmarshal(body, &doc); err != nil { + return nil, fmt.Errorf("%s/%s: %v", member, path, err) + } + m.Found = true + m.Refs = append(m.Refs, doc.Data...) + if remote.RepoType == models.RepoTypeLocal || len(doc.Data) == 0 || len(m.Refs) >= doc.Meta.Count { + return m, nil + } + } + return nil, fmt.Errorf("%s: more than %d pages of %s.%s versions", member, maxAnsibleMemberPages, ns, name) +} + +// ansibleMemberDoc reads a Galaxy API document from a member. +func (e *Engine) ansibleMemberDoc(ctx context.Context, remote models.Remote, path string) ([]byte, error) { + if remote.RepoType != models.RepoTypeLocal { + return e.fetchMemberPath(ctx, remote, path) + } + if e.AnsibleLocal == nil { + return nil, errors.New("local ansible indexes are not wired") + } + return e.AnsibleLocal(ctx, remote.Name, path) +} diff --git a/internal/virtual/ansible_merger_test.go b/internal/virtual/ansible_merger_test.go new file mode 100644 index 0000000..4365f5b --- /dev/null +++ b/internal/virtual/ansible_merger_test.go @@ -0,0 +1,194 @@ +package virtual + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "sync" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider/ansible" + "git.unkin.net/unkin/artifactapi/internal/proxy" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func refs(vs ...string) []ansible.VersionRef { + out := make([]ansible.VersionRef, len(vs)) + for i, v := range vs { + out[i] = ansible.VersionRef{Version: v} + } + return out +} + +func TestMergeAnsibleVersionsMemberOrderWins(t *testing.T) { + d, err := MergeAnsibleVersions([]AnsibleMember{ + {RemoteName: "internal", Found: true, Refs: refs("2.0.0", "1.0.0")}, + {RemoteName: "missing"}, + {RemoteName: "galaxy", Found: true, Refs: refs("3.0.0", "2.0.0", "0.9.0")}, + }) + if err != nil { + t.Fatal(err) + } + var got []string + for _, r := range d.Refs { + got = append(got, r.Version) + } + if strings.Join(got, ",") != "2.0.0,1.0.0,3.0.0,0.9.0" { + t.Fatalf("merged = %v", got) + } + for v, owner := range map[string]string{"2.0.0": "internal", "1.0.0": "internal", "3.0.0": "galaxy", "0.9.0": "galaxy"} { + if d.Owners[v] != owner { + t.Errorf("owner of %s = %q, want %q", v, d.Owners[v], owner) + } + } + if _, err := MergeAnsibleVersions([]AnsibleMember{{RemoteName: "a"}, {RemoteName: "b"}}); !errors.Is(err, ErrNotFound) { + t.Fatalf("no member has it: want ErrNotFound, got %v", err) + } + if d, err := MergeAnsibleVersions([]AnsibleMember{{RemoteName: "a", Found: true}}); err != nil || len(d.Refs) != 0 { + t.Fatalf("found with no versions = %v %v", d, err) + } +} + +// galaxyPages answers a remote's versions list like Galaxy, two per page. +func galaxyPages(versions []string, path string) ([]byte, error) { + _, q, _ := strings.Cut(path, "?") + offset := 0 + if _, off, ok := strings.Cut(q, "offset="); ok { + _, _ = fmt.Sscan(off, &offset) + } + end := min(offset+2, len(versions)) + return json.Marshal(map[string]any{"meta": map[string]int{"count": len(versions)}, "data": refs(versions[min(offset, end):end]...)}) +} + +func ansibleEngine(doc func(remote models.Remote, path string) ([]byte, error)) *Engine { + return &Engine{ + mergeTTL: time.Minute, + getRemote: func(_ context.Context, name string) (*models.Remote, error) { + rt := models.RepoTypeRemote + if name == "internal" { + rt = models.RepoTypeLocal + } + return &models.Remote{Name: name, RepoType: rt}, nil + }, + ansibleDoc: func(_ context.Context, remote models.Remote, path string) ([]byte, error) { return doc(remote, path) }, + } +} + +func ansibleVirt(members ...string) models.Virtual { + return models.Virtual{Name: "ansible", PackageType: models.PackageAnsible, Members: members} +} + +func TestAnsibleVersionsFollowsRemotePages(t *testing.T) { + var mu sync.Mutex + var asked []string + e := ansibleEngine(func(remote models.Remote, path string) ([]byte, error) { + mu.Lock() + asked = append(asked, remote.Name+" "+path) + mu.Unlock() + if remote.Name == "internal" { + return galaxyPages([]string{"1.0.0", "5.0.0"}, "") + } + return galaxyPages([]string{"5.0.0", "4.0.0", "3.0.0", "2.0.0", "1.5.0"}, path) + }) + virt := ansibleVirt("internal", "galaxy") + d, err := e.AnsibleVersions(context.Background(), virt, "community", "general") + if err != nil { + t.Fatal(err) + } + if len(d.Refs) != 6 || d.Owners["5.0.0"] != "internal" || d.Owners["1.5.0"] != "galaxy" { + t.Fatalf("merged = %+v", d) + } + want := map[string]bool{ + "internal api/v3/collections/community/general/versions/": true, + "galaxy api/v3/collections/community/general/versions/?limit=100": true, + "galaxy api/v3/collections/community/general/versions/?limit=100&offset=2": true, + "galaxy api/v3/collections/community/general/versions/?limit=100&offset=4": true, + } + if len(asked) != len(want) { + t.Fatalf("asked %v", asked) + } + for _, a := range asked { + if !want[a] { + t.Errorf("unexpected fetch %q", a) + } + } + if _, err := e.AnsibleVersions(context.Background(), virt, "community", "general"); err != nil || len(asked) != len(want) { + t.Fatalf("merge not cached: %v, %d fetches", err, len(asked)) + } +} + +func TestAnsibleVersionsFailsClosed(t *testing.T) { + e := ansibleEngine(func(remote models.Remote, path string) ([]byte, error) { + switch remote.Name { + case "internal": + return nil, ErrNotFound + case "gone": + return nil, &proxy.ProxyError{Status: http.StatusNotFound} + } + return nil, errors.New("upstream down") + }) + if _, err := e.AnsibleVersions(context.Background(), ansibleVirt("internal", "gone"), "a", "b"); !errors.Is(err, ErrNotFound) { + t.Fatalf("all members lack it: want ErrNotFound, got %v", err) + } + if _, err := e.AnsibleVersions(context.Background(), ansibleVirt("internal", "galaxy"), "a", "b"); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("member error: want a non-404 error, got %v", err) + } +} + +func TestAnsibleVersionRoutesViaVirtual(t *testing.T) { + detail := func(dl string) []byte { + b, _ := json.Marshal(map[string]any{"version": "1.0.0", "download_url": dl, "artifact": map[string]string{"sha256": "ab"}}) + return b + } + dls := map[string]string{ + "internal": "/api/v1/local/internal/artifacts/a-b-1.0.0.tar.gz", + "galaxy": "/api/v1/remote/galaxy/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-2.0.0.tar.gz", + "evil": "https://evil.example/api/v1/remote/evil/x.tar.gz", + "other": "/api/v1/remote/someoneelse/x.tar.gz", + } + e := ansibleEngine(func(remote models.Remote, path string) ([]byte, error) { + if strings.Contains(path, "/versions/?") || strings.HasSuffix(path, "/versions/") { + v := map[string]string{"internal": "1.0.0", "galaxy": "2.0.0", "evil": "3.0.0", "other": "4.0.0"}[remote.Name] + return galaxyPages([]string{v}, path) + } + return detail(dls[remote.Name]), nil + }) + virt := ansibleVirt("internal", "galaxy", "evil", "other") + for v, want := range map[string]string{ + "1.0.0": "https://h/api/v1/virtual/ansible/internal/artifacts/a-b-1.0.0.tar.gz", + "2.0.0": "https://h/api/v1/virtual/ansible/galaxy/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-2.0.0.tar.gz", + } { + body, err := e.AnsibleVersion(context.Background(), virt, "a", "b", v, "https://h") + if err != nil { + t.Fatal(err) + } + var got struct { + Href string + DownloadURL string `json:"download_url"` + Artifact struct{ Sha256 string } + } + _ = json.Unmarshal(body, &got) + if got.DownloadURL != want || got.Href != "https://h/api/v1/virtual/ansible/api/v3/collections/a/b/versions/"+v+"/" || got.Artifact.Sha256 != "ab" { + t.Errorf("%s detail = %+v", v, got) + } + loc, ok, err := e.MemberRedirect(context.Background(), virt, strings.TrimPrefix(want, "https://h/api/v1/virtual/ansible/"), "", "https://h") + if err != nil || !ok || loc != "https://h"+dls[map[string]string{"1.0.0": "internal", "2.0.0": "galaxy"}[v]] { + t.Errorf("%s redirect = %q %v %v", v, loc, ok, err) + } + } + for _, v := range []string{"3.0.0", "4.0.0"} { + if _, err := e.AnsibleVersion(context.Background(), virt, "a", "b", v, "https://h"); err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("%s: foreign download_url accepted: %v", v, err) + } + } + if _, err := e.AnsibleVersion(context.Background(), virt, "a", "b", "9.9.9", "https://h"); !errors.Is(err, ErrNotFound) { + t.Errorf("unknown version: want ErrNotFound, got %v", err) + } + if _, _, err := e.MemberRedirect(context.Background(), virt, "galaxy/../internal/x", "", "https://h"); !errors.Is(err, ErrBadPath) { + t.Errorf("escaping redirect: want ErrBadPath, got %v", err) + } +} diff --git a/internal/virtual/engine.go b/internal/virtual/engine.go index 99457e2..ed332c9 100644 --- a/internal/virtual/engine.go +++ b/internal/virtual/engine.go @@ -38,6 +38,11 @@ type Engine struct { // ErrNotFound. GemsLocal func(ctx context.Context, repo, path string) ([]byte, error) + // AnsibleLocal returns a local ansible repo's collection, full versions list + // or version detail with root-relative URLs, or ErrNotFound. + AnsibleLocal func(ctx context.Context, repo, path string) ([]byte, error) + ansibleDoc func(ctx context.Context, remote models.Remote, path string) ([]byte, error) + mergeTTL time.Duration sf singleflight.Group @@ -68,6 +73,7 @@ func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine { e.rpmMember = e.fetchRPMMember e.npmMember = e.fetchNPMMember e.gemsMember = e.fetchGemsMember + e.ansibleDoc = e.ansibleMemberDoc return e } @@ -192,16 +198,17 @@ var ( ErrBadPath = errors.New("invalid path") ) -// MemberRedirect maps an rpm or npm virtual package path (/, as -// written by MergeRPM/MergeNPM) to the owning member's route. ok is false for -// any other path; ErrBadPath means the path is absolute or escapes the member. +// MemberRedirect maps an rpm, npm or ansible virtual package path +// (/, as written by MergeRPM/MergeNPM/AnsibleVersion) to the +// owning member's route. ok is false for any other path; ErrBadPath means the +// path is absolute or escapes the member. func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) { switch virt.PackageType { case models.PackageRPM: if strings.HasPrefix(path, "repodata/") { return "", false, nil } - case models.PackageNPM: + case models.PackageNPM, models.PackageAnsible: default: return "", false, nil } -- 2.47.3 From 583a3ad4d1950406e6a79d0e14f46db7e42956c5 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 03:57:40 +1100 Subject: [PATCH 10/13] List ansible in the per-type caching docs --- docs/common/caching.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/common/caching.md b/docs/common/caching.md index 20fd733..223cc75 100644 --- a/docs/common/caching.md +++ b/docs/common/caching.md @@ -93,7 +93,7 @@ Force a moving path mutable, e.g. branch tarballs on `generic`: ## Built-in classification per type Each type's page lists its auto-detected mutable and immutable paths: -[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), +[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), [ansible](../ansible/remote.md), [cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md), [docker](../docker/remote.md), [generic](../generic/remote.md), [gems](../gems/remote.md), [goproxy](../goproxy/remote.md), [helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md), -- 2.47.3 From 7f7ea21c3b75b1f36d759333b19a2620413c376f Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 03:57:46 +1100 Subject: [PATCH 11/13] Emit root-relative Galaxy pagination links on local ansible repos --- internal/api/v1/ansible_handler_test.go | 4 ++ internal/provider/ansible/local.go | 37 ++++++++++---- internal/provider/ansible/local_test.go | 64 ++++++++++++++++++++++++- 3 files changed, 94 insertions(+), 11 deletions(-) diff --git a/internal/api/v1/ansible_handler_test.go b/internal/api/v1/ansible_handler_test.go index b3b48e5..cf4624f 100644 --- a/internal/api/v1/ansible_handler_test.go +++ b/internal/api/v1/ansible_handler_test.go @@ -122,6 +122,10 @@ func TestAnsiblePublishAndServe(t *testing.T) { t.Fatalf("page 1 = %+v", page) } next := *page.Links.Next + if !strings.HasPrefix(next, "/api/v1/local/"+repo+"/") { + t.Fatalf("next = %q, want root-relative", next) + } + next = srv.URL + next page.Links.Next = nil getJSON(t, next, &page) if len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next != nil { diff --git a/internal/provider/ansible/local.go b/internal/provider/ansible/local.go index db50164..988f9df 100644 --- a/internal/provider/ansible/local.go +++ b/internal/provider/ansible/local.go @@ -238,12 +238,14 @@ func (e Entry) Ref() VersionRef { // CollectionDoc describes a collection from its versions; base is the repo // URL the hrefs are built on. func CollectionDoc(base, ns, name string, refs []VersionRef) map[string]any { - created, updated := "", "" + var created, updated time.Time for _, r := range refs { - if created == "" || r.CreatedAt < created { - created = r.CreatedAt + if t, err := time.Parse(time.RFC3339Nano, r.CreatedAt); err == nil && (created.IsZero() || t.Before(created)) { + created = t + } + if t, err := time.Parse(time.RFC3339Nano, r.UpdatedAt); err == nil && t.After(updated) { + updated = t } - updated = max(updated, r.UpdatedAt) } return map[string]any{ "href": base + CollectionPath(ns, name), @@ -251,11 +253,18 @@ func CollectionDoc(base, ns, name string, refs []VersionRef) map[string]any { "name": name, "deprecated": false, "versions_url": base + CollectionPath(ns, name) + "versions/", - "created_at": created, - "updated_at": updated, + "created_at": formatTime(created), + "updated_at": formatTime(updated), } } +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339Nano) +} + // Page reads limit/offset as the remote forwards them, defaulting to 100/0. func Page(q url.Values) (limit, offset int) { limit, offset = maxPageLimit, 0 @@ -268,9 +277,11 @@ func Page(q url.Values) (limit, offset int) { return limit, offset } -// VersionsPage is one page of refs in Galaxy v3 list form. +// VersionsPage is one page of refs in Galaxy v3 list form. Links are +// root-relative like Galaxy's, since ansible-galaxy splices them onto the +// server URL. func VersionsPage(base, ns, name string, refs []VersionRef, limit, offset int) map[string]any { - list := base + CollectionPath(ns, name) + "versions/" + list := rootPath(base) + CollectionPath(ns, name) + "versions/" link := func(off int) any { return list + "?" + url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}.Encode() } @@ -294,11 +305,19 @@ func VersionsPage(base, ns, name string, refs []VersionRef, limit, offset int) m } return map[string]any{ "meta": map[string]any{"count": len(refs)}, - "links": map[string]any{"first": link(0), "previous": previous, "next": next, "last": link(max(len(refs)-limit, 0))}, + "links": map[string]any{"first": link(0), "previous": previous, "next": next, "last": link(max(len(refs)-1, 0) / limit * limit)}, "data": data, } } +// rootPath drops the scheme and host from base. +func rootPath(base string) string { + if u, err := url.Parse(base); err == nil { + return u.EscapedPath() + } + return base +} + // VersionDoc is a version's detail; the tarball is served from the repo at // artifacts/. func VersionDoc(base string, e Entry) map[string]any { diff --git a/internal/provider/ansible/local_test.go b/internal/provider/ansible/local_test.go index db25a7d..e23dee2 100644 --- a/internal/provider/ansible/local_test.go +++ b/internal/provider/ansible/local_test.go @@ -5,6 +5,7 @@ import ( "bytes" "compress/gzip" "encoding/json" + "fmt" "net/url" "strings" "testing" @@ -145,7 +146,7 @@ func TestParseAPIPath(t *testing.T) { func TestVersionsPage(t *testing.T) { refs := []VersionRef{{Version: "3.0.0"}, {Version: "2.0.0"}, {Version: "1.0.0"}} - list := "https://h/r/api/v3/collections/a/b/versions/" + list := "/r/api/v3/collections/a/b/versions/" links := func(p map[string]any) map[string]any { return p["links"].(map[string]any) } versions := func(p map[string]any) (out []string) { for _, d := range p["data"].([]map[string]any) { @@ -158,7 +159,7 @@ func TestVersionsPage(t *testing.T) { if got := versions(p); strings.Join(got, ",") != "3.0.0,2.0.0" { t.Fatalf("page 1 = %v", got) } - if l := links(p); l["previous"] != nil || l["next"] != list+"?limit=2&offset=2" || l["last"] != list+"?limit=2&offset=1" { + if l := links(p); l["previous"] != nil || l["next"] != list+"?limit=2&offset=2" || l["last"] != list+"?limit=2&offset=2" { t.Fatalf("page 1 links = %v", l) } if p["meta"].(map[string]any)["count"] != 3 { @@ -178,6 +179,65 @@ func TestVersionsPage(t *testing.T) { } } +// TestVersionsPageFollow pages through more than 100 versions the way +// ansible-galaxy does: old clients splice next onto the versions URL in place +// of its path, new ones urljoin it onto the server. +func TestVersionsPageFollow(t *testing.T) { + var refs []VersionRef + for i := 104; i >= 0; i-- { + refs = append(refs, VersionRef{Version: fmt.Sprintf("1.0.%d", i)}) + } + server := "https://h:8443/api/v1/local/r%2Bx/" + base := strings.TrimSuffix(server, "/") + versionsURL := server + "api/v3/collections/a/b/versions/" + for _, follow := range map[string]func(string) string{ + "replace": func(link string) string { + u, _ := url.Parse(versionsURL) + return strings.Replace(versionsURL, u.EscapedPath(), link, 1) + }, + "urljoin": func(link string) string { + s, _ := url.Parse(server) + l, _ := url.Parse(link) + return s.ResolveReference(l).String() + }, + } { + var got []string + next := versionsURL + "?limit=100" + for pages := 0; next != ""; pages++ { + if pages > 3 { + t.Fatal("pagination does not terminate") + } + u, err := url.Parse(next) + if err != nil || u.Host != "h:8443" || !strings.HasPrefix(u.EscapedPath(), "/api/v1/local/r%2Bx/api/v3/") { + t.Fatalf("followed link %q is not on the server", next) + } + limit, offset := Page(u.Query()) + p := VersionsPage(base, "a", "b", refs, limit, offset) + for _, d := range p["data"].([]map[string]any) { + got = append(got, d["version"].(string)) + } + next = "" + if l, ok := p["links"].(map[string]any)["next"].(string); ok { + next = follow(l) + } + } + if len(got) != 105 || got[0] != "1.0.104" || got[104] != "1.0.0" { + t.Fatalf("collected %d versions, first %v", len(got), got[:1]) + } + } +} + +func TestCollectionDocTimes(t *testing.T) { + d := CollectionDoc("", "a", "b", []VersionRef{ + {CreatedAt: "2026-01-01T00:00:05Z", UpdatedAt: "2026-01-01T00:00:05Z"}, + {CreatedAt: "2026-01-01T00:00:05.5Z", UpdatedAt: "2026-01-01T00:00:05.5Z"}, + {CreatedAt: "2026-01-01T00:00:04.9Z", UpdatedAt: "2026-01-01T00:00:04.9Z"}, + }) + if d["created_at"] != "2026-01-01T00:00:04.9Z" || d["updated_at"] != "2026-01-01T00:00:05.5Z" { + t.Fatalf("created/updated = %v/%v", d["created_at"], d["updated_at"]) + } +} + func TestPage(t *testing.T) { for _, tt := range []struct { q string -- 2.47.3 From c1a5feb64e037cbd6ae0efa5cd2806eedaaaacd2 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 04:02:08 +1100 Subject: [PATCH 12/13] Fail virtual ansible merges on a short member versions list --- internal/virtual/ansible_merger.go | 5 ++++- internal/virtual/ansible_merger_test.go | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/virtual/ansible_merger.go b/internal/virtual/ansible_merger.go index 181387d..a16d624 100644 --- a/internal/virtual/ansible_merger.go +++ b/internal/virtual/ansible_merger.go @@ -160,9 +160,12 @@ func (e *Engine) ansibleMember(ctx context.Context, member, ns, name string) (*A } m.Found = true m.Refs = append(m.Refs, doc.Data...) - if remote.RepoType == models.RepoTypeLocal || len(doc.Data) == 0 || len(m.Refs) >= doc.Meta.Count { + if remote.RepoType == models.RepoTypeLocal || len(m.Refs) >= doc.Meta.Count { return m, nil } + if len(doc.Data) == 0 { + return nil, fmt.Errorf("%s/%s: empty page after %d of %d versions", member, path, len(m.Refs), doc.Meta.Count) + } } return nil, fmt.Errorf("%s: more than %d pages of %s.%s versions", member, maxAnsibleMemberPages, ns, name) } diff --git a/internal/virtual/ansible_merger_test.go b/internal/virtual/ansible_merger_test.go index 4365f5b..96c7695 100644 --- a/internal/virtual/ansible_merger_test.go +++ b/internal/virtual/ansible_merger_test.go @@ -139,6 +139,18 @@ func TestAnsibleVersionsFailsClosed(t *testing.T) { } } +func TestAnsibleVersionsShortRemote(t *testing.T) { + e := ansibleEngine(func(_ models.Remote, path string) ([]byte, error) { + if strings.Contains(path, "offset=") { + return json.Marshal(map[string]any{"meta": map[string]int{"count": 3}, "data": []any{}}) + } + return galaxyPages([]string{"3.0.0", "2.0.0", "1.0.0"}, path) + }) + if _, err := e.AnsibleVersions(context.Background(), ansibleVirt("galaxy"), "a", "b"); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("empty page before count: want a non-404 error, got %v", err) + } +} + func TestAnsibleVersionRoutesViaVirtual(t *testing.T) { detail := func(dl string) []byte { b, _ := json.Marshal(map[string]any{"version": "1.0.0", "download_url": dl, "artifact": map[string]string{"sha256": "ab"}}) -- 2.47.3 From 41008cd8f69b1770b5400dea2fa2f2caf54d5ec4 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 18 Oct 2026 04:04:38 +1100 Subject: [PATCH 13/13] Expect root-relative pagination links from ansible virtuals --- internal/api/v1/ansible_handler_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/api/v1/ansible_handler_test.go b/internal/api/v1/ansible_handler_test.go index 38594b0..f3996c7 100644 --- a/internal/api/v1/ansible_handler_test.go +++ b/internal/api/v1/ansible_handler_test.go @@ -243,7 +243,10 @@ func TestAnsibleVirtualOverLocals(t *testing.T) { t.Fatalf("page 1 = %+v", page) } href := page.Data[0].Href - getJSON(t, *page.Links.Next, &page) + if !strings.HasPrefix(*page.Links.Next, "/api/v1/virtual/"+virt+"/") { + t.Fatalf("next = %q, want root-relative", *page.Links.Next) + } + getJSON(t, srv.URL+*page.Links.Next, &page) if len(page.Data) != 1 || page.Data[0].Version != "2.0.0" || page.Links.Next != nil { t.Fatalf("page 2 = %+v", page) } -- 2.47.3