Forward ansible pagination params and rewrite only Galaxy link fields

2026-10-17 12:35:03 +11:00
parent 7bb11b4761
commit af6a088ef4
3 changed files with 147 additions and 42 deletions
@@ -4,9 +4,11 @@
 package ansible

 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"net/http"
+	"net/url"
 	"strings"

 	"git.unkin.net/unkin/artifactapi/internal/auth"
@@ -40,26 +42,82 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
 	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
 }

-// ForwardQuery keeps limit/offset pagination in the proxied path.
-func (p *Provider) ForwardedQuery(string) []string { return nil }
+// ForwardedQuery keeps limit/offset pagination on metadata paths only.
+func (p *Provider) ForwardedQuery(path string) []string {
+	if isArtifact(path) {
+		return nil
+	}
+	return []string{"limit", "offset"}
+}

-// RewriteResponse points root-relative hrefs/links and absolute download_url
-// values at the remote. Links stay root-relative, which ansible-galaxy joins
-// onto the server URL, so the cached body is independent of the request host.
+// RewriteResponse points href, download_url and links.* at the remote. Links
+// stay root-relative, which ansible-galaxy joins onto the server URL, so the
+// cached body is independent of the request host.
 func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
-	if !json.Valid(body) {
+	dec := json.NewDecoder(bytes.NewReader(body))
+	dec.UseNumber()
+	var doc any
+	if err := dec.Decode(&doc); err != nil {
 		return nil, nil
 	}
-	prefix := `"/api/v1/remote/` + remote.Name + `/`
-	content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`)
-	content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix)
-	return []byte(content), nil
+	r := rewriter{
+		prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/",
+		base:   strings.TrimRight(remote.BaseURL, "/") + "/",
+	}
+	r.walk(doc)
+	var out bytes.Buffer
+	enc := json.NewEncoder(&out)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(doc); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
 }

 func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
 	return auth.BasicHeaders(remote), nil
 }

-func isArtifact(path string) bool {
-	return strings.HasSuffix(path, ".tar.gz")
+type rewriter struct{ prefix, base string }
+
+func (r rewriter) walk(v any) {
+	switch v := v.(type) {
+	case []any:
+		for _, e := range v {
+			r.walk(e)
+		}
+	case map[string]any:
+		for k, e := range v {
+			switch k {
+			case "href", "download_url":
+				v[k] = r.link(e)
+			case "links":
+				if links, ok := e.(map[string]any); ok {
+					for lk, l := range links {
+						links[lk] = r.link(l)
+					}
+					continue
+				}
+			}
+			r.walk(v[k])
+		}
+	}
+}
+
+func (r rewriter) link(v any) any {
+	s, ok := v.(string)
+	switch {
+	case !ok:
+		return v
+	case strings.HasPrefix(s, "/api/"):
+		return r.prefix + s[1:]
+	case strings.HasPrefix(s, r.base):
+		return r.prefix + strings.TrimPrefix(s, r.base)
+	}
+	return s
+}
+
+func isArtifact(path string) bool {
+	path, _, _ = strings.Cut(path, "?")
+	return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz")
 }
@@ -2,12 +2,15 @@ package ansible

 import (
 	"encoding/json"
+	"reflect"
 	"testing"

 	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )

+const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz"
+
 func TestClassify(t *testing.T) {
 	p := &Provider{}
 	tests := []struct {
@@ -16,8 +19,11 @@ func TestClassify(t *testing.T) {
 	}{
 		{"api/", provider.Mutable},
 		{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
+		{"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable},
 		{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
-		{"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable},
+		{"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable},
+		{tarball, provider.Immutable},
+		{tarball + "?limit=1", provider.Immutable},
 	}
 	for _, tt := range tests {
 		if got := p.Classify(tt.path); got != tt.want {
@@ -26,48 +32,87 @@ func TestClassify(t *testing.T) {
 	}
 }

-func TestUpstreamURLKeepsQuery(t *testing.T) {
-	p := &Provider{}
-	got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100")
-	want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100"
-	if got != want {
-		t.Errorf("UpstreamURL = %q, want %q", got, want)
+func TestContentTypeIgnoresQuery(t *testing.T) {
+	if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" {
+		t.Errorf("ContentType = %q, want application/json", got)
 	}
 }

-func TestRewriteResponse(t *testing.T) {
+func TestForwardedQuery(t *testing.T) {
 	p := &Provider{}
-	remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
-	body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` +
-		`"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` +
-		`"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` +
-		`"metadata":{"repository":"https://github.com/a/b"}}`
+	if got := p.ForwardedQuery("api/v3/collections/a/b/versions/"); !reflect.DeepEqual(got, []string{"limit", "offset"}) {
+		t.Errorf("metadata ForwardedQuery = %v", got)
+	}
+	if got := p.ForwardedQuery(tarball); got != nil {
+		t.Errorf("tarball ForwardedQuery = %v, want nil", got)
+	}
+}

-	out, err := p.RewriteResponse([]byte(body), remote, "")
+func rewrite(t *testing.T, remote models.Remote, body string) map[string]any {
+	t.Helper()
+	out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "")
 	if err != nil {
 		t.Fatal(err)
 	}
-	var got struct {
-		Links       map[string]any `json:"links"`
-		Href        string         `json:"href"`
-		DownloadURL string         `json:"download_url"`
-		Metadata    map[string]any `json:"metadata"`
-	}
+	var got map[string]any
 	if err := json.Unmarshal(out, &got); err != nil {
-		t.Fatalf("rewritten body is not JSON: %v", err)
+		t.Fatalf("rewritten body is not JSON: %v: %s", err, out)
 	}
+	return got
+}
+
+func TestRewriteResponse(t *testing.T) {
+	remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
+	got := rewrite(t, remote, `{
+		"links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null},
+		"data": [{
+			"href": "/api/v3/collections/a/b/versions/1.0.0/",
+			"download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",
+			"description": "/api/ is where it lives",
+			"docs": "/api/v3/not-a-link/",
+			"metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"}
+		}]
+	}`)
 	pre := "/api/v1/remote/galaxy/"
-	if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want {
-		t.Errorf("next = %v, want %q", got.Links["next"], want)
+	links := got["links"].(map[string]any)
+	if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want {
+		t.Errorf("links.next = %v, want %q", links["next"], want)
 	}
-	if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want {
-		t.Errorf("href = %q, want %q", got.Href, want)
+	if links["previous"] != nil {
+		t.Errorf("links.previous = %v, want null", links["previous"])
 	}
-	if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want {
-		t.Errorf("download_url = %q, want %q", got.DownloadURL, want)
+	item := got["data"].([]any)[0].(map[string]any)
+	if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want {
+		t.Errorf("href = %v, want %q", item["href"], want)
 	}
-	if got.Metadata["repository"] != "https://github.com/a/b" {
-		t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"])
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want {
+		t.Errorf("download_url = %v, want %q", item["download_url"], want)
+	}
+	for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} {
+		if item[k] != want {
+			t.Errorf("%s rewritten to %v", k, item[k])
+		}
+	}
+	meta := item["metadata"].(map[string]any)
+	if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" {
+		t.Errorf("unrelated URLs rewritten: %v", meta)
+	}
+}
+
+func TestRewriteResponseEscapesRemoteName(t *testing.T) {
+	got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`)
+	if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want {
+		t.Errorf("href = %v, want %q", got["href"], want)
+	}
+}
+
+func TestRewriteResponseKeepsLargeNumbers(t *testing.T) {
+	out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "{\"n\":12345678901234567890}\n"; string(out) != want {
+		t.Errorf("got %q, want %q", out, want)
 	}
 }

@@ -77,4 +122,3 @@ func TestRewriteResponseSkipsNonJSON(t *testing.T) {
 		t.Errorf("got %q, %v; want nil, nil", out, err)
 	}
 }
-
@@ -10,6 +10,9 @@ import (
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )

+// RemoteMountPath is the URL path remote proxies are served under.
+const RemoteMountPath = "/api/v1/remote/"
+
 type Mutability int

 const (