Add ansible Galaxy remote type (#137)

ansible-galaxy installs pull collections straight from galaxy.ansible.com because artifactapi has no Galaxy remote type.

- add `ansible` package type proxying the Galaxy v3 API (discovery, collection versions, artifact downloads)
- forward digits-only `limit` (capped at 100) and `offset` on metadata paths, canonicalised; tarball paths forward no query
- rewrite only `href`, `download_url` and `links.*` to root-relative remote paths so tarballs flow through artifactapi
- treat `artifacts/*.tar.gz` as immutable and JSON metadata as mutable
- add `docs/ansible/remote.md`, a UI type def and an ansible e2e caching case

requires unkin/artifactapi#146

Reviewed-on: #137
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

README.md

@@ -27,6 +27,7 @@ Classification precedence, TTLs and eviction for every remote: [docs/common/cach
 Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
 
 - `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
+- `ansible`: [remote](docs/ansible/remote.md)
 - `cargo`: [remote](docs/cargo/remote.md)
 - `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
 - `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)

docs/ansible/remote.md

@@ -0,0 +1,11 @@
+# `ansible` remote
+
+Shared classification, TTL and eviction rules: [caching](../common/caching.md).
+
+| Type | Mutable (auto-detected) | Immutable (auto-detected) |
+|---|---|---|
+| `ansible` | Galaxy `api/` JSON (links rewritten, `limit`/`offset` forwarded) | `artifacts/*.tar.gz` |
+
+```bash
+ansible-galaxy collection install community.general -s https://artifactapi.example/api/v1/remote/galaxy/api/
+```

docs/common/caching.md

@@ -93,7 +93,7 @@ Force a moving path mutable, e.g. branch tarballs on `generic`:
 ## Built-in classification per type
 
 Each type's page lists its auto-detected mutable and immutable paths:
-[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md),
+[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), [ansible](../ansible/remote.md),
 [cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md),
 [docker](../docker/remote.md), [generic](../generic/remote.md), [gems](../gems/remote.md), [goproxy](../goproxy/remote.md),
 [helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md),

e2e-docker/caching_ansible_test.go

@@ -0,0 +1,7 @@
+//go:build dockere2e
+
+package e2edocker
+
+func init() {
+	cachingCases = append(cachingCases, cachingCase{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"})
+}

internal/provider/all/ansible.go

@@ -0,0 +1,3 @@
+package all
+
+import _ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"

internal/provider/ansible/ansible.go

@@ -0,0 +1,139 @@
+// Package ansible proxies an Ansible Galaxy server (v3 collections API):
+// mutable JSON metadata whose links are rewritten back to the remote, plus
+// immutable collection tarballs.
+package ansible
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isArtifact(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	if isArtifact(path) {
+		return "application/gzip"
+	}
+	return "application/json"
+}
+
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
+}
+
+const maxPageLimit = 100
+
+// ForwardedQuery keeps limit/offset pagination on metadata paths only.
+func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator {
+	if isArtifact(path) {
+		return nil
+	}
+	return map[string]provider.QueryValidator{"limit": pageLimit, "offset": pageOffset}
+}
+
+func pageOffset(v string) (string, bool) {
+	n, err := strconv.ParseUint(v, 10, 32)
+	return strconv.FormatUint(n, 10), err == nil
+}
+
+func pageLimit(v string) (string, bool) {
+	n, err := strconv.ParseUint(v, 10, 32)
+	if err != nil || n == 0 {
+		return "", false
+	}
+	return strconv.FormatUint(min(n, maxPageLimit), 10), true
+}
+
+// RewriteResponse points href, download_url and links.* at the remote. Links
+// stay root-relative, which ansible-galaxy joins onto the server URL, so the
+// cached body is independent of the request host.
+func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
+	dec := json.NewDecoder(bytes.NewReader(body))
+	dec.UseNumber()
+	var doc any
+	if err := dec.Decode(&doc); err != nil {
+		return nil, nil
+	}
+	r := rewriter{
+		prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/",
+		base:   strings.TrimRight(remote.BaseURL, "/") + "/",
+	}
+	r.walk(doc)
+	var out bytes.Buffer
+	enc := json.NewEncoder(&out)
+	enc.SetEscapeHTML(false)
+	if err := enc.Encode(doc); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+type rewriter struct{ prefix, base string }
+
+func (r rewriter) walk(v any) {
+	switch v := v.(type) {
+	case []any:
+		for _, e := range v {
+			r.walk(e)
+		}
+	case map[string]any:
+		for k, e := range v {
+			switch k {
+			case "href", "download_url":
+				v[k] = r.link(e)
+			case "links":
+				if links, ok := e.(map[string]any); ok {
+					for lk, l := range links {
+						links[lk] = r.link(l)
+					}
+					continue
+				}
+			}
+			r.walk(v[k])
+		}
+	}
+}
+
+func (r rewriter) link(v any) any {
+	s, ok := v.(string)
+	switch {
+	case !ok:
+		return v
+	case strings.HasPrefix(s, "/api/"):
+		return r.prefix + s[1:]
+	case strings.HasPrefix(s, r.base):
+		return r.prefix + strings.TrimPrefix(s, r.base)
+	}
+	return s
+}
+
+func isArtifact(path string) bool {
+	path, _, _ = strings.Cut(path, "?")
+	return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz")
+}

internal/provider/ansible/ansible_test.go

@@ -0,0 +1,150 @@
+package ansible
+
+import (
+	"encoding/json"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz"
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"api/", provider.Mutable},
+		{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
+		{"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable},
+		{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
+		{"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable},
+		{tarball, provider.Immutable},
+		{tarball + "?limit=1", provider.Immutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestContentTypeIgnoresQuery(t *testing.T) {
+	if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" {
+		t.Errorf("ContentType = %q, want application/json", got)
+	}
+}
+
+func TestForwardedQuery(t *testing.T) {
+	p := &Provider{}
+	if got := p.ForwardedQuery(tarball); got != nil {
+		t.Errorf("tarball ForwardedQuery = %v, want nil", got)
+	}
+	q := p.ForwardedQuery("api/v3/collections/a/b/versions/")
+	if len(q) != 2 || q["limit"] == nil || q["offset"] == nil {
+		t.Fatalf("metadata ForwardedQuery keys = %v, want limit and offset", q)
+	}
+	tests := []struct {
+		param, in, want string
+		ok              bool
+	}{
+		{"limit", "10", "10", true},
+		{"limit", "007", "7", true},
+		{"limit", "100", "100", true},
+		{"limit", "5000", "100", true},
+		{"limit", "0", "", false},
+		{"limit", "", "", false},
+		{"limit", "-1", "", false},
+		{"limit", "+5", "", false},
+		{"limit", "1e3", "", false},
+		{"limit", "10 ", "", false},
+		{"offset", "0", "0", true},
+		{"offset", "000", "0", true},
+		{"offset", "0100", "100", true},
+		{"offset", "abc", "", false},
+		{"offset", "99999999999999999999", "", false},
+	}
+	for _, tt := range tests {
+		got, ok := q[tt.param](tt.in)
+		if ok != tt.ok || (ok && got != tt.want) {
+			t.Errorf("%s(%q) = %q, %v, want %q, %v", tt.param, tt.in, got, ok, tt.want, tt.ok)
+		}
+	}
+}
+
+func rewrite(t *testing.T, remote models.Remote, body string) map[string]any {
+	t.Helper()
+	out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got map[string]any
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatalf("rewritten body is not JSON: %v: %s", err, out)
+	}
+	return got
+}
+
+func TestRewriteResponse(t *testing.T) {
+	remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
+	got := rewrite(t, remote, `{
+		"links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null},
+		"data": [{
+			"href": "/api/v3/collections/a/b/versions/1.0.0/",
+			"download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",
+			"description": "/api/ is where it lives",
+			"docs": "/api/v3/not-a-link/",
+			"metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"}
+		}]
+	}`)
+	pre := "/api/v1/remote/galaxy/"
+	links := got["links"].(map[string]any)
+	if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want {
+		t.Errorf("links.next = %v, want %q", links["next"], want)
+	}
+	if links["previous"] != nil {
+		t.Errorf("links.previous = %v, want null", links["previous"])
+	}
+	item := got["data"].([]any)[0].(map[string]any)
+	if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want {
+		t.Errorf("href = %v, want %q", item["href"], want)
+	}
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want {
+		t.Errorf("download_url = %v, want %q", item["download_url"], want)
+	}
+	for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} {
+		if item[k] != want {
+			t.Errorf("%s rewritten to %v", k, item[k])
+		}
+	}
+	meta := item["metadata"].(map[string]any)
+	if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" {
+		t.Errorf("unrelated URLs rewritten: %v", meta)
+	}
+}
+
+func TestRewriteResponseEscapesRemoteName(t *testing.T) {
+	got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`)
+	if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want {
+		t.Errorf("href = %v, want %q", got["href"], want)
+	}
+}
+
+func TestRewriteResponseKeepsLargeNumbers(t *testing.T) {
+	out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if want := "{\"n\":12345678901234567890}\n"; string(out) != want {
+		t.Errorf("got %q, want %q", out, want)
+	}
+}
+
+func TestRewriteResponseSkipsNonJSON(t *testing.T) {
+	out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
+	if err != nil || out != nil {
+		t.Errorf("got %q, %v; want nil, nil", out, err)
+	}
+}

internal/provider/provider.go

@@ -8,6 +8,9 @@ import (
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
+// RemoteMountPath is the URL path remote proxies are served under.
+const RemoteMountPath = "/api/v1/remote/"
+
 type Mutability int
 
 const (

pkg/models/package_type_ansible.go

@@ -0,0 +1,5 @@
+package models
+
+const PackageAnsible PackageType = "ansible"
+
+func init() { registerPackageType(PackageAnsible) }

ui/src/packagetypes/defs/ansible.ts

@@ -0,0 +1,16 @@
+import type { PackageTypeUI } from '../types';
+
+const def: PackageTypeUI = {
+  color: 'red',
+  snippets: ({ proxy }) => {
+    return [
+      {
+        title: 'Install collections through this Galaxy proxy',
+        language: 'bash',
+        code: `ansible-galaxy collection install community.general -s ${proxy}/api/`,
+      },
+    ];
+  },
+};
+
+export default def;