Retrigger CI

- add ansible package type proxying the Galaxy v3 API
- rewrite href/links/download_url back to the remote so tarballs flow through it
- forward the query string for providers that opt in, keeping Galaxy pagination working
- add a UI usage snippet and an ansible e2e caching case

README.md

@@ -1,6 +1,6 @@
 # ArtifactAPI
 
-Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
+Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
 
 ## Quick Start
 
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
 | `terraform` | `*/versions` | `*/download/*/*` |
 | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
 | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
+| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` |
 | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
 
 Providers classify paths automatically. Users only configure what to proxy and TTLs.
@@ -261,21 +262,6 @@ there's nothing to provision. To bring your own key instead, point
 `TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
 `TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
 
-### Local helm repository
-
-A local `helm` repo is a real chart repository: `PUT` a packaged chart to
-`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
-generated from each chart's `Chart.yaml` (with sha256 digests) at
-`/api/v1/local/<repo>/index.yaml`. Add the local as a member of a helm virtual
-to serve it alongside remotes.
-
-```bash
-helm package ./mychart
-curl -fsSL --upload-file mychart-0.1.0.tgz \
-  https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
-helm repo add <repo> https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>
-```
-
 ### Local docker registry
 
 A local `docker` repo is a real container registry, not a mirror: it serves the

e2e-docker/README.md

@@ -23,7 +23,7 @@ already-running stack.
 
 - **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
 - **Caching** — one immutable artifact per remote package type (generic, docker,
-  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
+  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, ansible) proxied through the
   mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
   verified against the origin fixture.
 - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`

e2e-docker/caching_test.go

@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
 		{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
 		{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
 		{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
+		{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"},
 		{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
 		{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
 	}

e2e-docker/helm_local_test.go

@@ -1,71 +0,0 @@
-//go:build dockere2e
-
-package e2edocker
-
-import (
-	"net/http"
-	"os"
-	"os/exec"
-	"strings"
-	"testing"
-	"time"
-)
-
-// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it
-// to a helm local, then drives helm repo add/search/pull/template against the
-// local and against a helm virtual that has the local as a member.
-func TestLocalHelmRealClient(t *testing.T) {
-	network := os.Getenv("COMPOSE_NETWORK")
-	internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
-	if network == "" || internal == "" {
-		t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
-	}
-	if _, err := exec.LookPath("docker"); err != nil {
-		t.Skip("docker not available on the test host")
-	}
-
-	createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`)
-	defer deleteRepo(t, "local-helm")
-	createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`)
-	defer deleteVirtual(t, "vhelm")
-
-	helm := func(script string) string {
-		t.Helper()
-		out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh",
-			"alpine/helm:3.16.4", "-c", "set -eu; "+script).Output()
-		if err != nil {
-			stderr := ""
-			if ee, ok := err.(*exec.ExitError); ok {
-				stderr = string(ee.Stderr)
-			}
-			t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr)
-		}
-		return string(out)
-	}
-
-	chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz")
-	uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip")
-
-	// Chart metadata is recorded asynchronously after upload; poll the index.
-	var body []byte
-	for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) {
-		var resp *http.Response
-		resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "")
-		if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") {
-			break
-		}
-	}
-	if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") {
-		t.Fatalf("index.yaml never listed the chart:\n%s", body)
-	}
-
-	for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} {
-		out := helm("helm repo add e2e " + repoURL + " >/dev/null && " +
-			"helm search repo e2e/e2echart && " +
-			"cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " +
-			"helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'")
-		if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") {
-			t.Fatalf("helm via %s:\n%s", repoURL, out)
-		}
-	}
-}

internal/api/v1/proxy.go

@@ -92,6 +92,10 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if _, ok := prov.(provider.QueryForwarder); ok && r.URL.RawQuery != "" {
+		path += "?" + r.URL.RawQuery
+	}
+
 	result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
 	if err != nil {
 		var proxyErr *proxy.ProxyError

internal/database/helm_metadata.go

@@ -1,44 +0,0 @@
-package database
-
-import (
-	"context"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-)
-
-func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error {
-	_, err := db.Pool.Exec(ctx, `
-		INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml)
-		VALUES ($1,$2,$3,$4,$5,$6)
-		ON CONFLICT (repo_name, file_path) DO NOTHING
-	`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML)
-	return err
-}
-
-func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error {
-	_, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
-	return err
-}
-
-func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) {
-	rows, err := db.Pool.Query(ctx, `
-		SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at
-		FROM helm_metadata
-		WHERE repo_name = $1
-		ORDER BY name, version, file_path
-	`, repoName)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var result []provider.HelmMetadata
-	for rows.Next() {
-		var m provider.HelmMetadata
-		if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil {
-			return nil, err
-		}
-		result = append(result, m)
-	}
-	return result, rows.Err()
-}

internal/provider/ansible/ansible.go

@@ -0,0 +1,65 @@
+// Package ansible proxies an Ansible Galaxy server (v3 collections API):
+// mutable JSON metadata whose links are rewritten back to the remote, plus
+// immutable collection tarballs.
+package ansible
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isArtifact(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	if isArtifact(path) {
+		return "application/gzip"
+	}
+	return "application/json"
+}
+
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
+}
+
+// ForwardQuery keeps limit/offset pagination in the proxied path.
+func (p *Provider) ForwardQuery() {}
+
+// RewriteResponse points root-relative hrefs/links and absolute download_url
+// values at the remote. Links stay root-relative, which ansible-galaxy joins
+// onto the server URL, so the cached body is independent of the request host.
+func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
+	if !json.Valid(body) {
+		return nil, nil
+	}
+	prefix := `"/api/v1/remote/` + remote.Name + `/`
+	content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`)
+	content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix)
+	return []byte(content), nil
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+func isArtifact(path string) bool {
+	return strings.HasSuffix(path, ".tar.gz")
+}

internal/provider/ansible/ansible_test.go

@@ -0,0 +1,85 @@
+package ansible
+
+import (
+	"encoding/json"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"api/", provider.Mutable},
+		{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
+		{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
+		{"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestUpstreamURLKeepsQuery(t *testing.T) {
+	p := &Provider{}
+	got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100")
+	want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100"
+	if got != want {
+		t.Errorf("UpstreamURL = %q, want %q", got, want)
+	}
+}
+
+func TestRewriteResponse(t *testing.T) {
+	p := &Provider{}
+	remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
+	body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` +
+		`"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` +
+		`"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` +
+		`"metadata":{"repository":"https://github.com/a/b"}}`
+
+	out, err := p.RewriteResponse([]byte(body), remote, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Links       map[string]any `json:"links"`
+		Href        string         `json:"href"`
+		DownloadURL string         `json:"download_url"`
+		Metadata    map[string]any `json:"metadata"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatalf("rewritten body is not JSON: %v", err)
+	}
+	pre := "/api/v1/remote/galaxy/"
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want {
+		t.Errorf("next = %v, want %q", got.Links["next"], want)
+	}
+	if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want {
+		t.Errorf("href = %q, want %q", got.Href, want)
+	}
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want {
+		t.Errorf("download_url = %q, want %q", got.DownloadURL, want)
+	}
+	if got.Metadata["repository"] != "https://github.com/a/b" {
+		t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"])
+	}
+}
+
+func TestRewriteResponseSkipsNonJSON(t *testing.T) {
+	out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
+	if err != nil || out != nil {
+		t.Errorf("got %q, %v; want nil, nil", out, err)
+	}
+}
+
+func TestForwardsQuery(t *testing.T) {
+	if _, ok := any(&Provider{}).(provider.QueryForwarder); !ok {
+		t.Error("ansible provider must forward the query string for pagination")
+	}
+}

internal/provider/helm/helm.go

@@ -1,23 +1,12 @@
 package helm
 
 import (
-	"archive/tar"
-	"compress/gzip"
 	"context"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
 	"net/http"
-	"path"
 	"strings"
-	"time"
-
-	"gopkg.in/yaml.v3"
 
 	"git.unkin.net/unkin/artifactapi/internal/auth"
 	"git.unkin.net/unkin/artifactapi/internal/provider"
-	"git.unkin.net/unkin/artifactapi/internal/storage"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -67,167 +56,3 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU
 func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
 	return auth.BasicHeaders(remote), nil
 }
-
-// ValidateUpload stores charts flat by filename so index.yaml can reference
-// them with relative urls, as `helm repo index` does.
-func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
-	filename := path.Base(path.Clean("/" + filePath))
-	if !strings.HasSuffix(filename, ".tgz") || filename == ".tgz" {
-		return "", "", fmt.Errorf("file must be a .tgz chart package")
-	}
-	return filename, "application/gzip", nil
-}
-
-func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
-	return map[string]any{
-		"filename":     storagePath,
-		"content_hash": contentHash,
-		"size_bytes":   sizeBytes,
-	}
-}
-
-func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
-	reader, _, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
-	if err != nil {
-		slog.Error("helm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
-		return
-	}
-	defer func() { _ = reader.Close() }()
-
-	meta, err := parseChart(reader)
-	if err != nil {
-		slog.Error("helm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
-		return
-	}
-	meta.RepoName = repoName
-	meta.FilePath = storagePath
-	meta.ContentHash = contentHash
-
-	store, ok := db.(provider.HelmMetadataStore)
-	if !ok {
-		slog.Error("helm metadata: store does not support helm metadata", "repo", repoName)
-		return
-	}
-	if err := store.InsertHelmMetadata(ctx, meta); err != nil {
-		slog.Error("helm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
-		return
-	}
-	slog.Info("helm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version)
-}
-
-func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
-	deleter, ok := db.(provider.HelmMetadataDeleter)
-	if !ok {
-		return nil
-	}
-	return deleter.DeleteHelmMetadata(ctx, repoName, storagePath)
-}
-
-func isIndexPath(p string) bool {
-	return strings.TrimPrefix(path.Clean("/"+p), "/") == "index.yaml"
-}
-
-func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
-	if !isIndexPath(reqPath) {
-		return false
-	}
-	body, err := p.GenerateLocalIndex(r.Context(), files, repoName, reqPath)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return true
-	}
-	w.Header().Set("Content-Type", "text/yaml")
-	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write(body)
-	return true
-}
-
-func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, reqPath string) ([]byte, error) {
-	if !isIndexPath(reqPath) {
-		return nil, fmt.Errorf("unsupported index path: %q", reqPath)
-	}
-	reader, ok := files.(provider.HelmMetadataReader)
-	if !ok {
-		return nil, errors.New("helm metadata not available")
-	}
-	metas, err := reader.ListHelmMetadataEntries(ctx, repoName)
-	if err != nil {
-		return nil, err
-	}
-	return generateIndex(metas)
-}
-
-// generateIndex builds index.yaml from stored Chart.yaml documents; it is
-// stamped from the newest upload so every replica serves identical bytes.
-func generateIndex(metas []provider.HelmMetadata) ([]byte, error) {
-	entries := map[string][]map[string]yaml.Node{}
-	var newest time.Time
-	for _, m := range metas {
-		var chart map[string]yaml.Node
-		if err := yaml.Unmarshal([]byte(m.ChartYAML), &chart); err != nil {
-			return nil, fmt.Errorf("chart %s: %w", m.FilePath, err)
-		}
-		if chart == nil {
-			chart = map[string]yaml.Node{}
-		}
-		chart["name"] = node(m.Name)
-		chart["version"] = node(m.Version)
-		chart["urls"] = node([]string{m.FilePath})
-		chart["digest"] = node(strings.TrimPrefix(m.ContentHash, "sha256:"))
-		chart["created"] = node(m.CreatedAt.UTC().Format(time.RFC3339Nano))
-		entries[m.Name] = append(entries[m.Name], chart)
-		if m.CreatedAt.After(newest) {
-			newest = m.CreatedAt
-		}
-	}
-	return yaml.Marshal(map[string]any{
-		"apiVersion": "v1",
-		"entries":    entries,
-		"generated":  newest.UTC().Format(time.RFC3339Nano),
-	})
-}
-
-func node(v any) yaml.Node {
-	var n yaml.Node
-	_ = n.Encode(v)
-	return n
-}
-
-const maxChartYAML = 1 << 20
-
-// parseChart reads the top-level <chart>/Chart.yaml from a packaged chart.
-func parseChart(r io.Reader) (*provider.HelmMetadata, error) {
-	zr, err := gzip.NewReader(r)
-	if err != nil {
-		return nil, err
-	}
-	tr := tar.NewReader(zr)
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			return nil, errors.New("no Chart.yaml in chart archive")
-		}
-		if err != nil {
-			return nil, err
-		}
-		dir, file := path.Split(path.Clean(hdr.Name))
-		if file != "Chart.yaml" || strings.Count(dir, "/") != 1 {
-			continue
-		}
-		raw, err := io.ReadAll(io.LimitReader(tr, maxChartYAML))
-		if err != nil {
-			return nil, err
-		}
-		var chart struct {
-			Name    string `yaml:"name"`
-			Version string `yaml:"version"`
-		}
-		if err := yaml.Unmarshal(raw, &chart); err != nil {
-			return nil, fmt.Errorf("parse Chart.yaml: %w", err)
-		}
-		if chart.Name == "" || chart.Version == "" {
-			return nil, errors.New("chart metadata missing name or version")
-		}
-		return &provider.HelmMetadata{Name: chart.Name, Version: chart.Version, ChartYAML: string(raw)}, nil
-	}
-}

internal/provider/helm/helm_local_test.go

@@ -1,162 +0,0 @@
-package helm
-
-import (
-	"archive/tar"
-	"bytes"
-	"compress/gzip"
-	"context"
-	"io"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	"gopkg.in/yaml.v3"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-)
-
-func chartTgz(t *testing.T, files map[string]string) []byte {
-	t.Helper()
-	var buf bytes.Buffer
-	zw := gzip.NewWriter(&buf)
-	tw := tar.NewWriter(zw)
-	for name, body := range files {
-		if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
-			t.Fatal(err)
-		}
-		if _, err := tw.Write([]byte(body)); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if err := tw.Close(); err != nil {
-		t.Fatal(err)
-	}
-	if err := zw.Close(); err != nil {
-		t.Fatal(err)
-	}
-	return buf.Bytes()
-}
-
-type fakeBlobs struct{ data []byte }
-
-func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
-	return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
-}
-
-type fakeStore struct {
-	inserted *provider.HelmMetadata
-	metas    []provider.HelmMetadata
-}
-
-func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
-func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
-func (f *fakeStore) InsertHelmMetadata(_ context.Context, m *provider.HelmMetadata) error {
-	f.inserted = m
-	return nil
-}
-func (f *fakeStore) ListHelmMetadataEntries(context.Context, string) ([]provider.HelmMetadata, error) {
-	return f.metas, nil
-}
-func (f *fakeStore) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
-	return nil, nil
-}
-func (f *fakeStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
-
-func TestValidateUpload(t *testing.T) {
-	p := &Provider{}
-	for in, want := range map[string]string{
-		"mychart-0.1.0.tgz":        "mychart-0.1.0.tgz",
-		"charts/mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
-		"../../mychart-0.1.0.tgz":  "mychart-0.1.0.tgz",
-	} {
-		sp, ct, err := p.ValidateUpload(in)
-		if err != nil || sp != want || ct != "application/gzip" {
-			t.Errorf("ValidateUpload(%q) = %q %q %v", in, sp, ct, err)
-		}
-	}
-	for _, bad := range []string{"index.yaml", "chart.tar.gz", ".tgz", ""} {
-		if _, _, err := p.ValidateUpload(bad); err == nil {
-			t.Errorf("ValidateUpload(%q) accepted", bad)
-		}
-	}
-}
-
-func TestAfterUploadParsesTopLevelChart(t *testing.T) {
-	data := chartTgz(t, map[string]string{
-		"mychart/charts/sub/Chart.yaml": "apiVersion: v2\nname: sub\nversion: 9.9.9\n",
-		"mychart/Chart.yaml":            "apiVersion: v2\nname: mychart\nversion: 1.0\ndescription: hi\n",
-		"mychart/values.yaml":           "a: 1\n",
-	})
-	store := &fakeStore{}
-	(&Provider{}).AfterUpload(context.Background(), "r", "mychart-1.0.tgz", "sha256:ab", fakeBlobs{data}, store)
-	m := store.inserted
-	if m == nil || m.Name != "mychart" || m.Version != "1.0" || m.FilePath != "mychart-1.0.tgz" || m.ContentHash != "sha256:ab" {
-		t.Fatalf("inserted %+v", m)
-	}
-}
-
-func TestAfterUploadRejectsChartWithoutMetadata(t *testing.T) {
-	store := &fakeStore{}
-	data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\n"})
-	(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{data}, store)
-	if store.inserted != nil {
-		t.Fatal("chart without version was recorded")
-	}
-	(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{[]byte("not gzip")}, store)
-	if store.inserted != nil {
-		t.Fatal("non-gzip upload was recorded")
-	}
-}
-
-func TestServeLocalIndex(t *testing.T) {
-	t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
-	t2 := t1.Add(time.Hour)
-	store := &fakeStore{metas: []provider.HelmMetadata{
-		{FilePath: "mychart-1.0.tgz", ContentHash: "sha256:aa", Name: "mychart", Version: "1.0", CreatedAt: t1,
-			ChartYAML: "apiVersion: v2\nname: mychart\nversion: 1.0\nappVersion: \"2\"\ndescription: hi\n"},
-		{FilePath: "other-0.2.0.tgz", ContentHash: "sha256:bb", Name: "other", Version: "0.2.0", CreatedAt: t2,
-			ChartYAML: "apiVersion: v2\nname: other\nversion: 0.2.0\n"},
-	}}
-	p := &Provider{}
-
-	if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), store, "r", "mychart-1.0.tgz") {
-		t.Fatal("chart path handled as index")
-	}
-	rec := httptest.NewRecorder()
-	if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), store, "r", "./index.yaml") {
-		t.Fatal("index.yaml not handled")
-	}
-	if rec.Code != 200 || rec.Header().Get("Content-Type") != "text/yaml" {
-		t.Fatalf("code=%d ct=%q", rec.Code, rec.Header().Get("Content-Type"))
-	}
-
-	var idx struct {
-		APIVersion string `yaml:"apiVersion"`
-		Generated  string `yaml:"generated"`
-		Entries    map[string][]map[string]any
-	}
-	if err := yaml.Unmarshal(rec.Body.Bytes(), &idx); err != nil {
-		t.Fatal(err)
-	}
-	if idx.APIVersion != "v1" || idx.Generated != t2.Format(time.RFC3339Nano) {
-		t.Errorf("header %+v", idx)
-	}
-	e := idx.Entries["mychart"][0]
-	if e["digest"] != "aa" || e["description"] != "hi" || e["appVersion"] != "2" || e["apiVersion"] != "v2" {
-		t.Errorf("entry %v", e)
-	}
-	if urls, _ := e["urls"].([]any); len(urls) != 1 || urls[0] != "mychart-1.0.tgz" {
-		t.Errorf("urls %v", e["urls"])
-	}
-	if e["version"] != "1.0" || e["name"] != "mychart" {
-		t.Errorf("version must stay a string: %#v", e["version"])
-	}
-	if len(idx.Entries["other"]) != 1 {
-		t.Errorf("other entries %v", idx.Entries["other"])
-	}
-
-	again, err := p.GenerateLocalIndex(context.Background(), store, "r", "index.yaml")
-	if err != nil || !bytes.Equal(again, rec.Body.Bytes()) {
-		t.Errorf("index not deterministic: %v", err)
-	}
-}

internal/provider/provider.go

@@ -65,6 +65,13 @@ type MetadataStore interface {
 	InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
 }
 
+// QueryForwarder marks a remote provider whose upstream answers depend on the
+// query string (e.g. paginated JSON APIs); the query is kept as part of the
+// proxied and cached path.
+type QueryForwarder interface {
+	ForwardQuery()
+}
+
 // RemoteServer lets a remote provider fully answer a request itself instead of
 // going through the byte-proxy engine. It is the remote-side analog of
 // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
@@ -240,27 +247,3 @@ func Get(t models.PackageType) (Provider, error) {
 func All() map[models.PackageType]Provider {
 	return registry
 }
-
-type HelmMetadataStore interface {
-	InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error
-}
-
-type HelmMetadataDeleter interface {
-	DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error
-}
-
-type HelmMetadataReader interface {
-	ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error)
-}
-
-// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml
-// entries carry every field helm reads.
-type HelmMetadata struct {
-	RepoName    string
-	FilePath    string
-	ContentHash string
-	Name        string
-	Version     string
-	ChartYAML   string
-	CreatedAt   time.Time
-}

internal/server/server.go

@@ -22,6 +22,7 @@ import (
 	"git.unkin.net/unkin/artifactapi/internal/gc"
 	"git.unkin.net/unkin/artifactapi/internal/githubauth"
 	"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
+	_ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
 	"git.unkin.net/unkin/artifactapi/internal/provider/deb"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"

internal/virtual/helm_merger.go

@@ -16,17 +16,22 @@ func init() {
 type HelmMerger struct{}
 
 type helmIndex struct {
-	APIVersion string                            `yaml:"apiVersion"`
-	Entries    map[string][]map[string]yaml.Node `yaml:"entries"`
-	Generated  string                            `yaml:"generated,omitempty"`
+	APIVersion string                        `yaml:"apiVersion"`
+	Entries    map[string][]helmChartVersion `yaml:"entries"`
+	Generated  string                        `yaml:"generated,omitempty"`
+}
+
+type helmChartVersion struct {
+	Name    string   `yaml:"name"`
+	Version string   `yaml:"version"`
+	URLs    []string `yaml:"urls"`
+	rest    map[string]any
 }
 
-// MergeIndexes keeps each chart version's fields as raw nodes so digest,
-// apiVersion and the rest survive the merge untouched; only urls are rewritten.
 func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
 	merged := &helmIndex{
 		APIVersion: "v1",
-		Entries:    make(map[string][]map[string]yaml.Node),
+		Entries:    make(map[string][]helmChartVersion),
 	}
 
 	seen := map[string]map[string]bool{}
@@ -42,26 +47,42 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
 				seen[chart] = map[string]bool{}
 			}
 			for _, ver := range versions {
-				version := ver["version"].Value
-				if seen[chart][version] {
+				key := chart + ":" + ver.Version
+				if seen[chart][ver.Version] {
 					continue
 				}
-				seen[chart][version] = true
+				seen[chart][ver.Version] = true
 
-				if urlsNode, ok := ver["urls"]; ok && proxyBaseURL != "" {
-					var urls []string
-					if err := urlsNode.Decode(&urls); err != nil {
-						continue
+				if proxyBaseURL != "" {
+					routePrefix := "remote"
+					if member.RepoType == "local" {
+						routePrefix = "local"
 					}
-					rewriteHelmURLs(urls, member, proxyBaseURL)
-					var n yaml.Node
-					if err := n.Encode(urls); err != nil {
-						return nil, err
+					baseHost := extractHost(member.BaseURL)
+
+					for i, u := range ver.URLs {
+						if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
+							if baseHost != "" && extractHost(u) != baseHost {
+								continue
+							}
+							relPath := extractPathRelativeToBase(u, member.BaseURL)
+							ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
+								strings.TrimRight(proxyBaseURL, "/"),
+								routePrefix,
+								member.RemoteName,
+								relPath)
+						} else {
+							ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
+								strings.TrimRight(proxyBaseURL, "/"),
+								routePrefix,
+								member.RemoteName,
+								u)
+						}
 					}
-					ver["urls"] = n
 				}
 
 				merged.Entries[chart] = append(merged.Entries[chart], ver)
+				_ = key
 			}
 		}
 	}
@@ -69,34 +90,6 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
 	return yaml.Marshal(merged)
 }
 
-func rewriteHelmURLs(urls []string, member MemberIndex, proxyBaseURL string) {
-	routePrefix := "remote"
-	if member.RepoType == "local" {
-		routePrefix = "local"
-	}
-	baseHost := extractHost(member.BaseURL)
-
-	for i, u := range urls {
-		if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
-			if baseHost != "" && extractHost(u) != baseHost {
-				continue
-			}
-			relPath := extractPathRelativeToBase(u, member.BaseURL)
-			urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
-				strings.TrimRight(proxyBaseURL, "/"),
-				routePrefix,
-				member.RemoteName,
-				relPath)
-		} else {
-			urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
-				strings.TrimRight(proxyBaseURL, "/"),
-				routePrefix,
-				member.RemoteName,
-				u)
-		}
-	}
-}
-
 func extractHost(rawURL string) string {
 	idx := strings.Index(rawURL, "://")
 	if idx == -1 {

internal/virtual/helm_merger_test.go

@@ -122,36 +122,3 @@ entries:
 		t.Error("expected fallback repo to be excluded for duplicate")
 	}
 }
-
-func TestHelmMerger_PreservesChartFields(t *testing.T) {
-	m := &virtual.HelmMerger{}
-	body := []byte(`apiVersion: v1
-entries:
-  app:
-    - apiVersion: v2
-      name: app
-      version: "1.0"
-      appVersion: "3.1"
-      description: in-house
-      digest: deadbeef
-      urls:
-        - app-1.0.tgz
-`)
-	result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "charts", RepoType: "local", Body: body}}, "https://proxy.example.com")
-	if err != nil {
-		t.Fatal(err)
-	}
-	out := string(result)
-	for _, want := range []string{
-		"digest: deadbeef",
-		"apiVersion: v2",
-		`appVersion: "3.1"`,
-		"description: in-house",
-		`version: "1.0"`,
-		"https://proxy.example.com/api/v1/local/charts/app-1.0.tgz",
-	} {
-		if !strings.Contains(out, want) {
-			t.Errorf("merged index missing %q:\n%s", want, out)
-		}
-	}
-}

migrations/0003_helm_metadata.sql

@@ -1,13 +0,0 @@
-CREATE TABLE IF NOT EXISTS helm_metadata (
-	id           BIGSERIAL PRIMARY KEY,
-	repo_name    TEXT NOT NULL,
-	file_path    TEXT NOT NULL,
-	content_hash TEXT NOT NULL,
-	name         TEXT NOT NULL,
-	version      TEXT NOT NULL,
-	chart_yaml   TEXT NOT NULL,
-	created_at   TIMESTAMPTZ DEFAULT NOW(),
-	UNIQUE(repo_name, file_path)
-);
-
-CREATE INDEX IF NOT EXISTS idx_helm_metadata_repo ON helm_metadata(repo_name);

pkg/models/package_type.go

@@ -17,6 +17,7 @@ const (
 	PackageTerraform    PackageType = "terraform"
 	PackageGoProxy      PackageType = "goproxy"
 	PackageCargo        PackageType = "cargo"
+	PackageAnsible      PackageType = "ansible"
 	PackageGitHubRPM    PackageType = "github_rpm"
 	PackageGitHubDeb    PackageType = "github_deb"
 	PackageGitHubAlpine PackageType = "github_alpine"
@@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{
 	PackageTerraform:    true,
 	PackageGoProxy:      true,
 	PackageCargo:        true,
+	PackageAnsible:      true,
 	PackageGitHubRPM:    true,
 	PackageGitHubDeb:    true,
 	PackageGitHubAlpine: true,

pkg/models/package_type_test.go

@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
 		models.PackageTerraform,
 		models.PackageGoProxy,
 		models.PackageCargo,
+		models.PackageAnsible,
 		models.PackageGitHubRPM,
 		models.PackageGitHubDeb,
 		models.PackageGitHubAlpine,

ui/src/components/UsageInstructions.tsx

@@ -167,23 +167,12 @@ docker push        ${host}/${name}/myapp:latest`,
     case 'helm':
       return [
         {
-          title: isLocal ? 'Add the Helm repo (index.yaml auto-generated)' : 'Add the Helm repo',
+          title: 'Add the Helm repo',
           language: 'bash',
-          code: `helm repo add ${name} ${isLocal ? `${url}/api/v1/local/${name}` : proxy}/
+          code: `helm repo add ${name} ${proxy}/
 helm repo update
 helm install <release> ${name}/<chart>`,
         },
-        ...(isLocal
-          ? [
-              {
-                title: 'Publish a chart',
-                language: 'bash',
-                code: `helm package ./mychart
-curl -fsSL --upload-file mychart-0.1.0.tgz \\
-  ${url}/api/v2/remotes/${name}/files/mychart-0.1.0.tgz`,
-              },
-            ]
-          : []),
       ];
 
     case 'alpine':
@@ -254,6 +243,15 @@ index = "sparse+${proxy}/"`,
         },
       ];
 
+    case 'ansible':
+      return [
+        {
+          title: 'Install collections through this Galaxy proxy',
+          language: 'bash',
+          code: `ansible-galaxy collection install community.general -s ${proxy}/api/`,
+        },
+      ];
+
     case 'puppet':
       return [
         {

ui/src/pages/Remotes.tsx

@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   terraform: 'blue',
   goproxy: 'green',
   cargo: 'yellow',
+  ansible: 'red',
 };
 
 export function Remotes() {