Retrigger CI

- add ansible package type proxying the Galaxy v3 API
- rewrite href/links/download_url back to the remote so tarballs flow through it
- forward the query string for providers that opt in, keeping Galaxy pagination working
- add a UI usage snippet and an ansible e2e caching case

README.md

@@ -1,6 +1,6 @@
 # ArtifactAPI
 
-Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
+Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
 
 ## Quick Start
 
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
 | `terraform` | `*/versions` | `*/download/*/*` |
 | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
 | `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
+| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` |
 | `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
 
 Providers classify paths automatically. Users only configure what to proxy and TTLs.
@@ -261,21 +262,6 @@ there's nothing to provision. To bring your own key instead, point
 `TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
 `TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
 
-### Local puppet forge
-
-A local `puppet` repo is a Forge: `PUT` a module tarball to
-`/api/v2/remotes/<repo>/files/<owner>-<module>-<version>.tar.gz` and the Forge
-v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each
-release's `metadata.json` at `/api/v1/local/<repo>`. Versions are immutable.
-
-```bash
-puppet module build ./hello
-curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \
-  https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/<repo>/files/unkin-hello-0.1.0.tar.gz
-puppet module install unkin-hello \
-  --module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>
-```
-
 ### Local docker registry
 
 A local `docker` repo is a real container registry, not a mirror: it serves the

e2e-docker/README.md

@@ -23,11 +23,10 @@ already-running stack.
 
 - **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
 - **Caching** — one immutable artifact per remote package type (generic, docker,
-  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
+  helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, ansible) proxied through the
   mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
   verified against the origin fixture.
-- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`),
-  generic (upload/download), pypi (wheel + generated `simple/`
+- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
   index), rpm (real package + **automatic repodata** generation).
 - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
   across two members.

e2e-docker/caching_test.go

@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
 		{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
 		{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
 		{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
+		{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"},
 		{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
 		{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
 	}

e2e-docker/puppet_local_test.go

@@ -1,88 +0,0 @@
-//go:build dockere2e
-
-package e2edocker
-
-import (
-	"archive/tar"
-	"bytes"
-	"compress/gzip"
-	"net/http"
-	"os"
-	"os/exec"
-	"strings"
-	"testing"
-	"time"
-)
-
-const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}`
-
-func puppetModuleTarball(t *testing.T) []byte {
-	t.Helper()
-	var buf bytes.Buffer
-	zw := gzip.NewWriter(&buf)
-	tw := tar.NewWriter(zw)
-	for name, body := range map[string]string{
-		"unkin-hello-0.1.0/metadata.json":     puppetHelloMetadata,
-		"unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n",
-	} {
-		if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil {
-			t.Fatal(err)
-		}
-		_, _ = tw.Write([]byte(body))
-	}
-	_ = tw.Close()
-	_ = zw.Close()
-	return buf.Bytes()
-}
-
-// TestLocalPuppetRealClient publishes a module to a puppet local and installs
-// it with the real r10k and puppet module tool. The stdlib dependency is not on
-// a local-only forge, so puppet runs with --ignore-dependencies.
-func TestLocalPuppetRealClient(t *testing.T) {
-	network := os.Getenv("COMPOSE_NETWORK")
-	internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
-	if network == "" || internal == "" {
-		t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
-	}
-	if _, err := exec.LookPath("docker"); err != nil {
-		t.Skip("docker not available on the test host")
-	}
-
-	createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`)
-	defer deleteRepo(t, "local-puppet")
-
-	upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz")
-	defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "")
-	tarball := puppetModuleTarball(t)
-	if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated {
-		t.Fatalf("upload: %d %s", resp.StatusCode, body)
-	}
-	if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict {
-		t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode)
-	}
-	if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) {
-		t.Fatalf("release never indexed: %d %s", resp.StatusCode, body)
-	}
-
-	forge := internal + "/api/v1/local/local-puppet"
-	run := func(image, script string) string {
-		t.Helper()
-		out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput()
-		if err != nil {
-			t.Fatalf("%s failed: %v\n%s", image, err, out)
-		}
-		return string(out)
-	}
-
-	out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n  baseurl: "+forge+"\\n' > r10k.yaml && "+
-		"printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+
-		"r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json")
-	if !strings.Contains(out, `"unkin-hello"`) {
-		t.Fatalf("r10k did not install unkin-hello:\n%s", out)
-	}
-
-	out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods")
-	if !strings.Contains(out, "unkin-hello (v0.1.0)") {
-		t.Fatalf("puppet module install did not install unkin-hello:\n%s", out)
-	}
-}

internal/api/v1/proxy.go

@@ -92,6 +92,10 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if _, ok := prov.(provider.QueryForwarder); ok && r.URL.RawQuery != "" {
+		path += "?" + r.URL.RawQuery
+	}
+
 	result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
 	if err != nil {
 		var proxyErr *proxy.ProxyError

internal/database/puppet_metadata.go

@@ -1,46 +0,0 @@
-package database
-
-import (
-	"context"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-)
-
-func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error {
-	_, err := db.Pool.Exec(ctx, `
-		INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5)
-		VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
-		ON CONFLICT (repo_name, file_path) DO NOTHING
-	`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5)
-	return err
-}
-
-func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error {
-	_, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
-	return err
-}
-
-// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to
-// one module slug (empty slug lists every module).
-func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) {
-	rows, err := db.Pool.Query(ctx, `
-		SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at
-		FROM puppet_metadata
-		WHERE repo_name = $1 AND ($2 = '' OR slug = $2)
-		ORDER BY slug, file_path
-	`, repoName, slug)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var result []provider.PuppetMetadata
-	for rows.Next() {
-		var m provider.PuppetMetadata
-		if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil {
-			return nil, err
-		}
-		result = append(result, m)
-	}
-	return result, rows.Err()
-}

internal/provider/ansible/ansible.go

@@ -0,0 +1,65 @@
+// Package ansible proxies an Ansible Galaxy server (v3 collections API):
+// mutable JSON metadata whose links are rewritten back to the remote, plus
+// immutable collection tarballs.
+package ansible
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	"git.unkin.net/unkin/artifactapi/internal/auth"
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func init() {
+	provider.Register(&Provider{})
+}
+
+type Provider struct{}
+
+func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
+
+func (p *Provider) Classify(path string) provider.Mutability {
+	if isArtifact(path) {
+		return provider.Immutable
+	}
+	return provider.Mutable
+}
+
+func (p *Provider) ContentType(path string) string {
+	if isArtifact(path) {
+		return "application/gzip"
+	}
+	return "application/json"
+}
+
+func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
+	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
+}
+
+// ForwardQuery keeps limit/offset pagination in the proxied path.
+func (p *Provider) ForwardQuery() {}
+
+// RewriteResponse points root-relative hrefs/links and absolute download_url
+// values at the remote. Links stay root-relative, which ansible-galaxy joins
+// onto the server URL, so the cached body is independent of the request host.
+func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
+	if !json.Valid(body) {
+		return nil, nil
+	}
+	prefix := `"/api/v1/remote/` + remote.Name + `/`
+	content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`)
+	content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix)
+	return []byte(content), nil
+}
+
+func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
+	return auth.BasicHeaders(remote), nil
+}
+
+func isArtifact(path string) bool {
+	return strings.HasSuffix(path, ".tar.gz")
+}

internal/provider/ansible/ansible_test.go

@@ -0,0 +1,85 @@
+package ansible
+
+import (
+	"encoding/json"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func TestClassify(t *testing.T) {
+	p := &Provider{}
+	tests := []struct {
+		path string
+		want provider.Mutability
+	}{
+		{"api/", provider.Mutable},
+		{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
+		{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
+		{"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable},
+	}
+	for _, tt := range tests {
+		if got := p.Classify(tt.path); got != tt.want {
+			t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
+		}
+	}
+}
+
+func TestUpstreamURLKeepsQuery(t *testing.T) {
+	p := &Provider{}
+	got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100")
+	want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100"
+	if got != want {
+		t.Errorf("UpstreamURL = %q, want %q", got, want)
+	}
+}
+
+func TestRewriteResponse(t *testing.T) {
+	p := &Provider{}
+	remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
+	body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` +
+		`"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` +
+		`"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` +
+		`"metadata":{"repository":"https://github.com/a/b"}}`
+
+	out, err := p.RewriteResponse([]byte(body), remote, "")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got struct {
+		Links       map[string]any `json:"links"`
+		Href        string         `json:"href"`
+		DownloadURL string         `json:"download_url"`
+		Metadata    map[string]any `json:"metadata"`
+	}
+	if err := json.Unmarshal(out, &got); err != nil {
+		t.Fatalf("rewritten body is not JSON: %v", err)
+	}
+	pre := "/api/v1/remote/galaxy/"
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want {
+		t.Errorf("next = %v, want %q", got.Links["next"], want)
+	}
+	if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want {
+		t.Errorf("href = %q, want %q", got.Href, want)
+	}
+	if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want {
+		t.Errorf("download_url = %q, want %q", got.DownloadURL, want)
+	}
+	if got.Metadata["repository"] != "https://github.com/a/b" {
+		t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"])
+	}
+}
+
+func TestRewriteResponseSkipsNonJSON(t *testing.T) {
+	out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
+	if err != nil || out != nil {
+		t.Errorf("got %q, %v; want nil, nil", out, err)
+	}
+}
+
+func TestForwardsQuery(t *testing.T) {
+	if _, ok := any(&Provider{}).(provider.QueryForwarder); !ok {
+		t.Error("ansible provider must forward the query string for pagination")
+	}
+}

internal/provider/provider.go

@@ -65,6 +65,13 @@ type MetadataStore interface {
 	InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
 }
 
+// QueryForwarder marks a remote provider whose upstream answers depend on the
+// query string (e.g. paginated JSON APIs); the query is kept as part of the
+// proxied and cached path.
+type QueryForwarder interface {
+	ForwardQuery()
+}
+
 // RemoteServer lets a remote provider fully answer a request itself instead of
 // going through the byte-proxy engine. It is the remote-side analog of
 // LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
@@ -240,29 +247,3 @@ func Get(t models.PackageType) (Provider, error) {
 func All() map[models.PackageType]Provider {
 	return registry
 }
-
-type PuppetMetadataStore interface {
-	InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error
-}
-
-type PuppetMetadataDeleter interface {
-	DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error
-}
-
-type PuppetMetadataReader interface {
-	ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error)
-}
-
-// PuppetMetadata is a local module release; MetadataJSON is the tarball's
-// metadata.json verbatim so Forge release objects carry every field.
-type PuppetMetadata struct {
-	RepoName     string
-	FilePath     string
-	ContentHash  string
-	Slug         string
-	Version      string
-	MetadataJSON string
-	Size         int64
-	MD5          string
-	CreatedAt    time.Time
-}

internal/provider/puppet/local.go

@@ -1,379 +0,0 @@
-package puppet
-
-import (
-	"archive/tar"
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/md5"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"net/http"
-	"net/url"
-	"path"
-	"regexp"
-	"slices"
-	"strconv"
-	"strings"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-	"git.unkin.net/unkin/artifactapi/internal/storage"
-)
-
-const filesPrefix = "v3/files/"
-
-// releaseRe matches <owner>-<name>-<semver>, the Forge release slug and the
-// tarball basename that `puppet module build` produces.
-var releaseRe = regexp.MustCompile(`^([a-z0-9]+)-([a-z][a-z0-9_]*)-(\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?)$`)
-
-func parseRelease(s string) (slug, version string, ok bool) {
-	m := releaseRe.FindStringSubmatch(strings.ToLower(s))
-	if m == nil {
-		return "", "", false
-	}
-	return m[1] + "-" + m[2], m[3], true
-}
-
-func normalizeSlug(s string) string {
-	return strings.ToLower(strings.ReplaceAll(s, "/", "-"))
-}
-
-// ValidateUpload stores tarballs under v3/files/ so Forge file_uri downloads
-// resolve straight to the stored file.
-func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
-	filename := strings.ToLower(path.Base(path.Clean("/" + filePath)))
-	if _, _, ok := parseRelease(strings.TrimSuffix(filename, ".tar.gz")); !ok || !strings.HasSuffix(filename, ".tar.gz") {
-		return "", "", fmt.Errorf("file must be named <owner>-<module>-<version>.tar.gz")
-	}
-	return filesPrefix + filename, "application/gzip", nil
-}
-
-func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
-	return map[string]any{
-		"filename":     path.Base(storagePath),
-		"content_hash": contentHash,
-		"size_bytes":   sizeBytes,
-	}
-}
-
-func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
-	reader, size, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
-	if err != nil {
-		slog.Error("puppet metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
-		return
-	}
-	defer func() { _ = reader.Close() }()
-
-	raw, err := io.ReadAll(reader)
-	if err != nil {
-		slog.Error("puppet metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
-		return
-	}
-	meta, err := parseModule(raw)
-	if err != nil {
-		slog.Error("puppet metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
-		return
-	}
-	slug, version, _ := parseRelease(strings.TrimSuffix(path.Base(storagePath), ".tar.gz"))
-	if meta.Slug != slug || meta.Version != version {
-		slog.Error("puppet metadata: metadata.json does not match filename", "repo", repoName, "path", storagePath, "name", meta.Slug, "version", meta.Version)
-		return
-	}
-	sum := md5.Sum(raw)
-	meta.RepoName = repoName
-	meta.FilePath = storagePath
-	meta.ContentHash = contentHash
-	meta.Size = size
-	meta.MD5 = hex.EncodeToString(sum[:])
-
-	store, ok := db.(provider.PuppetMetadataStore)
-	if !ok {
-		slog.Error("puppet metadata: store does not support puppet metadata", "repo", repoName)
-		return
-	}
-	if err := store.InsertPuppetMetadata(ctx, meta); err != nil {
-		slog.Error("puppet metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
-		return
-	}
-	slog.Info("puppet metadata: parsed", "repo", repoName, "module", meta.Slug, "version", meta.Version)
-}
-
-func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
-	deleter, ok := db.(provider.PuppetMetadataDeleter)
-	if !ok {
-		return nil
-	}
-	return deleter.DeletePuppetMetadata(ctx, repoName, storagePath)
-}
-
-const maxMetadataJSON = 1 << 20
-
-// parseModule reads the top-level <dir>/metadata.json from a module tarball.
-func parseModule(raw []byte) (*provider.PuppetMetadata, error) {
-	zr, err := gzip.NewReader(bytes.NewReader(raw))
-	if err != nil {
-		return nil, err
-	}
-	tr := tar.NewReader(zr)
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			return nil, errors.New("no metadata.json in module tarball")
-		}
-		if err != nil {
-			return nil, err
-		}
-		dir, file := path.Split(strings.TrimPrefix(path.Clean(hdr.Name), "./"))
-		if file != "metadata.json" || strings.Count(dir, "/") != 1 {
-			continue
-		}
-		body, err := io.ReadAll(io.LimitReader(tr, maxMetadataJSON))
-		if err != nil {
-			return nil, err
-		}
-		var md struct {
-			Name    string `json:"name"`
-			Version string `json:"version"`
-		}
-		if err := json.Unmarshal(body, &md); err != nil {
-			return nil, fmt.Errorf("parse metadata.json: %w", err)
-		}
-		if md.Name == "" || md.Version == "" {
-			return nil, errors.New("metadata.json missing name or version")
-		}
-		return &provider.PuppetMetadata{Slug: normalizeSlug(md.Name), Version: md.Version, MetadataJSON: string(body)}, nil
-	}
-}
-
-func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
-	clean := strings.TrimPrefix(path.Clean("/"+reqPath), "/")
-	var list func(slug string) ([]provider.PuppetMetadata, error)
-	if reader, ok := files.(provider.PuppetMetadataReader); ok {
-		list = func(slug string) ([]provider.PuppetMetadata, error) {
-			metas, err := reader.ListPuppetMetadataEntries(r.Context(), repoName, slug)
-			sortReleases(metas)
-			return metas, err
-		}
-	} else {
-		list = func(string) ([]provider.PuppetMetadata, error) {
-			return nil, errors.New("puppet metadata not available")
-		}
-	}
-
-	switch {
-	case clean == "v3/releases":
-		serveReleaseList(w, r, list)
-	case strings.HasPrefix(clean, "v3/releases/"):
-		slug, version, ok := parseRelease(strings.TrimPrefix(clean, "v3/releases/"))
-		if !ok {
-			notFound(w)
-			return true
-		}
-		metas, err := list(slug)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return true
-		}
-		for _, m := range metas {
-			if m.Version == version {
-				writeJSON(w, releaseObject(m))
-				return true
-			}
-		}
-		notFound(w)
-	case strings.HasPrefix(clean, "v3/modules/"):
-		slug := normalizeSlug(strings.TrimPrefix(clean, "v3/modules/"))
-		metas, err := list(slug)
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return true
-		}
-		if len(metas) == 0 {
-			notFound(w)
-			return true
-		}
-		writeJSON(w, moduleObject(metas))
-	default:
-		return false
-	}
-	return true
-}
-
-func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
-	return nil, fmt.Errorf("puppet local index generation for virtual repos not supported")
-}
-
-func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) ([]provider.PuppetMetadata, error)) {
-	q := r.URL.Query()
-	slug := normalizeSlug(q.Get("module"))
-	limit, err := strconv.Atoi(q.Get("limit"))
-	if err != nil || limit <= 0 {
-		limit = 20
-	}
-	limit = min(limit, 100)
-	offset, err := strconv.Atoi(q.Get("offset"))
-	if err != nil || offset < 0 {
-		offset = 0
-	}
-
-	metas, err := list(slug)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return
-	}
-	results := []map[string]any{}
-	for _, m := range metas[min(offset, len(metas)):min(offset+limit, len(metas))] {
-		results = append(results, releaseObject(m))
-	}
-
-	page := func(off int) any {
-		if off < 0 || off >= len(metas) && off != 0 {
-			return nil
-		}
-		v := url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}
-		if slug != "" {
-			v.Set("module", slug)
-		}
-		return "/v3/releases?" + v.Encode()
-	}
-	var previous any
-	if offset > 0 {
-		previous = page(max(offset-limit, 0))
-	}
-	writeJSON(w, map[string]any{
-		"pagination": map[string]any{
-			"limit":    limit,
-			"offset":   offset,
-			"first":    page(0),
-			"previous": previous,
-			"current":  page(offset),
-			"next":     page(offset + limit),
-			"total":    len(metas),
-		},
-		"results": results,
-	})
-}
-
-func owner(slug string) map[string]any {
-	name, _, _ := strings.Cut(slug, "-")
-	return map[string]any{"uri": "/v3/users/" + name, "slug": name, "username": name}
-}
-
-func moduleRef(slug string) map[string]any {
-	_, name, _ := strings.Cut(slug, "-")
-	return map[string]any{"uri": "/v3/modules/" + slug, "slug": slug, "name": name, "deprecated_at": nil, "owner": owner(slug)}
-}
-
-const forgeTime = "2006-01-02 15:04:05 -0700"
-
-func releaseObject(m provider.PuppetMetadata) map[string]any {
-	slug := m.Slug + "-" + m.Version
-	return map[string]any{
-		"uri":         "/v3/releases/" + slug,
-		"slug":        slug,
-		"module":      moduleRef(m.Slug),
-		"version":     m.Version,
-		"metadata":    json.RawMessage(m.MetadataJSON),
-		"tags":        []string{},
-		"supported":   false,
-		"pdk":         false,
-		"file_uri":    "/" + m.FilePath,
-		"file_size":   m.Size,
-		"file_md5":    m.MD5,
-		"file_sha256": strings.TrimPrefix(m.ContentHash, "sha256:"),
-		"downloads":   0,
-		"readme":      nil,
-		"changelog":   nil,
-		"license":     nil,
-		"created_at":  m.CreatedAt.Format(forgeTime),
-		"updated_at":  m.CreatedAt.Format(forgeTime),
-		"deleted_at":  nil,
-	}
-}
-
-// moduleObject expects metas sorted newest version first.
-func moduleObject(metas []provider.PuppetMetadata) map[string]any {
-	obj := moduleRef(metas[0].Slug)
-	releases := make([]map[string]any, 0, len(metas))
-	for _, m := range metas {
-		slug := m.Slug + "-" + m.Version
-		releases = append(releases, map[string]any{
-			"uri": "/v3/releases/" + slug, "slug": slug, "version": m.Version,
-			"file_uri": "/" + m.FilePath, "file_size": m.Size, "supported": false,
-			"created_at": m.CreatedAt.Format(forgeTime), "deleted_at": nil,
-		})
-	}
-	obj["current_release"] = releaseObject(metas[0])
-	obj["releases"] = releases
-	obj["downloads"] = 0
-	obj["created_at"] = metas[len(metas)-1].CreatedAt.Format(forgeTime)
-	obj["updated_at"] = metas[0].CreatedAt.Format(forgeTime)
-	return obj
-}
-
-func writeJSON(w http.ResponseWriter, v any) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusOK)
-	enc := json.NewEncoder(w)
-	enc.SetEscapeHTML(false)
-	_ = enc.Encode(v)
-}
-
-func notFound(w http.ResponseWriter) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusNotFound)
-	_, _ = io.WriteString(w, `{"message":"404 Not Found","errors":["The requested resource could not be found"]}`)
-}
-
-// sortReleases orders by module slug, then newest semver first.
-func sortReleases(metas []provider.PuppetMetadata) {
-	slices.SortStableFunc(metas, func(a, b provider.PuppetMetadata) int {
-		if c := strings.Compare(a.Slug, b.Slug); c != 0 {
-			return c
-		}
-		return compareSemver(b.Version, a.Version)
-	})
-}
-
-func compareSemver(a, b string) int {
-	a, _, _ = strings.Cut(a, "+")
-	b, _, _ = strings.Cut(b, "+")
-	aCore, aPre, aHasPre := strings.Cut(a, "-")
-	bCore, bPre, bHasPre := strings.Cut(b, "-")
-	if c := compareIdents(strings.Split(aCore, "."), strings.Split(bCore, ".")); c != 0 {
-		return c
-	}
-	switch {
-	case aHasPre && !bHasPre:
-		return -1
-	case !aHasPre && bHasPre:
-		return 1
-	}
-	return compareIdents(strings.Split(aPre, "."), strings.Split(bPre, "."))
-}
-
-func compareIdents(a, b []string) int {
-	for i := 0; i < len(a) && i < len(b); i++ {
-		an, aErr := strconv.Atoi(a[i])
-		bn, bErr := strconv.Atoi(b[i])
-		var c int
-		switch {
-		case aErr == nil && bErr == nil:
-			c = an - bn
-		case aErr == nil:
-			c = -1
-		case bErr == nil:
-			c = 1
-		default:
-			c = strings.Compare(a[i], b[i])
-		}
-		if c != 0 {
-			return c
-		}
-	}
-	return len(a) - len(b)
-}

internal/provider/puppet/local_test.go

@@ -1,214 +0,0 @@
-package puppet
-
-import (
-	"archive/tar"
-	"bytes"
-	"compress/gzip"
-	"context"
-	"crypto/md5"
-	"encoding/hex"
-	"encoding/json"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-)
-
-func moduleTarball(t *testing.T, dir, metadata string) []byte {
-	t.Helper()
-	var buf bytes.Buffer
-	zw := gzip.NewWriter(&buf)
-	tw := tar.NewWriter(zw)
-	for name, body := range map[string]string{dir + "/metadata.json": metadata, dir + "/manifests/init.pp": "class hello {}\n"} {
-		if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
-			t.Fatal(err)
-		}
-		_, _ = tw.Write([]byte(body))
-	}
-	_ = tw.Close()
-	_ = zw.Close()
-	return buf.Bytes()
-}
-
-const helloMetadata = `{"name":"unkin-hello","version":"0.1.0","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0"}]}`
-
-type fakeBlobs struct{ data []byte }
-
-func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
-	return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
-}
-
-type fakeStore struct{ inserted *provider.PuppetMetadata }
-
-func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
-func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
-func (f *fakeStore) InsertPuppetMetadata(_ context.Context, m *provider.PuppetMetadata) error {
-	f.inserted = m
-	return nil
-}
-
-type fakeReader struct{ metas []provider.PuppetMetadata }
-
-func (f fakeReader) ListPuppetMetadataEntries(_ context.Context, _, slug string) ([]provider.PuppetMetadata, error) {
-	var out []provider.PuppetMetadata
-	for _, m := range f.metas {
-		if slug == "" || m.Slug == slug {
-			out = append(out, m)
-		}
-	}
-	return out, nil
-}
-func (fakeReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
-	return nil, nil
-}
-func (fakeReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
-
-func TestValidateUpload(t *testing.T) {
-	p := &Provider{}
-	sp, ct, err := p.ValidateUpload("some/dir/Unkin-hello-0.1.0.tar.gz")
-	if err != nil || sp != "v3/files/unkin-hello-0.1.0.tar.gz" || ct != "application/gzip" {
-		t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
-	}
-	for _, bad := range []string{"hello-0.1.0.tar.gz", "unkin-hello-0.1.tar.gz", "unkin-hello-0.1.0.zip", "unkin-Hello_x-0.1.0.tgz"} {
-		if _, _, err := p.ValidateUpload(bad); err == nil {
-			t.Errorf("%q accepted", bad)
-		}
-	}
-}
-
-func TestAfterUploadRecordsMetadata(t *testing.T) {
-	raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata)
-	store := &fakeStore{}
-	(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
-	m := store.inserted
-	if m == nil {
-		t.Fatal("nothing inserted")
-	}
-	sum := md5.Sum(raw)
-	if m.Slug != "unkin-hello" || m.Version != "0.1.0" || m.MD5 != hex.EncodeToString(sum[:]) || m.Size != int64(len(raw)) || m.MetadataJSON != helloMetadata {
-		t.Errorf("got %+v", m)
-	}
-}
-
-func TestAfterUploadRejectsMismatchedFilename(t *testing.T) {
-	raw := moduleTarball(t, "unkin-hello-0.2.0", `{"name":"unkin-hello","version":"0.2.0"}`)
-	store := &fakeStore{}
-	(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
-	if store.inserted != nil {
-		t.Error("mismatched version indexed")
-	}
-}
-
-func TestParseModuleRequiresTopLevelMetadata(t *testing.T) {
-	if _, err := parseModule(moduleTarball(t, "a/b", helloMetadata)); err == nil {
-		t.Error("nested metadata.json accepted")
-	}
-	if _, err := parseModule(moduleTarball(t, "x", `{"name":"unkin-hello"}`)); err == nil {
-		t.Error("missing version accepted")
-	}
-}
-
-func get(t *testing.T, files provider.FileStore, target string) (int, map[string]any) {
-	t.Helper()
-	rec := httptest.NewRecorder()
-	req := httptest.NewRequest(http.MethodGet, "/x/"+target, nil)
-	path, _, _ := strings.Cut(target, "?")
-	if !(&Provider{}).ServeLocalIndex(rec, req, files, "forge", path) {
-		return 0, nil
-	}
-	var body map[string]any
-	_ = json.Unmarshal(rec.Body.Bytes(), &body)
-	return rec.Code, body
-}
-
-func releases(versions ...string) fakeReader {
-	var f fakeReader
-	for i, v := range versions {
-		f.metas = append(f.metas, provider.PuppetMetadata{
-			FilePath: "v3/files/unkin-hello-" + v + ".tar.gz", ContentHash: "sha256:feed", Slug: "unkin-hello",
-			Version: v, MetadataJSON: `{"name":"unkin-hello","version":"` + v + `"}`, MD5: "m" + v,
-			CreatedAt: time.Unix(int64(i), 0),
-		})
-	}
-	f.metas = append(f.metas, provider.PuppetMetadata{Slug: "other-mod", Version: "9.9.9", MetadataJSON: `{}`})
-	return f
-}
-
-func TestReleaseListPaginatesNewestFirst(t *testing.T) {
-	files := releases("0.1.0", "0.10.0", "0.2.0", "1.0.0-rc.1", "1.0.0")
-	code, body := get(t, files, "v3/releases?module=unkin/hello&limit=2")
-	if code != http.StatusOK {
-		t.Fatalf("code %d", code)
-	}
-	results := body["results"].([]any)
-	pg := body["pagination"].(map[string]any)
-	if len(results) != 2 || results[0].(map[string]any)["version"] != "1.0.0" || results[1].(map[string]any)["version"] != "1.0.0-rc.1" {
-		t.Fatalf("page 1: %v", results)
-	}
-	if pg["total"] != float64(5) || pg["next"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" || pg["previous"] != nil {
-		t.Errorf("pagination: %v", pg)
-	}
-
-	_, body = get(t, files, "v3/releases?module=unkin-hello&limit=2&offset=4")
-	results = body["results"].([]any)
-	pg = body["pagination"].(map[string]any)
-	if len(results) != 1 || results[0].(map[string]any)["version"] != "0.1.0" || pg["next"] != nil || pg["previous"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" {
-		t.Errorf("last page: %v %v", results, pg)
-	}
-
-	_, body = get(t, files, "v3/releases?module=nobody-none")
-	if len(body["results"].([]any)) != 0 {
-		t.Error("unknown module returned results")
-	}
-}
-
-func TestReleaseObject(t *testing.T) {
-	code, body := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.1.0")
-	if code != http.StatusOK {
-		t.Fatalf("code %d", code)
-	}
-	if body["file_uri"] != "/v3/files/unkin-hello-0.1.0.tar.gz" || body["file_md5"] != "m0.1.0" || body["file_sha256"] != "feed" {
-		t.Errorf("release: %v", body)
-	}
-	if body["metadata"].(map[string]any)["name"] != "unkin-hello" || body["module"].(map[string]any)["owner"].(map[string]any)["username"] != "unkin" {
-		t.Errorf("release refs: %v", body)
-	}
-	if code, _ := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.2.0"); code != http.StatusNotFound {
-		t.Errorf("missing version: %d", code)
-	}
-}
-
-func TestModuleObject(t *testing.T) {
-	code, body := get(t, releases("0.9.0", "0.10.0"), "v3/modules/unkin-hello")
-	if code != http.StatusOK {
-		t.Fatalf("code %d", code)
-	}
-	if body["slug"] != "unkin-hello" || body["name"] != "hello" || body["current_release"].(map[string]any)["version"] != "0.10.0" || len(body["releases"].([]any)) != 2 {
-		t.Errorf("module: %v", body)
-	}
-	if code, _ := get(t, releases(), "v3/modules/unkin-missing"); code != http.StatusNotFound {
-		t.Errorf("missing module: %d", code)
-	}
-}
-
-func TestFilesFallThrough(t *testing.T) {
-	if code, _ := get(t, releases(), "v3/files/unkin-hello-0.1.0.tar.gz"); code != 0 {
-		t.Error("file downloads must fall through to the stored file")
-	}
-}
-
-func TestCompareSemver(t *testing.T) {
-	ordered := []string{"0.1.0", "0.2.0", "0.10.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0", "1.0.1+build"}
-	for i := 1; i < len(ordered); i++ {
-		if compareSemver(ordered[i-1], ordered[i]) >= 0 || compareSemver(ordered[i], ordered[i-1]) <= 0 {
-			t.Errorf("%s should sort before %s", ordered[i-1], ordered[i])
-		}
-	}
-	if compareSemver("1.0.0+a", "1.0.0+b") != 0 {
-		t.Error("build metadata must not affect precedence")
-	}
-}

internal/server/server.go

@@ -22,6 +22,7 @@ import (
 	"git.unkin.net/unkin/artifactapi/internal/gc"
 	"git.unkin.net/unkin/artifactapi/internal/githubauth"
 	"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
+	_ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
 	"git.unkin.net/unkin/artifactapi/internal/provider/deb"
 	_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"

migrations/0004_puppet_metadata.sql

@@ -1,15 +0,0 @@
-CREATE TABLE IF NOT EXISTS puppet_metadata (
-	id            BIGSERIAL PRIMARY KEY,
-	repo_name     TEXT NOT NULL,
-	file_path     TEXT NOT NULL,
-	content_hash  TEXT NOT NULL,
-	slug          TEXT NOT NULL,
-	version       TEXT NOT NULL,
-	metadata_json TEXT NOT NULL,
-	file_size     BIGINT NOT NULL DEFAULT 0,
-	file_md5      TEXT NOT NULL,
-	created_at    TIMESTAMPTZ DEFAULT NOW(),
-	UNIQUE(repo_name, file_path)
-);
-
-CREATE INDEX IF NOT EXISTS idx_puppet_metadata_repo_slug ON puppet_metadata(repo_name, slug);

pkg/models/package_type.go

@@ -17,6 +17,7 @@ const (
 	PackageTerraform    PackageType = "terraform"
 	PackageGoProxy      PackageType = "goproxy"
 	PackageCargo        PackageType = "cargo"
+	PackageAnsible      PackageType = "ansible"
 	PackageGitHubRPM    PackageType = "github_rpm"
 	PackageGitHubDeb    PackageType = "github_deb"
 	PackageGitHubAlpine PackageType = "github_alpine"
@@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{
 	PackageTerraform:    true,
 	PackageGoProxy:      true,
 	PackageCargo:        true,
+	PackageAnsible:      true,
 	PackageGitHubRPM:    true,
 	PackageGitHubDeb:    true,
 	PackageGitHubAlpine: true,

pkg/models/package_type_test.go

@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
 		models.PackageTerraform,
 		models.PackageGoProxy,
 		models.PackageCargo,
+		models.PackageAnsible,
 		models.PackageGitHubRPM,
 		models.PackageGitHubDeb,
 		models.PackageGitHubAlpine,

ui/src/components/UsageInstructions.tsx

@@ -243,24 +243,23 @@ index = "sparse+${proxy}/"`,
         },
       ];
 
+    case 'ansible':
+      return [
+        {
+          title: 'Install collections through this Galaxy proxy',
+          language: 'bash',
+          code: `ansible-galaxy collection install community.general -s ${proxy}/api/`,
+        },
+      ];
+
     case 'puppet':
       return [
         {
-          title: isLocal ? 'Install a module from this forge' : 'Install a module from the Forge proxy',
+          title: 'Install a module from the Forge proxy',
           language: 'bash',
           code: `puppet module install <author>-<module> \\
-  --module_repository ${isLocal ? `${url}/api/v1/local/${name}` : proxy}`,
+  --module_repository ${proxy}`,
         },
-        ...(isLocal
-          ? [
-              {
-                title: 'Publish a module (versions are immutable)',
-                language: 'bash',
-                code: `puppet module build && curl -fsSL --upload-file pkg/<author>-<module>-<version>.tar.gz \\
-  ${url}/api/v2/remotes/${name}/files/<author>-<module>-<version>.tar.gz`,
-              },
-            ]
-          : []),
       ];
 
     case 'deb':

ui/src/pages/Remotes.tsx

@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
   terraform: 'blue',
   goproxy: 'green',
   cargo: 'yellow',
+  ansible: 'red',
 };
 
 export function Remotes() {