Add ansible virtual repositories (#158)

Clients need one Galaxy server to install both internal collections and galaxy.ansible.com ones; ansible has no virtual type.

- merge member collection version lists in member order, the earlier member winning a duplicate version, cached with the shared virtual merge cache
- paginate limit/offset over the merged list, following remote member pages
- serve version detail from the owning member with href and download_url via the virtual; downloads 302 to the member through the member-redirect validation
- fail closed with 502 on a member error or a member versions list shorter than its count
- add docs/ansible/virtual.md and an e2e-docker case driving real ansible-galaxy

requires unkin/artifactapi#157

Reviewed-on: #158
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

internal/api/v1/ansible_handler_test.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"mime/multipart"
@@ -13,6 +14,7 @@ import (
 	"strings"
 	"testing"
 
+	"git.unkin.net/unkin/artifactapi/internal/virtual"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -214,3 +216,60 @@ func TestAnsibleLocalPerHost(t *testing.T) {
 		t.Errorf("Cache-Control %q Vary %q", cc, vary)
 	}
 }
+
+func TestAnsibleVirtualOverLocals(t *testing.T) {
+	h, srv := npmTestServer(t)
+	h.virtualEngine = virtual.NewEngine(h.db, nil)
+	h.virtualEngine.AnsibleLocal = h.LocalAnsible
+	a, b := npmRepo(t, h, models.PackageAnsible), npmRepo(t, h, models.PackageAnsible)
+	for _, p := range []struct{ repo, version string }{{a, "1.0.0"}, {b, "1.0.0"}, {b, "2.0.0"}} {
+		if code, body := ansiblePublish(t, srv.URL+"/api/v1/local/"+p.repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "virt", p.version), ""); code != http.StatusAccepted {
+			t.Fatalf("publish %s to %s = %d: %s", p.version, p.repo, code, body)
+		}
+	}
+	virt := "v" + a
+	if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageAnsible, Members: []string{a, b}}); err != nil {
+		t.Fatal(err)
+	}
+	base := srv.URL + "/api/v1/virtual/" + virt + "/"
+
+	var page struct {
+		Meta  struct{ Count int }
+		Links struct{ Next *string }
+		Data  []struct{ Version, Href string }
+	}
+	getJSON(t, base+"api/v3/collections/unkin/virt/versions/?limit=1", &page)
+	if page.Meta.Count != 2 || len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next == nil {
+		t.Fatalf("page 1 = %+v", page)
+	}
+	href := page.Data[0].Href
+	if !strings.HasPrefix(*page.Links.Next, "/api/v1/virtual/"+virt+"/") {
+		t.Fatalf("next = %q, want root-relative", *page.Links.Next)
+	}
+	getJSON(t, srv.URL+*page.Links.Next, &page)
+	if len(page.Data) != 1 || page.Data[0].Version != "2.0.0" || page.Links.Next != nil {
+		t.Fatalf("page 2 = %+v", page)
+	}
+
+	var detail struct {
+		DownloadURL string `json:"download_url"`
+	}
+	getJSON(t, href, &detail)
+	if detail.DownloadURL != base+a+"/artifacts/unkin-virt-1.0.0.tar.gz" {
+		t.Fatalf("download_url = %q, want the first member via the virtual", detail.DownloadURL)
+	}
+	noFollow := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
+	resp, err := noFollow.Get(detail.DownloadURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_ = resp.Body.Close()
+	if resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != srv.URL+"/api/v1/local/"+a+"/artifacts/unkin-virt-1.0.0.tar.gz" {
+		t.Fatalf("download = %d -> %q", resp.StatusCode, resp.Header.Get("Location"))
+	}
+	for _, p := range []string{"api/v3/collections/unkin/nope/", "notamember/x.tar.gz"} {
+		if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound {
+			t.Errorf("GET %s = %d, want 404", p, code)
+		}
+	}
+}

internal/api/v1/ansible_virtual.go

@@ -0,0 +1,64 @@
+package v1
+
+import (
+	"encoding/json"
+	"errors"
+	"log/slog"
+	"net/http"
+	"net/url"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
+	"git.unkin.net/unkin/artifactapi/internal/virtual"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+// serveVirtualAnsible serves the merged Galaxy v3 API and 302s downloads to
+// the member owning that version.
+func (h *ProxyHandler) serveVirtualAnsible(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) {
+	ap := ansible.ParseAPIPath(path)
+	base := proxyBaseURL + "/api/v1/virtual/" + url.PathEscape(virt.Name)
+	var doc any
+	var err error
+	switch ap.Kind {
+	case ansible.APIRoot:
+		doc = ansible.Root()
+	case ansible.APIVersion:
+		var body []byte
+		if body, err = h.virtualEngine.AnsibleVersion(r.Context(), virt, ap.Namespace, ap.Name, ap.Version, proxyBaseURL); err == nil {
+			doc = json.RawMessage(body)
+		}
+	case ansible.APICollection, ansible.APIVersions:
+		var d *virtual.AnsibleDoc
+		if d, err = h.virtualEngine.AnsibleVersions(r.Context(), virt, ap.Namespace, ap.Name); err == nil {
+			if ap.Kind == ansible.APICollection {
+				doc = ansible.CollectionDoc(base, ap.Namespace, ap.Name, d.Refs)
+			} else {
+				limit, offset := ansible.Page(r.URL.Query())
+				doc = ansible.VersionsPage(base, ap.Namespace, ap.Name, d.Refs, limit, offset)
+			}
+		}
+	default:
+		loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL)
+		switch {
+		case err != nil:
+			ansibleError(w, http.StatusBadRequest, err.Error())
+		case !ok:
+			ansibleError(w, http.StatusNotFound, "not found")
+		default:
+			w.Header().Set("Cache-Control", "no-store")
+			w.Header().Set("Vary", "Host, X-Forwarded-Proto")
+			http.Redirect(w, r, loc, http.StatusFound)
+		}
+		return
+	}
+	switch {
+	case errors.Is(err, virtual.ErrNotFound):
+		ansibleError(w, http.StatusNotFound, "not found")
+	case err != nil:
+		slog.Error("virtual ansible fetch failed", "virtual", virt.Name, "path", path, "error", err)
+		ansibleError(w, http.StatusBadGateway, "bad gateway")
+	default:
+		w.Header().Set("X-Artifact-Source", "virtual")
+		writeAnsibleJSON(w, http.StatusOK, doc)
+	}
+}

internal/api/v1/proxy.go

@@ -211,6 +211,10 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
 		h.serveVirtualGems(w, r, *virt, path, proxyBaseURL)
 		return
 	}
+	if virt.PackageType == models.PackageAnsible {
+		h.serveVirtualAnsible(w, r, *virt, path, proxyBaseURL)
+		return
+	}
 
 	loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
 	if err != nil {