Add ansible Galaxy remote type (#137)
ansible-galaxy installs pull collections straight from galaxy.ansible.com because artifactapi has no Galaxy remote type. - add `ansible` package type proxying the Galaxy v3 API (discovery, collection versions, artifact downloads) - forward digits-only `limit` (capped at 100) and `offset` on metadata paths, canonicalised; tarball paths forward no query - rewrite only `href`, `download_url` and `links.*` to root-relative remote paths so tarballs flow through artifactapi - treat `artifacts/*.tar.gz` as immutable and JSON metadata as mutable - add `docs/ansible/remote.md`, a UI type def and an ansible e2e caching case requires unkin/artifactapi#146 Reviewed-on: #137 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #137.
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
@@ -0,0 +1,139 @@
|
||||
// Package ansible proxies an Ansible Galaxy server (v3 collections API):
|
||||
// mutable JSON metadata whose links are rewritten back to the remote, plus
|
||||
// immutable collection tarballs.
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if isArtifact(path) {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if isArtifact(path) {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
const maxPageLimit = 100
|
||||
|
||||
// ForwardedQuery keeps limit/offset pagination on metadata paths only.
|
||||
func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator {
|
||||
if isArtifact(path) {
|
||||
return nil
|
||||
}
|
||||
return map[string]provider.QueryValidator{"limit": pageLimit, "offset": pageOffset}
|
||||
}
|
||||
|
||||
func pageOffset(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
return strconv.FormatUint(n, 10), err == nil
|
||||
}
|
||||
|
||||
func pageLimit(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
if err != nil || n == 0 {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatUint(min(n, maxPageLimit), 10), true
|
||||
}
|
||||
|
||||
// RewriteResponse points href, download_url and links.* at the remote. Links
|
||||
// stay root-relative, which ansible-galaxy joins onto the server URL, so the
|
||||
// cached body is independent of the request host.
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.UseNumber()
|
||||
var doc any
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
r := rewriter{
|
||||
prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/",
|
||||
base: strings.TrimRight(remote.BaseURL, "/") + "/",
|
||||
}
|
||||
r.walk(doc)
|
||||
var out bytes.Buffer
|
||||
enc := json.NewEncoder(&out)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
type rewriter struct{ prefix, base string }
|
||||
|
||||
func (r rewriter) walk(v any) {
|
||||
switch v := v.(type) {
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
r.walk(e)
|
||||
}
|
||||
case map[string]any:
|
||||
for k, e := range v {
|
||||
switch k {
|
||||
case "href", "download_url":
|
||||
v[k] = r.link(e)
|
||||
case "links":
|
||||
if links, ok := e.(map[string]any); ok {
|
||||
for lk, l := range links {
|
||||
links[lk] = r.link(l)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
r.walk(v[k])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r rewriter) link(v any) any {
|
||||
s, ok := v.(string)
|
||||
switch {
|
||||
case !ok:
|
||||
return v
|
||||
case strings.HasPrefix(s, "/api/"):
|
||||
return r.prefix + s[1:]
|
||||
case strings.HasPrefix(s, r.base):
|
||||
return r.prefix + strings.TrimPrefix(s, r.base)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func isArtifact(path string) bool {
|
||||
path, _, _ = strings.Cut(path, "?")
|
||||
return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz")
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz"
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"api/", provider.Mutable},
|
||||
{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
|
||||
{"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable},
|
||||
{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
|
||||
{"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable},
|
||||
{tarball, provider.Immutable},
|
||||
{tarball + "?limit=1", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentTypeIgnoresQuery(t *testing.T) {
|
||||
if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" {
|
||||
t.Errorf("ContentType = %q, want application/json", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForwardedQuery(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if got := p.ForwardedQuery(tarball); got != nil {
|
||||
t.Errorf("tarball ForwardedQuery = %v, want nil", got)
|
||||
}
|
||||
q := p.ForwardedQuery("api/v3/collections/a/b/versions/")
|
||||
if len(q) != 2 || q["limit"] == nil || q["offset"] == nil {
|
||||
t.Fatalf("metadata ForwardedQuery keys = %v, want limit and offset", q)
|
||||
}
|
||||
tests := []struct {
|
||||
param, in, want string
|
||||
ok bool
|
||||
}{
|
||||
{"limit", "10", "10", true},
|
||||
{"limit", "007", "7", true},
|
||||
{"limit", "100", "100", true},
|
||||
{"limit", "5000", "100", true},
|
||||
{"limit", "0", "", false},
|
||||
{"limit", "", "", false},
|
||||
{"limit", "-1", "", false},
|
||||
{"limit", "+5", "", false},
|
||||
{"limit", "1e3", "", false},
|
||||
{"limit", "10 ", "", false},
|
||||
{"offset", "0", "0", true},
|
||||
{"offset", "000", "0", true},
|
||||
{"offset", "0100", "100", true},
|
||||
{"offset", "abc", "", false},
|
||||
{"offset", "99999999999999999999", "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, ok := q[tt.param](tt.in)
|
||||
if ok != tt.ok || (ok && got != tt.want) {
|
||||
t.Errorf("%s(%q) = %q, %v, want %q, %v", tt.param, tt.in, got, ok, tt.want, tt.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func rewrite(t *testing.T, remote models.Remote, body string) map[string]any {
|
||||
t.Helper()
|
||||
out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(out, &got); err != nil {
|
||||
t.Fatalf("rewritten body is not JSON: %v: %s", err, out)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
|
||||
got := rewrite(t, remote, `{
|
||||
"links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null},
|
||||
"data": [{
|
||||
"href": "/api/v3/collections/a/b/versions/1.0.0/",
|
||||
"download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",
|
||||
"description": "/api/ is where it lives",
|
||||
"docs": "/api/v3/not-a-link/",
|
||||
"metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"}
|
||||
}]
|
||||
}`)
|
||||
pre := "/api/v1/remote/galaxy/"
|
||||
links := got["links"].(map[string]any)
|
||||
if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want {
|
||||
t.Errorf("links.next = %v, want %q", links["next"], want)
|
||||
}
|
||||
if links["previous"] != nil {
|
||||
t.Errorf("links.previous = %v, want null", links["previous"])
|
||||
}
|
||||
item := got["data"].([]any)[0].(map[string]any)
|
||||
if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want {
|
||||
t.Errorf("href = %v, want %q", item["href"], want)
|
||||
}
|
||||
if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want {
|
||||
t.Errorf("download_url = %v, want %q", item["download_url"], want)
|
||||
}
|
||||
for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} {
|
||||
if item[k] != want {
|
||||
t.Errorf("%s rewritten to %v", k, item[k])
|
||||
}
|
||||
}
|
||||
meta := item["metadata"].(map[string]any)
|
||||
if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" {
|
||||
t.Errorf("unrelated URLs rewritten: %v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseEscapesRemoteName(t *testing.T) {
|
||||
got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`)
|
||||
if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want {
|
||||
t.Errorf("href = %v, want %q", got["href"], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseKeepsLargeNumbers(t *testing.T) {
|
||||
out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "{\"n\":12345678901234567890}\n"; string(out) != want {
|
||||
t.Errorf("got %q, want %q", out, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseSkipsNonJSON(t *testing.T) {
|
||||
out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
|
||||
if err != nil || out != nil {
|
||||
t.Errorf("got %q, %v; want nil, nil", out, err)
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,9 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// RemoteMountPath is the URL path remote proxies are served under.
|
||||
const RemoteMountPath = "/api/v1/remote/"
|
||||
|
||||
type Mutability int
|
||||
|
||||
const (
|
||||
|
||||
Reference in New Issue
Block a user