Add ansible Galaxy remote type #137

Open
unkin-agent wants to merge 7 commits from benvin/remote-ansible into benvin/remote-query-forwarder
12 changed files with 309 additions and 1 deletions
+1
View File
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
| `terraform` | `*/versions` | `*/download/*/*` |
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
| `ansible` | Galaxy `api/` JSON (links rewritten, `limit`/`offset` forwarded) | `artifacts/*.tar.gz` |
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
+1 -1
View File
@@ -23,7 +23,7 @@ already-running stack.
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget, ansible) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
+1
View File
@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"},
{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
+139
View File
@@ -0,0 +1,139 @@
// Package ansible proxies an Ansible Galaxy server (v3 collections API):
// mutable JSON metadata whose links are rewritten back to the remote, plus
// immutable collection tarballs.
package ansible
import (
"bytes"
"context"
"encoding/json"
"net/http"
"net/url"
"strconv"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
func (p *Provider) Classify(path string) provider.Mutability {
if isArtifact(path) {
return provider.Immutable
}
return provider.Mutable
}
func (p *Provider) ContentType(path string) string {
if isArtifact(path) {
return "application/gzip"
}
return "application/json"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
const maxPageLimit = 100
// ForwardedQuery keeps limit/offset pagination on metadata paths only.
func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator {
if isArtifact(path) {
return nil
}
return map[string]provider.QueryValidator{"limit": pageLimit, "offset": pageOffset}
}
func pageOffset(v string) (string, bool) {
n, err := strconv.ParseUint(v, 10, 32)
return strconv.FormatUint(n, 10), err == nil
}
func pageLimit(v string) (string, bool) {
n, err := strconv.ParseUint(v, 10, 32)
if err != nil || n == 0 {
return "", false
}
return strconv.FormatUint(min(n, maxPageLimit), 10), true
}
// RewriteResponse points href, download_url and links.* at the remote. Links
// stay root-relative, which ansible-galaxy joins onto the server URL, so the
// cached body is independent of the request host.
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
dec := json.NewDecoder(bytes.NewReader(body))
dec.UseNumber()
var doc any
if err := dec.Decode(&doc); err != nil {
return nil, nil
}
r := rewriter{
prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/",
base: strings.TrimRight(remote.BaseURL, "/") + "/",
}
r.walk(doc)
var out bytes.Buffer
enc := json.NewEncoder(&out)
enc.SetEscapeHTML(false)
if err := enc.Encode(doc); err != nil {
return nil, err
}
return out.Bytes(), nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
type rewriter struct{ prefix, base string }
func (r rewriter) walk(v any) {
switch v := v.(type) {
case []any:
for _, e := range v {
r.walk(e)
}
case map[string]any:
for k, e := range v {
switch k {
case "href", "download_url":
v[k] = r.link(e)
case "links":
if links, ok := e.(map[string]any); ok {
for lk, l := range links {
links[lk] = r.link(l)
}
continue
}
}
r.walk(v[k])
}
}
}
func (r rewriter) link(v any) any {
s, ok := v.(string)
switch {
case !ok:
return v
case strings.HasPrefix(s, "/api/"):
return r.prefix + s[1:]
case strings.HasPrefix(s, r.base):
return r.prefix + strings.TrimPrefix(s, r.base)
}
return s
}
func isArtifact(path string) bool {
path, _, _ = strings.Cut(path, "?")
return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz")
}
+150
View File
@@ -0,0 +1,150 @@
package ansible
import (
"encoding/json"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz"
func TestClassify(t *testing.T) {
p := &Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"api/", provider.Mutable},
{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
{"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable},
{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
{"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable},
{tarball, provider.Immutable},
{tarball + "?limit=1", provider.Immutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestContentTypeIgnoresQuery(t *testing.T) {
if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" {
t.Errorf("ContentType = %q, want application/json", got)
}
}
func TestForwardedQuery(t *testing.T) {
p := &Provider{}
if got := p.ForwardedQuery(tarball); got != nil {
t.Errorf("tarball ForwardedQuery = %v, want nil", got)
}
q := p.ForwardedQuery("api/v3/collections/a/b/versions/")
if len(q) != 2 || q["limit"] == nil || q["offset"] == nil {
t.Fatalf("metadata ForwardedQuery keys = %v, want limit and offset", q)
}
tests := []struct {
param, in, want string
ok bool
}{
{"limit", "10", "10", true},
{"limit", "007", "7", true},
{"limit", "100", "100", true},
{"limit", "5000", "100", true},
{"limit", "0", "", false},
{"limit", "", "", false},
{"limit", "-1", "", false},
{"limit", "+5", "", false},
{"limit", "1e3", "", false},
{"limit", "10 ", "", false},
{"offset", "0", "0", true},
{"offset", "000", "0", true},
{"offset", "0100", "100", true},
{"offset", "abc", "", false},
{"offset", "99999999999999999999", "", false},
}
for _, tt := range tests {
got, ok := q[tt.param](tt.in)
if ok != tt.ok || (ok && got != tt.want) {
t.Errorf("%s(%q) = %q, %v, want %q, %v", tt.param, tt.in, got, ok, tt.want, tt.ok)
}
}
}
func rewrite(t *testing.T, remote models.Remote, body string) map[string]any {
t.Helper()
out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "")
if err != nil {
t.Fatal(err)
}
var got map[string]any
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("rewritten body is not JSON: %v: %s", err, out)
}
return got
}
func TestRewriteResponse(t *testing.T) {
remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
got := rewrite(t, remote, `{
"links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null},
"data": [{
"href": "/api/v3/collections/a/b/versions/1.0.0/",
"download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",
"description": "/api/ is where it lives",
"docs": "/api/v3/not-a-link/",
"metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"}
}]
}`)
pre := "/api/v1/remote/galaxy/"
links := got["links"].(map[string]any)
if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want {
t.Errorf("links.next = %v, want %q", links["next"], want)
}
if links["previous"] != nil {
t.Errorf("links.previous = %v, want null", links["previous"])
}
item := got["data"].([]any)[0].(map[string]any)
if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want {
t.Errorf("href = %v, want %q", item["href"], want)
}
if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want {
t.Errorf("download_url = %v, want %q", item["download_url"], want)
}
for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} {
if item[k] != want {
t.Errorf("%s rewritten to %v", k, item[k])
}
}
meta := item["metadata"].(map[string]any)
if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" {
t.Errorf("unrelated URLs rewritten: %v", meta)
}
}
func TestRewriteResponseEscapesRemoteName(t *testing.T) {
got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`)
if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want {
t.Errorf("href = %v, want %q", got["href"], want)
}
}
func TestRewriteResponseKeepsLargeNumbers(t *testing.T) {
out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "")
if err != nil {
t.Fatal(err)
}
if want := "{\"n\":12345678901234567890}\n"; string(out) != want {
t.Errorf("got %q, want %q", out, want)
}
}
func TestRewriteResponseSkipsNonJSON(t *testing.T) {
out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
if err != nil || out != nil {
t.Errorf("got %q, %v; want nil, nil", out, err)
}
}
+3
View File
@@ -10,6 +10,9 @@ import (
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// RemoteMountPath is the URL path remote proxies are served under.
const RemoteMountPath = "/api/v1/remote/"
type Mutability int
const (
+1
View File
@@ -22,6 +22,7 @@ import (
"git.unkin.net/unkin/artifactapi/internal/gc"
"git.unkin.net/unkin/artifactapi/internal/githubauth"
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
_ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
+2
View File
@@ -17,6 +17,7 @@ const (
PackageTerraform PackageType = "terraform"
PackageGoProxy PackageType = "goproxy"
PackageCargo PackageType = "cargo"
PackageAnsible PackageType = "ansible"
PackageNuGet PackageType = "nuget"
PackageGitHubRPM PackageType = "github_rpm"
PackageGitHubDeb PackageType = "github_deb"
@@ -36,6 +37,7 @@ var validPackageTypes = map[PackageType]bool{
PackageTerraform: true,
PackageGoProxy: true,
PackageCargo: true,
PackageAnsible: true,
PackageNuGet: true,
PackageGitHubRPM: true,
PackageGitHubDeb: true,
+1
View File
@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
models.PackageTerraform,
models.PackageGoProxy,
models.PackageCargo,
models.PackageAnsible,
models.PackageNuGet,
models.PackageGitHubRPM,
models.PackageGitHubDeb,
+9
View File
@@ -243,6 +243,15 @@ index = "sparse+${proxy}/"`,
},
];
case 'ansible':
return [
{
title: 'Install collections through this Galaxy proxy',
language: 'bash',
code: `ansible-galaxy collection install community.general -s ${proxy}/api/`,
},
];
case 'nuget':
return [
{
+1
View File
@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
terraform: 'blue',
goproxy: 'green',
cargo: 'yellow',
ansible: 'red',
nuget: 'blue',
};