Add ansible Galaxy remote type #137
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
|
||||
| `ansible` | Galaxy `api/` JSON (links rewritten, `limit`/`offset` forwarded) | `artifacts/*.tar.gz` |
|
||||
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
|
||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ already-running stack.
|
||||
|
||||
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
|
||||
- **Caching** — one immutable artifact per remote package type (generic, docker,
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the
|
||||
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget, ansible) proxied through the
|
||||
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
|
||||
verified against the origin fixture.
|
||||
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
|
||||
|
||||
@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
|
||||
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
|
||||
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
|
||||
{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
|
||||
{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"},
|
||||
{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"},
|
||||
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
|
||||
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
|
||||
|
||||
Binary file not shown.
@@ -0,0 +1,139 @@
|
||||
// Package ansible proxies an Ansible Galaxy server (v3 collections API):
|
||||
// mutable JSON metadata whose links are rewritten back to the remote, plus
|
||||
// immutable collection tarballs.
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if isArtifact(path) {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if isArtifact(path) {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
const maxPageLimit = 100
|
||||
|
||||
// ForwardedQuery keeps limit/offset pagination on metadata paths only.
|
||||
func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator {
|
||||
if isArtifact(path) {
|
||||
return nil
|
||||
}
|
||||
return map[string]provider.QueryValidator{"limit": pageLimit, "offset": pageOffset}
|
||||
}
|
||||
|
||||
func pageOffset(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
return strconv.FormatUint(n, 10), err == nil
|
||||
}
|
||||
|
||||
func pageLimit(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
if err != nil || n == 0 {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatUint(min(n, maxPageLimit), 10), true
|
||||
}
|
||||
|
||||
// RewriteResponse points href, download_url and links.* at the remote. Links
|
||||
// stay root-relative, which ansible-galaxy joins onto the server URL, so the
|
||||
// cached body is independent of the request host.
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.UseNumber()
|
||||
var doc any
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
r := rewriter{
|
||||
prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/",
|
||||
base: strings.TrimRight(remote.BaseURL, "/") + "/",
|
||||
}
|
||||
r.walk(doc)
|
||||
var out bytes.Buffer
|
||||
enc := json.NewEncoder(&out)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
type rewriter struct{ prefix, base string }
|
||||
|
||||
func (r rewriter) walk(v any) {
|
||||
switch v := v.(type) {
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
r.walk(e)
|
||||
}
|
||||
case map[string]any:
|
||||
for k, e := range v {
|
||||
switch k {
|
||||
case "href", "download_url":
|
||||
v[k] = r.link(e)
|
||||
case "links":
|
||||
if links, ok := e.(map[string]any); ok {
|
||||
for lk, l := range links {
|
||||
links[lk] = r.link(l)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
r.walk(v[k])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r rewriter) link(v any) any {
|
||||
s, ok := v.(string)
|
||||
switch {
|
||||
case !ok:
|
||||
return v
|
||||
case strings.HasPrefix(s, "/api/"):
|
||||
return r.prefix + s[1:]
|
||||
case strings.HasPrefix(s, r.base):
|
||||
return r.prefix + strings.TrimPrefix(s, r.base)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func isArtifact(path string) bool {
|
||||
path, _, _ = strings.Cut(path, "?")
|
||||
return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz")
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz"
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"api/", provider.Mutable},
|
||||
{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
|
||||
{"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable},
|
||||
{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
|
||||
{"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable},
|
||||
{tarball, provider.Immutable},
|
||||
{tarball + "?limit=1", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentTypeIgnoresQuery(t *testing.T) {
|
||||
if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" {
|
||||
t.Errorf("ContentType = %q, want application/json", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForwardedQuery(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if got := p.ForwardedQuery(tarball); got != nil {
|
||||
t.Errorf("tarball ForwardedQuery = %v, want nil", got)
|
||||
}
|
||||
q := p.ForwardedQuery("api/v3/collections/a/b/versions/")
|
||||
if len(q) != 2 || q["limit"] == nil || q["offset"] == nil {
|
||||
t.Fatalf("metadata ForwardedQuery keys = %v, want limit and offset", q)
|
||||
}
|
||||
tests := []struct {
|
||||
param, in, want string
|
||||
ok bool
|
||||
}{
|
||||
{"limit", "10", "10", true},
|
||||
{"limit", "007", "7", true},
|
||||
{"limit", "100", "100", true},
|
||||
{"limit", "5000", "100", true},
|
||||
{"limit", "0", "", false},
|
||||
{"limit", "", "", false},
|
||||
{"limit", "-1", "", false},
|
||||
{"limit", "+5", "", false},
|
||||
{"limit", "1e3", "", false},
|
||||
{"limit", "10 ", "", false},
|
||||
{"offset", "0", "0", true},
|
||||
{"offset", "000", "0", true},
|
||||
{"offset", "0100", "100", true},
|
||||
{"offset", "abc", "", false},
|
||||
{"offset", "99999999999999999999", "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, ok := q[tt.param](tt.in)
|
||||
if ok != tt.ok || (ok && got != tt.want) {
|
||||
t.Errorf("%s(%q) = %q, %v, want %q, %v", tt.param, tt.in, got, ok, tt.want, tt.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func rewrite(t *testing.T, remote models.Remote, body string) map[string]any {
|
||||
t.Helper()
|
||||
out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(out, &got); err != nil {
|
||||
t.Fatalf("rewritten body is not JSON: %v: %s", err, out)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
|
||||
got := rewrite(t, remote, `{
|
||||
"links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null},
|
||||
"data": [{
|
||||
"href": "/api/v3/collections/a/b/versions/1.0.0/",
|
||||
"download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",
|
||||
"description": "/api/ is where it lives",
|
||||
"docs": "/api/v3/not-a-link/",
|
||||
"metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"}
|
||||
}]
|
||||
}`)
|
||||
pre := "/api/v1/remote/galaxy/"
|
||||
links := got["links"].(map[string]any)
|
||||
if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want {
|
||||
t.Errorf("links.next = %v, want %q", links["next"], want)
|
||||
}
|
||||
if links["previous"] != nil {
|
||||
t.Errorf("links.previous = %v, want null", links["previous"])
|
||||
}
|
||||
item := got["data"].([]any)[0].(map[string]any)
|
||||
if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want {
|
||||
t.Errorf("href = %v, want %q", item["href"], want)
|
||||
}
|
||||
if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want {
|
||||
t.Errorf("download_url = %v, want %q", item["download_url"], want)
|
||||
}
|
||||
for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} {
|
||||
if item[k] != want {
|
||||
t.Errorf("%s rewritten to %v", k, item[k])
|
||||
}
|
||||
}
|
||||
meta := item["metadata"].(map[string]any)
|
||||
if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" {
|
||||
t.Errorf("unrelated URLs rewritten: %v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseEscapesRemoteName(t *testing.T) {
|
||||
got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`)
|
||||
if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want {
|
||||
t.Errorf("href = %v, want %q", got["href"], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseKeepsLargeNumbers(t *testing.T) {
|
||||
out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "{\"n\":12345678901234567890}\n"; string(out) != want {
|
||||
t.Errorf("got %q, want %q", out, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseSkipsNonJSON(t *testing.T) {
|
||||
out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
|
||||
if err != nil || out != nil {
|
||||
t.Errorf("got %q, %v; want nil, nil", out, err)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,9 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// RemoteMountPath is the URL path remote proxies are served under.
|
||||
const RemoteMountPath = "/api/v1/remote/"
|
||||
|
||||
type Mutability int
|
||||
|
||||
const (
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
|
||||
@@ -17,6 +17,7 @@ const (
|
||||
PackageTerraform PackageType = "terraform"
|
||||
PackageGoProxy PackageType = "goproxy"
|
||||
PackageCargo PackageType = "cargo"
|
||||
PackageAnsible PackageType = "ansible"
|
||||
PackageNuGet PackageType = "nuget"
|
||||
PackageGitHubRPM PackageType = "github_rpm"
|
||||
PackageGitHubDeb PackageType = "github_deb"
|
||||
@@ -36,6 +37,7 @@ var validPackageTypes = map[PackageType]bool{
|
||||
PackageTerraform: true,
|
||||
PackageGoProxy: true,
|
||||
PackageCargo: true,
|
||||
PackageAnsible: true,
|
||||
PackageNuGet: true,
|
||||
PackageGitHubRPM: true,
|
||||
PackageGitHubDeb: true,
|
||||
|
||||
@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
|
||||
models.PackageTerraform,
|
||||
models.PackageGoProxy,
|
||||
models.PackageCargo,
|
||||
models.PackageAnsible,
|
||||
models.PackageNuGet,
|
||||
models.PackageGitHubRPM,
|
||||
models.PackageGitHubDeb,
|
||||
|
||||
@@ -243,6 +243,15 @@ index = "sparse+${proxy}/"`,
|
||||
},
|
||||
];
|
||||
|
||||
case 'ansible':
|
||||
return [
|
||||
{
|
||||
title: 'Install collections through this Galaxy proxy',
|
||||
language: 'bash',
|
||||
code: `ansible-galaxy collection install community.general -s ${proxy}/api/`,
|
||||
},
|
||||
];
|
||||
|
||||
case 'nuget':
|
||||
return [
|
||||
{
|
||||
|
||||
@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
|
||||
terraform: 'blue',
|
||||
goproxy: 'green',
|
||||
cargo: 'yellow',
|
||||
ansible: 'red',
|
||||
nuget: 'blue',
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user