Add local ansible Galaxy repositories
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful

This commit is contained in:
2026-10-18 03:37:52 +11:00
parent f4d3093952
commit ba8d3a3c39
11 changed files with 1149 additions and 3 deletions
+1 -1
View File
@@ -27,7 +27,7 @@ Classification precedence, TTLs and eviction for every remote: [docs/common/cach
Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
- `ansible`: [remote](docs/ansible/remote.md)
- `ansible`: [remote](docs/ansible/remote.md), [local](docs/ansible/local.md)
- `cargo`: [remote](docs/cargo/remote.md)
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
+26
View File
@@ -0,0 +1,26 @@
# Local ansible repository
A local `ansible` repo is a Galaxy v3 server: `ansible-galaxy collection publish`
and `ansible-galaxy collection install` work against `/api/v1/local/<repo>`.
`publish` needs an API key; any value works.
```bash
ansible-galaxy collection build
ansible-galaxy collection publish unkin-hello-1.0.0.tar.gz \
--server https://artifactapi.example.com/api/v1/local/<repo>/ --api-key unused
ansible-galaxy collection install unkin.hello \
--server https://artifactapi.example.com/api/v1/local/<repo>/
```
Publishing reads `collection_info` from the tarball's `MANIFEST.json` and
finishes the import synchronously, so the import task is already `completed`.
Uploads over 256 MiB, tarballs with members or symlinks escaping the collection root, or
with an invalid name, version or dependency map are rejected with 400.
The repo serves `api/v3/collections/<ns>/<name>/`, the paginated
`versions/` list (`limit` up to 100, `offset`), `versions/<version>/` with
`download_url`, `artifact.sha256` and `metadata.dependencies`, and
`artifacts/<ns>-<name>-<version>.tar.gz`.
Versions are immutable: publishing an existing `namespace.name:version` is
rejected with 409.
+2 -1
View File
@@ -31,7 +31,8 @@ already-running stack.
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
second version, republish rejected), gems (stock `ruby:3.3`: `gem push`,
`gem install`, `bundle install` + `require`, republish rejected, `gem yank`,
yanked re-push rejected).
yanked re-push rejected), ansible (stock `ansible-galaxy`: `collection build`,
`publish`, republish rejected, `install` + module run).
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
across two members; npm packument merge over two locals via a stock `node:22`
client (merged versions, install through the member redirect); gems compact
+66
View File
@@ -0,0 +1,66 @@
//go:build dockere2e
package e2edocker
import (
"fmt"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// ansibleHelloScript builds unkin.e2e_hello 1.0.0 with one module into /w/dist.
const ansibleHelloScript = `pip install -q ansible-core >/dev/null 2>&1
mkdir -p /w && cd /w && ansible-galaxy collection init unkin.e2e_hello >/dev/null
cd unkin/e2e_hello && sed -i 's/^version: .*/version: 1.0.0/' galaxy.yml && mkdir -p plugins/modules
printf '%s\n' 'DOCUMENTATION = r"""' 'module: hello' 'short_description: e2e greeting' 'description: Returns a greeting.' 'options:' ' name: {type: str, default: world, description: Who to greet.}' '"""' \
'from ansible.module_utils.basic import AnsibleModule' \
'm = AnsibleModule(argument_spec=dict(name=dict(type="str", default="world")))' \
'm.exit_json(changed=False, msg="hello " + m.params["name"])' > plugins/modules/hello.py
ansible-galaxy collection build --output-path /w/dist >/dev/null
`
// ansibleHelloPlay runs unkin.e2e_hello.hello from collections in /w/coll.
const ansibleHelloPlay = `export ANSIBLE_COLLECTIONS_PATH=/w/coll
ansible-doc unkin.e2e_hello.hello | grep -q 'Returns a greeting'
printf -- '- hosts: localhost\n gather_facts: false\n tasks:\n - unkin.e2e_hello.hello: {name: e2e}\n register: r\n - debug: {msg: "PLAY {{ r.msg }}"}\n' > /w/play.yml
ansible-playbook -i localhost, -c local /w/play.yml | grep -o 'PLAY hello e2e' | head -1
`
// TestLocalAnsiblePublishInstall builds a collection with a stock
// ansible-galaxy client, publishes it to a local ansible repo, rejects the
// republish, installs it back and runs its module.
func TestLocalAnsiblePublishInstall(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
name := fmt.Sprintf("e2e-ansible-local-%d", time.Now().UnixNano())
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","repo_type":"local"}`, name))
defer deleteRepo(t, name)
script := "set -euo pipefail\n" + ansibleHelloScript + `
ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused
if ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused; then echo "republish accepted"; exit 1; fi
ansible-galaxy collection install unkin.e2e_hello:==1.0.0 --server "$SRV" -p /w/coll
` + ansibleHelloPlay
srv := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
out, err := exec.Command("docker", "run", "--rm", "--network", network,
"-e", "SRV="+srv, "python:3.12-slim", "bash", "-c", script).CombinedOutput()
if err != nil {
t.Fatalf("ansible-galaxy client run failed: %v\n%s", err, out)
}
for _, want := range []string{"successfully published and imported", "HTTP Code: 409", "unkin.e2e_hello:1.0.0 was installed successfully", "PLAY hello e2e"} {
if !strings.Contains(string(out), want) {
t.Fatalf("missing %q in client output:\n%s", want, out)
}
}
}
+289
View File
@@ -0,0 +1,289 @@
package v1
import (
"bytes"
"cmp"
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"slices"
"strconv"
"strings"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var ansibleMaxPublishBytes int64 = 256 << 20
// ansibleError writes a Galaxy v3 error document, which ansible-galaxy prints.
func ansibleError(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{"errors": []map[string]string{{
"status": strconv.Itoa(status), "code": strings.ToLower(strings.ReplaceAll(http.StatusText(status), " ", "_")),
"title": http.StatusText(status), "detail": msg,
}}})
}
func writeAnsibleJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
w.WriteHeader(status)
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
_ = enc.Encode(v)
}
func isAnsiblePublish(path string) bool {
return strings.TrimSuffix(path, "/")+"/" == ansible.PublishPath
}
func (h *ProxyHandler) handleAnsiblePublish(w http.ResponseWriter, r *http.Request) {
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageAnsible {
ansibleError(w, http.StatusMethodNotAllowed, "publish is only supported on local ansible repositories")
return
}
base, ok := requestBaseURL(w, r)
if !ok {
return
}
r.Body = http.MaxBytesReader(w, r.Body, ansibleMaxPublishBytes)
file, sum, err := readAnsibleUpload(r)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
ansibleError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("upload exceeds %d bytes", ansibleMaxPublishBytes))
return
}
ansibleError(w, http.StatusBadRequest, err.Error())
return
}
digest := sha256.Sum256(file)
if got := hex.EncodeToString(digest[:]); sum != "" && !strings.EqualFold(sum, got) {
ansibleError(w, http.StatusBadRequest, "sha256 does not match the uploaded file")
return
}
e, err := ansible.ParseCollection(bytes.NewReader(file))
if err != nil {
ansibleError(w, http.StatusBadRequest, err.Error())
return
}
e.SHA256, e.Size, e.CreatedAt = hex.EncodeToString(digest[:]), int64(len(file)), time.Now().UTC()
id := e.Namespace + "." + e.Name + ":" + e.Version
unlock, err := h.engine.Lock(r.Context(), remote.Name, "_ansiblelock/"+e.TaskID(), gemsLockWait, 5*time.Minute)
if err != nil {
ansibleError(w, http.StatusServiceUnavailable, id+" is being published, retry")
return
}
defer unlock()
conflict := func() {
ansibleError(w, http.StatusConflict, id+" already exists; overwrites are not allowed")
}
if f, err := h.db.GetLocalFile(r.Context(), remote.Name, e.IndexPath()); err != nil {
ansibleError(w, http.StatusInternalServerError, err.Error())
return
} else if f != nil {
conflict()
return
}
// The version is not live, so a tarball left by a failed publish is safe to drop.
artifact := ansible.ArtifactPrefix + e.Filename()
fail := func(status int, msg string) {
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, artifact); err != nil {
slog.Error("ansible publish cleanup failed", "repo", remote.Name, "path", artifact, "error", err)
}
ansibleError(w, status, msg)
}
if ok, err := h.npmStoreImmutable(r.Context(), remote.Name, artifact, file, "application/gzip"); err != nil {
fail(http.StatusInternalServerError, err.Error())
return
} else if !ok {
fail(http.StatusConflict, id+" already exists; overwrites are not allowed")
return
}
entry, err := json.Marshal(e)
if err != nil {
fail(http.StatusInternalServerError, err.Error())
return
}
// The index entry gates visibility, so it is written last.
if _, err := h.npmStore(r.Context(), remote.Name, e.IndexPath(), entry, "application/json", false); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
conflict()
return
}
fail(http.StatusInternalServerError, err.Error())
return
}
slog.Info("local ansible collection published", "repo", remote.Name, "collection", id)
repoBase := base + "/api/v1/local/" + url.PathEscape(remote.Name)
writeAnsibleJSON(w, http.StatusAccepted, map[string]string{"task": repoBase + "/api/v3/imports/collections/" + e.TaskID() + "/"})
}
// readAnsibleUpload returns the multipart "file" part, which ansible-galaxy
// base64-encodes, and the optional "sha256" field sent with it.
func readAnsibleUpload(r *http.Request) ([]byte, string, error) {
mr, err := r.MultipartReader()
if err != nil {
return nil, "", errors.New("expected a multipart/form-data upload")
}
var file []byte
var sum string
for {
part, err := mr.NextPart()
if err == io.EOF {
break
}
if err != nil {
return nil, "", err
}
switch part.FormName() {
case "file":
if file != nil {
return nil, "", errors.New("multiple file parts")
}
var src io.Reader = part
switch enc := strings.ToLower(part.Header.Get("Content-Transfer-Encoding")); enc {
case "base64":
src = base64.NewDecoder(base64.StdEncoding, part)
case "", "binary", "7bit", "8bit":
default:
return nil, "", fmt.Errorf("unsupported Content-Transfer-Encoding %q", enc)
}
if file, err = io.ReadAll(src); err != nil {
return nil, "", err
}
case "sha256":
b, err := io.ReadAll(io.LimitReader(part, 128))
if err != nil {
return nil, "", err
}
sum = strings.TrimSpace(string(b))
}
}
if file == nil {
return nil, "", errors.New("missing file part")
}
return file, sum, nil
}
// serveLocalAnsible answers the Galaxy v3 API of a local ansible repo.
func (h *ProxyHandler) serveLocalAnsible(w http.ResponseWriter, r *http.Request, remote *models.Remote, ap ansible.APIPath) {
base, ok := requestBaseURL(w, r)
if !ok {
return
}
body, err := h.localAnsibleDoc(r.Context(), remote.Name, ap, r.URL.Query(), base+"/api/v1/local/"+url.PathEscape(remote.Name))
if errors.Is(err, virtual.ErrNotFound) {
ansibleError(w, http.StatusNotFound, "not found")
return
}
if err != nil {
ansibleError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("X-Artifact-Source", "local")
writeAnsibleJSON(w, http.StatusOK, body)
}
// LocalAnsible serves a local ansible repo's collection, full versions list
// or version detail to ansible virtuals, with root-relative URLs.
func (h *ProxyHandler) LocalAnsible(ctx context.Context, repo, path string) ([]byte, error) {
doc, err := h.localAnsibleDoc(ctx, repo, ansible.ParseAPIPath(path), nil, "/api/v1/local/"+url.PathEscape(repo))
if err != nil {
return nil, err
}
return json.Marshal(doc)
}
// localAnsibleDoc builds the document for ap; a nil q lists every version.
func (h *ProxyHandler) localAnsibleDoc(ctx context.Context, repo string, ap ansible.APIPath, q url.Values, base string) (any, error) {
switch ap.Kind {
case ansible.APIRoot:
return ansible.Root(), nil
case ansible.APIImport:
ns, rest, _ := strings.Cut(ap.TaskID, "-")
name, version, _ := strings.Cut(rest, "-")
entries, err := h.localAnsibleEntries(ctx, repo, ns, name)
if err != nil {
return nil, err
}
for _, e := range entries {
if e.Version == version {
return ansible.ImportTask(e), nil
}
}
case ansible.APICollection, ansible.APIVersions, ansible.APIVersion:
entries, err := h.localAnsibleEntries(ctx, repo, ap.Namespace, ap.Name)
if err != nil || len(entries) == 0 {
return nil, cmp.Or(err, virtual.ErrNotFound)
}
refs := make([]ansible.VersionRef, len(entries))
for i, e := range entries {
refs[i] = e.Ref()
}
switch ap.Kind {
case ansible.APICollection:
return ansible.CollectionDoc(base, ap.Namespace, ap.Name, refs), nil
case ansible.APIVersions:
limit, offset := len(refs), 0
if q != nil {
limit, offset = ansible.Page(q)
}
return ansible.VersionsPage(base, ap.Namespace, ap.Name, refs, limit, offset), nil
}
for _, e := range entries {
if e.Version == ap.Version {
return ansible.VersionDoc(base, e), nil
}
}
}
return nil, virtual.ErrNotFound
}
// localAnsibleEntries returns a collection's versions, newest publish first.
func (h *ProxyHandler) localAnsibleEntries(ctx context.Context, repo, ns, name string) ([]ansible.Entry, error) {
if !ansible.NameRe.MatchString(ns) || !ansible.NameRe.MatchString(name) {
return nil, nil
}
prefix := ansible.IndexPrefix + ns + "/" + name + "/"
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
if err != nil {
return nil, err
}
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(b.ID, a.ID) })
var out []ansible.Entry
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
if !strings.HasPrefix(f.FilePath, prefix) {
continue
}
raw, err := h.readLocalBlob(ctx, f.ContentHash)
if err != nil {
return nil, err
}
var e ansible.Entry
if err := json.Unmarshal(raw, &e); err != nil {
return nil, fmt.Errorf("decode %s: %w", f.FilePath, err)
}
out = append(out, e)
}
return out, nil
}
+212
View File
@@ -0,0 +1,212 @@
package v1
import (
"archive/tar"
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/json"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/textproto"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func testCollection(t *testing.T, ns, name, version string) []byte {
t.Helper()
manifest := `{"collection_info":{"namespace":"` + ns + `","name":"` + name + `","version":"` + version + `","dependencies":{"ansible.posix":">=1.0.0"}}}`
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
_ = tw.WriteHeader(&tar.Header{Name: "MANIFEST.json", Mode: 0o644, Size: int64(len(manifest))})
_, _ = tw.Write([]byte(manifest))
_ = tw.Close()
_ = zw.Close()
return buf.Bytes()
}
// ansiblePublish uploads like ansible-galaxy: the file part base64-encoded
// whenever a sha256 field is sent, raw otherwise.
func ansiblePublish(t *testing.T, url string, tarball []byte, sum string) (int, string) {
t.Helper()
var body bytes.Buffer
mw := multipart.NewWriter(&body)
hdr := textproto.MIMEHeader{"Content-Disposition": {`form-data; name="file"; filename="c.tar.gz"`}}
data := tarball
if sum != "" {
_ = mw.WriteField("sha256", sum)
hdr.Set("Content-Transfer-Encoding", "base64")
data = []byte(base64.StdEncoding.EncodeToString(tarball))
}
fw, _ := mw.CreatePart(hdr)
_, _ = fw.Write(data)
_ = mw.Close()
req, _ := http.NewRequest(http.MethodPost, url, &body)
req.Header.Set("Content-Type", mw.FormDataContentType())
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
var b bytes.Buffer
_, _ = b.ReadFrom(resp.Body)
return resp.StatusCode, b.String()
}
func getJSON(t *testing.T, url string, v any) {
t.Helper()
code, body := npmDo(t, http.MethodGet, url, "")
if code != http.StatusOK {
t.Fatalf("GET %s = %d: %s", url, code, body)
}
if err := json.Unmarshal([]byte(body), v); err != nil {
t.Fatalf("GET %s: %v: %s", url, err, body)
}
}
func TestAnsiblePublishAndServe(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageAnsible)
base := srv.URL + "/api/v1/local/" + repo + "/"
publish := base + "api/v3/artifacts/collections/"
v1 := testCollection(t, "unkin", "e2e_hello", "1.0.0")
code, body := ansiblePublish(t, publish, v1, sha256Hex(v1))
if code != http.StatusAccepted {
t.Fatalf("publish = %d: %s", code, body)
}
var task struct{ Task string }
_ = json.Unmarshal([]byte(body), &task)
if task.Task != base+"api/v3/imports/collections/unkin-e2e_hello-1.0.0/" {
t.Fatalf("task = %q", task.Task)
}
var st map[string]any
getJSON(t, task.Task, &st)
if st["state"] != "completed" || st["finished_at"] == "" {
t.Fatalf("task = %v", st)
}
if code, _ := npmDo(t, http.MethodGet, base+"api/v3/imports/collections/unkin-e2e_hello-9.9.9/", ""); code != http.StatusNotFound {
t.Errorf("unknown task = %d, want 404", code)
}
for _, v := range []string{"1.1.0", "2.0.0"} {
tb := testCollection(t, "unkin", "e2e_hello", v)
if code, body := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
t.Fatalf("publish %s = %d: %s", v, code, body)
}
}
if code, body := ansiblePublish(t, publish, v1, ""); code != http.StatusConflict || !strings.Contains(body, "already exists") {
t.Errorf("identical republish = %d: %s, want 409", code, body)
}
if code, _ := ansiblePublish(t, publish, testCollection(t, "unkin", "e2e_hello", "1.0.0")[:20], ""); code != http.StatusBadRequest {
t.Errorf("truncated tarball = %d, want 400", code)
}
var root map[string]map[string]string
getJSON(t, base+"api/", &root)
if root["available_versions"]["v3"] != "v3/" {
t.Fatalf("root = %v", root)
}
var page struct {
Meta struct{ Count int }
Links struct{ Next, Previous *string }
Data []struct{ Version, Href string }
}
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/versions/?limit=2", &page)
if page.Meta.Count != 3 || len(page.Data) != 2 || page.Data[0].Version != "2.0.0" || page.Links.Next == nil || page.Links.Previous != nil {
t.Fatalf("page 1 = %+v", page)
}
next := *page.Links.Next
page.Links.Next = nil
getJSON(t, next, &page)
if len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next != nil {
t.Fatalf("page 2 (%s) = %+v", next, page)
}
var detail struct {
DownloadURL string `json:"download_url"`
Artifact struct{ Sha256 string }
Metadata struct{ Dependencies map[string]string }
Namespace struct{ Name string }
Collection struct{ Name string }
}
getJSON(t, page.Data[0].Href, &detail)
if detail.DownloadURL != base+"artifacts/unkin-e2e_hello-1.0.0.tar.gz" || detail.Artifact.Sha256 != sha256Hex(v1) ||
detail.Metadata.Dependencies["ansible.posix"] != ">=1.0.0" || detail.Namespace.Name != "unkin" || detail.Collection.Name != "e2e_hello" {
t.Fatalf("detail = %+v", detail)
}
if code, got := npmDo(t, http.MethodGet, detail.DownloadURL, ""); code != http.StatusOK || got != string(v1) {
t.Fatalf("download = %d (%d bytes)", code, len(got))
}
var coll map[string]any
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/", &coll)
if coll["versions_url"] != base+"api/v3/collections/unkin/e2e_hello/versions/" || coll["updated_at"] == "" {
t.Fatalf("collection = %v", coll)
}
for _, p := range []string{"api/v3/collections/unkin/nope/", "api/v3/collections/unkin/e2e_hello/versions/9.0.0/", "_index/unkin/e2e_hello/1.0.0", "other"} {
if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound {
t.Errorf("GET %s = %d, want 404", p, code)
}
}
}
func TestAnsiblePublishRejects(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageAnsible)
publish := srv.URL + "/api/v1/local/" + repo + "/api/v3/artifacts/collections/"
tb := testCollection(t, "unkin", "rej", "1.0.0")
if code, _ := ansiblePublish(t, publish, tb, strings.Repeat("0", 64)); code != http.StatusBadRequest {
t.Errorf("sha256 mismatch = %d, want 400", code)
}
if code, _ := npmDo(t, http.MethodPost, publish, "raw"); code != http.StatusBadRequest {
t.Errorf("non-multipart = %d, want 400", code)
}
old := ansibleMaxPublishBytes
ansibleMaxPublishBytes = 64
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusRequestEntityTooLarge {
t.Errorf("oversize = %d, want 413", code)
}
ansibleMaxPublishBytes = old
generic := npmRepo(t, h, models.PackageGeneric)
if code, _ := ansiblePublish(t, srv.URL+"/api/v1/local/"+generic+"/api/v3/artifacts/collections/", tb, ""); code != http.StatusMethodNotAllowed {
t.Errorf("publish to generic local = %d, want 405", code)
}
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
t.Errorf("publish after rejects = %d, want 202", code)
}
}
func TestAnsibleLocalPerHost(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageAnsible)
if code, b := ansiblePublish(t, srv.URL+"/api/v1/local/"+repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "host", "1.0.0"), ""); code != http.StatusAccepted {
t.Fatalf("publish = %d: %s", code, b)
}
routes := h.Routes()
path := "/local/" + repo + "/api/v3/collections/unkin/host/versions/1.0.0/"
r := httptest.NewRequest(http.MethodGet, path, nil)
r.Host = "x@evil.com"
w := httptest.NewRecorder()
routes.ServeHTTP(w, r)
if w.Code != http.StatusBadRequest {
t.Errorf("bad Host = %d, want 400", w.Code)
}
r = httptest.NewRequest(http.MethodGet, path, nil)
r.Host = "aa.example"
r.Header.Set("X-Forwarded-Proto", "https")
w = httptest.NewRecorder()
routes.ServeHTTP(w, r)
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"download_url":"https://aa.example/api/v1/local/`+repo+`/artifacts/unkin-host-1.0.0.tar.gz"`) {
t.Fatalf("detail = %d: %s", w.Code, w.Body)
}
if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto" {
t.Errorf("Cache-Control %q Vary %q", cc, vary)
}
}
+4
View File
@@ -64,6 +64,10 @@ func (h *ProxyHandler) lockGemVersion(w http.ResponseWriter, r *http.Request, re
}
func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) {
if isAnsiblePublish(chi.URLParam(r, "*")) {
h.handleAnsiblePublish(w, r)
return
}
if chi.URLParam(r, "*") != "api/v1/gems" {
gemsText(w, http.StatusNotFound, "not found")
return
+12
View File
@@ -17,6 +17,7 @@ import (
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/virtual"
@@ -275,6 +276,17 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
}
}
if remote.PackageType == models.PackageAnsible {
if ap := ansible.ParseAPIPath(path); ap.Kind != ansible.APIUnknown {
h.serveLocalAnsible(w, r, remote, ap)
return
}
if !strings.HasPrefix(path, ansible.ArtifactPrefix) {
http.Error(w, "not found", http.StatusNotFound)
return
}
}
prov, _ := provider.Get(remote.PackageType)
if indexer, ok := prov.(provider.LocalIndexer); ok {
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
+328
View File
@@ -0,0 +1,328 @@
package ansible
import (
"archive/tar"
"compress/gzip"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"path"
"regexp"
"strconv"
"strings"
"time"
)
// Local ansible repos speak the Galaxy v3 publish and install API. Per version:
// artifacts/<ns>-<name>-<version>.tar.gz collection tarball (immutable)
// _index/<ns>/<name>/<version> Entry JSON; gates visibility
const (
IndexPrefix = "_index/"
ArtifactPrefix = "artifacts/"
PublishPath = "api/v3/artifacts/collections/"
)
var (
NameRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
VersionRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`)
)
var (
MaxManifestBytes int64 = 1 << 20
MaxScanBytes int64 = 512 << 20
)
// Entry is one published collection version.
type Entry struct {
Namespace string `json:"namespace"`
Name string `json:"name"`
Version string `json:"version"`
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
Metadata json.RawMessage `json:"metadata"`
CreatedAt time.Time `json:"created_at"`
}
func (e Entry) Filename() string { return e.Namespace + "-" + e.Name + "-" + e.Version + ".tar.gz" }
// IndexPath is the entry's hidden index file.
func (e Entry) IndexPath() string { return IndexPrefix + e.Namespace + "/" + e.Name + "/" + e.Version }
// TaskID names the import task returned on publish.
func (e Entry) TaskID() string { return e.Namespace + "-" + e.Name + "-" + e.Version }
// ParseCollection reads collection_info from a collection tarball's top-level
// MANIFEST.json. Every member is checked, so a tarball whose paths or links
// escape the collection root is rejected even after the manifest.
func ParseCollection(r io.Reader) (*Entry, error) {
zr, err := gzip.NewReader(r)
if err != nil {
return nil, fmt.Errorf("not a gzip tarball: %w", err)
}
scan := &io.LimitedReader{R: zr, N: MaxScanBytes}
tr := tar.NewReader(scan)
var e *Entry
for {
hdr, err := tr.Next()
if scan.N <= 0 {
return nil, errors.New("collection exceeds decompressed size limit")
}
if err == io.EOF {
break
}
if err != nil {
return nil, fmt.Errorf("read tarball: %w", err)
}
name, err := memberPath(hdr)
if err != nil {
return nil, err
}
if name != "MANIFEST.json" || hdr.Typeflag != tar.TypeReg {
continue
}
if e != nil {
return nil, errors.New("duplicate MANIFEST.json")
}
if e, err = parseManifest(io.LimitReader(tr, MaxManifestBytes+1)); err != nil {
return nil, err
}
}
if e == nil {
return nil, errors.New("no MANIFEST.json in collection tarball")
}
return e, nil
}
// memberPath returns hdr's cleaned name, rejecting absolute or escaping
// paths and links, and anything but files, directories and symlinks.
func memberPath(hdr *tar.Header) (string, error) {
name := path.Clean(strings.TrimPrefix(hdr.Name, "./"))
if escapes(name) {
return "", fmt.Errorf("tarball member %q escapes the collection root", hdr.Name)
}
switch hdr.Typeflag {
case tar.TypeReg, tar.TypeDir:
case tar.TypeSymlink:
if path.IsAbs(hdr.Linkname) || escapes(path.Join(path.Dir(name), hdr.Linkname)) {
return "", fmt.Errorf("tarball symlink %q escapes the collection root", hdr.Name)
}
default:
return "", fmt.Errorf("tarball member %q has unsupported type %q", hdr.Name, hdr.Typeflag)
}
return name, nil
}
func escapes(p string) bool {
return path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") || strings.Contains(p, "\\")
}
func parseManifest(r io.Reader) (*Entry, error) {
raw, err := io.ReadAll(r)
if err != nil {
return nil, fmt.Errorf("read MANIFEST.json: %w", err)
}
if int64(len(raw)) > MaxManifestBytes {
return nil, fmt.Errorf("MANIFEST.json exceeds %d bytes", MaxManifestBytes)
}
var doc struct {
CollectionInfo map[string]json.RawMessage `json:"collection_info"`
}
if err := json.Unmarshal(raw, &doc); err != nil || doc.CollectionInfo == nil {
return nil, errors.New("MANIFEST.json has no collection_info")
}
info := doc.CollectionInfo
e := &Entry{}
for k, dst := range map[string]*string{"namespace": &e.Namespace, "name": &e.Name, "version": &e.Version} {
if err := json.Unmarshal(info[k], dst); err != nil {
return nil, fmt.Errorf("collection_info.%s is not a string", k)
}
}
if !NameRe.MatchString(e.Namespace) || !NameRe.MatchString(e.Name) || !VersionRe.MatchString(e.Version) {
return nil, fmt.Errorf("invalid collection %q.%q version %q", e.Namespace, e.Name, e.Version)
}
deps := map[string]string{}
if d := info["dependencies"]; len(d) > 0 && string(d) != "null" {
if err := json.Unmarshal(d, &deps); err != nil {
return nil, errors.New("collection_info.dependencies must map collection names to version ranges")
}
}
for k := range deps {
ns, n, ok := strings.Cut(k, ".")
if !ok || !NameRe.MatchString(ns) || !NameRe.MatchString(n) {
return nil, fmt.Errorf("invalid dependency %q", k)
}
}
info["dependencies"], _ = json.Marshal(deps)
if e.Metadata, err = json.Marshal(info); err != nil {
return nil, err
}
return e, nil
}
// APIPath is a parsed Galaxy v3 request path below a repo.
type APIPath struct {
Kind APIKind
Namespace, Name, Version string
TaskID string
}
type APIKind int
const (
APIUnknown APIKind = iota
APIRoot
APICollection
APIVersions
APIVersion
APIImport
)
// ParseAPIPath parses api/, api/v3/collections/<ns>/<name>/[versions/[<v>/]]
// and api/v3/imports/collections/<id>/, with or without trailing slashes.
func ParseAPIPath(p string) APIPath {
p = strings.TrimSuffix(p, "/")
if p == "api" {
return APIPath{Kind: APIRoot}
}
if id, ok := strings.CutPrefix(p, "api/v3/imports/collections/"); ok && id != "" && !strings.Contains(id, "/") {
return APIPath{Kind: APIImport, TaskID: id}
}
rest, ok := strings.CutPrefix(p, "api/v3/collections/")
if !ok {
return APIPath{}
}
segs := strings.Split(rest, "/")
if len(segs) < 2 || !NameRe.MatchString(segs[0]) || !NameRe.MatchString(segs[1]) {
return APIPath{}
}
ap := APIPath{Namespace: segs[0], Name: segs[1]}
switch {
case len(segs) == 2:
ap.Kind = APICollection
case len(segs) == 3 && segs[2] == "versions":
ap.Kind = APIVersions
case len(segs) == 4 && segs[2] == "versions" && VersionRe.MatchString(segs[3]):
ap.Kind, ap.Version = APIVersion, segs[3]
default:
return APIPath{}
}
return ap
}
// Root is the API root ansible-galaxy discovers.
func Root() map[string]any {
return map[string]any{"available_versions": map[string]string{"v3": "v3/"}}
}
func CollectionPath(ns, name string) string { return "/api/v3/collections/" + ns + "/" + name + "/" }
func VersionPath(ns, name, version string) string {
return CollectionPath(ns, name) + "versions/" + url.PathEscape(version) + "/"
}
// VersionRef is one entry of a versions list.
type VersionRef struct {
Version string `json:"version"`
CreatedAt string `json:"created_at,omitempty"`
UpdatedAt string `json:"updated_at,omitempty"`
}
func (e Entry) Ref() VersionRef {
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
return VersionRef{Version: e.Version, CreatedAt: t, UpdatedAt: t}
}
// CollectionDoc describes a collection from its versions; base is the repo
// URL the hrefs are built on.
func CollectionDoc(base, ns, name string, refs []VersionRef) map[string]any {
created, updated := "", ""
for _, r := range refs {
if created == "" || r.CreatedAt < created {
created = r.CreatedAt
}
updated = max(updated, r.UpdatedAt)
}
return map[string]any{
"href": base + CollectionPath(ns, name),
"namespace": ns,
"name": name,
"deprecated": false,
"versions_url": base + CollectionPath(ns, name) + "versions/",
"created_at": created,
"updated_at": updated,
}
}
// Page reads limit/offset as the remote forwards them, defaulting to 100/0.
func Page(q url.Values) (limit, offset int) {
limit, offset = maxPageLimit, 0
if v, ok := pageLimit(q.Get("limit")); ok {
limit, _ = strconv.Atoi(v)
}
if v, ok := pageOffset(q.Get("offset")); ok {
offset, _ = strconv.Atoi(v)
}
return limit, offset
}
// VersionsPage is one page of refs in Galaxy v3 list form.
func VersionsPage(base, ns, name string, refs []VersionRef, limit, offset int) map[string]any {
list := base + CollectionPath(ns, name) + "versions/"
link := func(off int) any {
return list + "?" + url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}.Encode()
}
offset = min(offset, len(refs))
var previous, next any
if offset > 0 {
previous = link(max(offset-limit, 0))
}
if offset+limit < len(refs) {
next = link(offset + limit)
}
data := []map[string]any{}
for _, r := range refs[offset:min(offset+limit, len(refs))] {
data = append(data, map[string]any{
"version": r.Version,
"href": base + VersionPath(ns, name, r.Version),
"created_at": r.CreatedAt,
"updated_at": r.UpdatedAt,
"marks": []string{},
})
}
return map[string]any{
"meta": map[string]any{"count": len(refs)},
"links": map[string]any{"first": link(0), "previous": previous, "next": next, "last": link(max(len(refs)-limit, 0))},
"data": data,
}
}
// VersionDoc is a version's detail; the tarball is served from the repo at
// artifacts/<filename>.
func VersionDoc(base string, e Entry) map[string]any {
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
return map[string]any{
"href": base + VersionPath(e.Namespace, e.Name, e.Version),
"namespace": map[string]string{"name": e.Namespace},
"collection": map[string]string{"name": e.Name},
"name": e.Name,
"version": e.Version,
"download_url": base + "/" + ArtifactPrefix + e.Filename(),
"artifact": map[string]any{"filename": e.Filename(), "sha256": e.SHA256, "size": e.Size},
"metadata": e.Metadata,
"signatures": []string{},
"created_at": t,
"updated_at": t,
}
}
// ImportTask is a finished import: publishing completes synchronously.
func ImportTask(e Entry) map[string]any {
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
return map[string]any{
"id": e.TaskID(), "state": "completed", "created_at": t, "started_at": t, "finished_at": t,
"messages": []any{}, "error": nil,
}
}
+196
View File
@@ -0,0 +1,196 @@
package ansible
import (
"archive/tar"
"bytes"
"compress/gzip"
"encoding/json"
"net/url"
"strings"
"testing"
)
type member struct {
hdr tar.Header
body string
}
func collectionTar(t *testing.T, members ...member) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for _, m := range members {
h := m.hdr
if h.Typeflag == 0 {
h.Typeflag = tar.TypeReg
}
if h.Typeflag == tar.TypeReg {
h.Size = int64(len(m.body))
}
h.Mode = 0o644
if err := tw.WriteHeader(&h); err != nil {
t.Fatal(err)
}
_, _ = tw.Write([]byte(m.body))
}
_ = tw.Close()
_ = zw.Close()
return buf.Bytes()
}
func manifest(ns, name, version, deps string) member {
return member{tar.Header{Name: "MANIFEST.json"}, `{"collection_info":{"namespace":"` + ns + `","name":"` + name +
`","version":"` + version + `","authors":["a"],"dependencies":` + deps + `}}`}
}
func TestParseCollection(t *testing.T) {
e, err := ParseCollection(bytes.NewReader(collectionTar(t,
member{tar.Header{Name: "plugins/", Typeflag: tar.TypeDir}, ""},
member{tar.Header{Name: "plugins/modules/hello.py"}, "x"},
member{tar.Header{Name: "docs/link", Typeflag: tar.TypeSymlink, Linkname: "../plugins/modules/hello.py"}, ""},
manifest("unkin", "e2e_hello", "1.0.0-rc.1+b1", `{"ansible.posix":">=1.0.0"}`),
)))
if err != nil {
t.Fatal(err)
}
if e.Namespace != "unkin" || e.Name != "e2e_hello" || e.Version != "1.0.0-rc.1+b1" || e.Filename() != "unkin-e2e_hello-1.0.0-rc.1+b1.tar.gz" {
t.Fatalf("entry = %+v", e)
}
var md struct {
Authors []string `json:"authors"`
Dependencies map[string]string `json:"dependencies"`
}
if err := json.Unmarshal(e.Metadata, &md); err != nil || md.Dependencies["ansible.posix"] != ">=1.0.0" || len(md.Authors) != 1 {
t.Fatalf("metadata = %s (%v)", e.Metadata, err)
}
e, err = ParseCollection(bytes.NewReader(collectionTar(t, manifest("a", "b", "1.0.0", "null"))))
if err != nil || !strings.Contains(string(e.Metadata), `"dependencies":{}`) {
t.Fatalf("null dependencies = %s (%v), want {}", e.Metadata, err)
}
}
func TestParseCollectionRejects(t *testing.T) {
ok := manifest("a", "b", "1.0.0", "{}")
tests := []struct {
name string
members []member
want string
}{
{"no manifest", []member{{tar.Header{Name: "README.md"}, "x"}}, "no MANIFEST.json"},
{"nested manifest only", []member{{tar.Header{Name: "x/MANIFEST.json"}, ok.body}}, "no MANIFEST.json"},
{"dotdot", []member{ok, {tar.Header{Name: "../evil"}, "x"}}, "escapes"},
{"dotdot after clean", []member{ok, {tar.Header{Name: "a/../../evil"}, "x"}}, "escapes"},
{"absolute", []member{ok, {tar.Header{Name: "/etc/passwd"}, "x"}}, "escapes"},
{"symlink escape", []member{ok, {tar.Header{Name: "a/l", Typeflag: tar.TypeSymlink, Linkname: "../../x"}, ""}}, "symlink"},
{"absolute symlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeSymlink, Linkname: "/etc"}, ""}}, "symlink"},
{"hardlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeLink, Linkname: "MANIFEST.json"}, ""}}, "unsupported type"},
{"duplicate manifest", []member{ok, ok}, "duplicate"},
{"bad namespace", []member{manifest("Bad", "b", "1.0.0", "{}")}, "invalid collection"},
{"bad version", []member{manifest("a", "b", "1.0", "{}")}, "invalid collection"},
{"bad dependency", []member{manifest("a", "b", "1.0.0", `{"nodot":"*"}`)}, "invalid dependency"},
{"list dependencies", []member{manifest("a", "b", "1.0.0", `["x.y"]`)}, "dependencies must map"},
{"huge manifest", []member{{tar.Header{Name: "MANIFEST.json"}, strings.Repeat(" ", int(MaxManifestBytes)+1)}}, "exceeds"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := ParseCollection(bytes.NewReader(collectionTar(t, tt.members...)))
if err == nil || !strings.Contains(err.Error(), tt.want) {
t.Fatalf("err = %v, want %q", err, tt.want)
}
})
}
if _, err := ParseCollection(strings.NewReader("not gzip")); err == nil {
t.Error("non-gzip input accepted")
}
}
func TestParseCollectionScanLimit(t *testing.T) {
old := MaxScanBytes
MaxScanBytes = 4096
defer func() { MaxScanBytes = old }()
_, err := ParseCollection(bytes.NewReader(collectionTar(t,
member{tar.Header{Name: "big"}, strings.Repeat("x", 8192)},
manifest("a", "b", "1.0.0", "{}"),
)))
if err == nil || !strings.Contains(err.Error(), "decompressed size limit") {
t.Fatalf("err = %v, want decompressed size limit", err)
}
}
func TestParseAPIPath(t *testing.T) {
tests := []struct {
in string
want APIPath
}{
{"api", APIPath{Kind: APIRoot}},
{"api/", APIPath{Kind: APIRoot}},
{"api/v3/collections/unkin/e2e_hello/", APIPath{Kind: APICollection, Namespace: "unkin", Name: "e2e_hello"}},
{"api/v3/collections/unkin/e2e_hello/versions", APIPath{Kind: APIVersions, Namespace: "unkin", Name: "e2e_hello"}},
{"api/v3/collections/unkin/e2e_hello/versions/1.0.0/", APIPath{Kind: APIVersion, Namespace: "unkin", Name: "e2e_hello", Version: "1.0.0"}},
{"api/v3/imports/collections/unkin-e2e_hello-1.0.0/", APIPath{Kind: APIImport, TaskID: "unkin-e2e_hello-1.0.0"}},
{"api/v3/collections/unkin/e2e_hello/versions/1.0/", APIPath{}},
{"api/v3/collections/../x/", APIPath{}},
{"api/v3/collections/unkin/e2e_hello/versions/1.0.0/extra/", APIPath{}},
{"artifacts/unkin-e2e_hello-1.0.0.tar.gz", APIPath{}},
{"api/v3/imports/collections/a/b/", APIPath{}},
}
for _, tt := range tests {
if got := ParseAPIPath(tt.in); got != tt.want {
t.Errorf("ParseAPIPath(%q) = %+v, want %+v", tt.in, got, tt.want)
}
}
}
func TestVersionsPage(t *testing.T) {
refs := []VersionRef{{Version: "3.0.0"}, {Version: "2.0.0"}, {Version: "1.0.0"}}
list := "https://h/r/api/v3/collections/a/b/versions/"
links := func(p map[string]any) map[string]any { return p["links"].(map[string]any) }
versions := func(p map[string]any) (out []string) {
for _, d := range p["data"].([]map[string]any) {
out = append(out, d["version"].(string))
}
return out
}
p := VersionsPage("https://h/r", "a", "b", refs, 2, 0)
if got := versions(p); strings.Join(got, ",") != "3.0.0,2.0.0" {
t.Fatalf("page 1 = %v", got)
}
if l := links(p); l["previous"] != nil || l["next"] != list+"?limit=2&offset=2" || l["last"] != list+"?limit=2&offset=1" {
t.Fatalf("page 1 links = %v", l)
}
if p["meta"].(map[string]any)["count"] != 3 {
t.Fatalf("count = %v", p["meta"])
}
p = VersionsPage("https://h/r", "a", "b", refs, 2, 2)
if got := versions(p); strings.Join(got, ",") != "1.0.0" {
t.Fatalf("page 2 = %v", got)
}
if l := links(p); l["next"] != nil || l["previous"] != list+"?limit=2&offset=0" {
t.Fatalf("page 2 links = %v", l)
}
if got := versions(VersionsPage("", "a", "b", refs, 2, 99)); len(got) != 0 {
t.Fatalf("offset past end = %v, want empty", got)
}
}
func TestPage(t *testing.T) {
for _, tt := range []struct {
q string
limit, offset int
}{
{"", 100, 0},
{"limit=5&offset=10", 5, 10},
{"limit=5000", 100, 0},
{"limit=0&offset=-1", 100, 0},
} {
q, _ := url.ParseQuery(tt.q)
if l, o := Page(q); l != tt.limit || o != tt.offset {
t.Errorf("Page(%q) = %d,%d want %d,%d", tt.q, l, o, tt.limit, tt.offset)
}
}
}
+13 -1
View File
@@ -2,7 +2,19 @@ import type { PackageTypeUI } from '../types';
const def: PackageTypeUI = {
color: 'red',
snippets: ({ proxy }) => {
snippets: ({ name, url, proxy, isLocal }) => {
if (isLocal) {
const src = `${url}/api/v1/local/${name}/`;
return [
{
title: 'Publish and install',
language: 'bash',
code: `ansible-galaxy collection publish <ns>-<name>-<version>.tar.gz --server ${src} --api-key unused
ansible-galaxy collection install <ns>.<name> --server ${src}`,
note: 'publish needs an API key; any value works. Publishing an existing version is rejected.',
},
];
}
return [
{
title: 'Install collections through this Galaxy proxy',