Add local ansible Galaxy repositories (#157)
Internal Ansible collections can only be installed from galaxy.ansible.com through the remote; there is nowhere to publish them for ansible-galaxy. - add the Galaxy v3 publish endpoint (multipart, base64 parts) with an import task that is already completed - read collection_info from MANIFEST.json under size and decompression caps, rejecting escaping paths and links - serve collection, paginated versions (root-relative links, as Galaxy does), version detail and artifact downloads - reject an existing namespace.name:version with 409 - add docs/ansible/local.md, a UI snippet and an e2e-docker case driving real ansible-galaxy requires unkin/artifactapi#137 Reviewed-on: #157 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #157.
This commit is contained in:
@@ -27,7 +27,7 @@ Classification precedence, TTLs and eviction for every remote: [docs/common/cach
|
||||
Each type's paths, Terraform examples and local/virtual behaviour are in `docs/<type>/`:
|
||||
|
||||
- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md)
|
||||
- `ansible`: [remote](docs/ansible/remote.md)
|
||||
- `ansible`: [remote](docs/ansible/remote.md), [local](docs/ansible/local.md)
|
||||
- `cargo`: [remote](docs/cargo/remote.md)
|
||||
- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md)
|
||||
- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md)
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
# Local ansible repository
|
||||
|
||||
A local `ansible` repo is a Galaxy v3 server: `ansible-galaxy collection publish`
|
||||
and `ansible-galaxy collection install` work against `/api/v1/local/<repo>`.
|
||||
`publish` needs an API key; any value works.
|
||||
|
||||
```bash
|
||||
ansible-galaxy collection build
|
||||
ansible-galaxy collection publish unkin-hello-1.0.0.tar.gz \
|
||||
--server https://artifactapi.example.com/api/v1/local/<repo>/ --api-key unused
|
||||
ansible-galaxy collection install unkin.hello \
|
||||
--server https://artifactapi.example.com/api/v1/local/<repo>/
|
||||
```
|
||||
|
||||
Publishing reads `collection_info` from the tarball's `MANIFEST.json` and
|
||||
finishes the import synchronously, so the import task is already `completed`.
|
||||
Uploads over 256 MiB, tarballs with members or symlinks escaping the collection root, or
|
||||
with an invalid name, version or dependency map are rejected with 400.
|
||||
|
||||
The repo serves `api/v3/collections/<ns>/<name>/`, the paginated
|
||||
`versions/` list (`limit` up to 100, `offset`), `versions/<version>/` with
|
||||
`download_url`, `artifact.sha256` and `metadata.dependencies`, and
|
||||
`artifacts/<ns>-<name>-<version>.tar.gz`.
|
||||
|
||||
Versions are immutable: publishing an existing `namespace.name:version` is
|
||||
rejected with 409.
|
||||
@@ -31,7 +31,8 @@ already-running stack.
|
||||
`node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`,
|
||||
second version, republish rejected), gems (stock `ruby:3.3`: `gem push`,
|
||||
`gem install`, `bundle install` + `require`, republish rejected, `gem yank`,
|
||||
yanked re-push rejected).
|
||||
yanked re-push rejected), ansible (stock `ansible-galaxy`: `collection build`,
|
||||
`publish`, republish rejected, `install` + module run).
|
||||
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
|
||||
across two members; npm packument merge over two locals via a stock `node:22`
|
||||
client (merged versions, install through the member redirect); gems compact
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
//go:build dockere2e
|
||||
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ansibleHelloScript builds unkin.e2e_hello 1.0.0 with one module into /w/dist.
|
||||
const ansibleHelloScript = `pip install -q ansible-core >/dev/null 2>&1
|
||||
mkdir -p /w && cd /w && ansible-galaxy collection init unkin.e2e_hello >/dev/null
|
||||
cd unkin/e2e_hello && sed -i 's/^version: .*/version: 1.0.0/' galaxy.yml && mkdir -p plugins/modules
|
||||
printf '%s\n' 'DOCUMENTATION = r"""' 'module: hello' 'short_description: e2e greeting' 'description: Returns a greeting.' 'options:' ' name: {type: str, default: world, description: Who to greet.}' '"""' \
|
||||
'from ansible.module_utils.basic import AnsibleModule' \
|
||||
'm = AnsibleModule(argument_spec=dict(name=dict(type="str", default="world")))' \
|
||||
'm.exit_json(changed=False, msg="hello " + m.params["name"])' > plugins/modules/hello.py
|
||||
ansible-galaxy collection build --output-path /w/dist >/dev/null
|
||||
`
|
||||
|
||||
// ansibleHelloPlay runs unkin.e2e_hello.hello from collections in /w/coll.
|
||||
const ansibleHelloPlay = `export ANSIBLE_COLLECTIONS_PATH=/w/coll
|
||||
ansible-doc unkin.e2e_hello.hello | grep -q 'Returns a greeting'
|
||||
printf -- '- hosts: localhost\n gather_facts: false\n tasks:\n - unkin.e2e_hello.hello: {name: e2e}\n register: r\n - debug: {msg: "PLAY {{ r.msg }}"}\n' > /w/play.yml
|
||||
ansible-playbook -i localhost, -c local /w/play.yml | grep -o 'PLAY hello e2e' | head -1
|
||||
`
|
||||
|
||||
// TestLocalAnsiblePublishInstall builds a collection with a stock
|
||||
// ansible-galaxy client, publishes it to a local ansible repo, rejects the
|
||||
// republish, installs it back and runs its module.
|
||||
func TestLocalAnsiblePublishInstall(t *testing.T) {
|
||||
network := os.Getenv("COMPOSE_NETWORK")
|
||||
internal := os.Getenv("ARTIFACTAPI_INTERNAL")
|
||||
if network == "" || internal == "" {
|
||||
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
|
||||
}
|
||||
if _, err := exec.LookPath("docker"); err != nil {
|
||||
t.Skip("docker not available on the test host")
|
||||
}
|
||||
|
||||
name := fmt.Sprintf("e2e-ansible-local-%d", time.Now().UnixNano())
|
||||
createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"ansible","repo_type":"local"}`, name))
|
||||
defer deleteRepo(t, name)
|
||||
|
||||
script := "set -euo pipefail\n" + ansibleHelloScript + `
|
||||
ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused
|
||||
if ansible-galaxy collection publish /w/dist/unkin-e2e_hello-1.0.0.tar.gz --server "$SRV" --api-key unused; then echo "republish accepted"; exit 1; fi
|
||||
ansible-galaxy collection install unkin.e2e_hello:==1.0.0 --server "$SRV" -p /w/coll
|
||||
` + ansibleHelloPlay
|
||||
|
||||
srv := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/"
|
||||
out, err := exec.Command("docker", "run", "--rm", "--network", network,
|
||||
"-e", "SRV="+srv, "python:3.12-slim", "bash", "-c", script).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("ansible-galaxy client run failed: %v\n%s", err, out)
|
||||
}
|
||||
for _, want := range []string{"successfully published and imported", "HTTP Code: 409", "unkin.e2e_hello:1.0.0 was installed successfully", "PLAY hello e2e"} {
|
||||
if !strings.Contains(string(out), want) {
|
||||
t.Fatalf("missing %q in client output:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var ansibleMaxPublishBytes int64 = 256 << 20
|
||||
|
||||
// ansibleError writes a Galaxy v3 error document, which ansible-galaxy prints.
|
||||
func ansibleError(w http.ResponseWriter, status int, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"errors": []map[string]string{{
|
||||
"status": strconv.Itoa(status), "code": strings.ToLower(strings.ReplaceAll(http.StatusText(status), " ", "_")),
|
||||
"title": http.StatusText(status), "detail": msg,
|
||||
}}})
|
||||
}
|
||||
|
||||
func writeAnsibleJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
|
||||
w.WriteHeader(status)
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetEscapeHTML(false)
|
||||
_ = enc.Encode(v)
|
||||
}
|
||||
|
||||
func isAnsiblePublish(path string) bool {
|
||||
return strings.TrimSuffix(path, "/")+"/" == ansible.PublishPath
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleAnsiblePublish(w http.ResponseWriter, r *http.Request) {
|
||||
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
|
||||
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageAnsible {
|
||||
ansibleError(w, http.StatusMethodNotAllowed, "publish is only supported on local ansible repositories")
|
||||
return
|
||||
}
|
||||
base, ok := requestBaseURL(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, ansibleMaxPublishBytes)
|
||||
file, sum, err := readAnsibleUpload(r)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
ansibleError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("upload exceeds %d bytes", ansibleMaxPublishBytes))
|
||||
return
|
||||
}
|
||||
ansibleError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
digest := sha256.Sum256(file)
|
||||
if got := hex.EncodeToString(digest[:]); sum != "" && !strings.EqualFold(sum, got) {
|
||||
ansibleError(w, http.StatusBadRequest, "sha256 does not match the uploaded file")
|
||||
return
|
||||
}
|
||||
e, err := ansible.ParseCollection(bytes.NewReader(file))
|
||||
if err != nil {
|
||||
ansibleError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
e.SHA256, e.Size, e.CreatedAt = hex.EncodeToString(digest[:]), int64(len(file)), time.Now().UTC()
|
||||
|
||||
id := e.Namespace + "." + e.Name + ":" + e.Version
|
||||
unlock, err := h.engine.Lock(r.Context(), remote.Name, "_ansiblelock/"+e.TaskID(), gemsLockWait, 5*time.Minute)
|
||||
if err != nil {
|
||||
ansibleError(w, http.StatusServiceUnavailable, id+" is being published, retry")
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
conflict := func() {
|
||||
ansibleError(w, http.StatusConflict, id+" already exists; overwrites are not allowed")
|
||||
}
|
||||
if f, err := h.db.GetLocalFile(r.Context(), remote.Name, e.IndexPath()); err != nil {
|
||||
ansibleError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
} else if f != nil {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
// The version is not live, so a tarball left by a failed publish is safe to drop.
|
||||
artifact := ansible.ArtifactPrefix + e.Filename()
|
||||
fail := func(status int, msg string) {
|
||||
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, artifact); err != nil {
|
||||
slog.Error("ansible publish cleanup failed", "repo", remote.Name, "path", artifact, "error", err)
|
||||
}
|
||||
ansibleError(w, status, msg)
|
||||
}
|
||||
if ok, err := h.npmStoreImmutable(r.Context(), remote.Name, artifact, file, "application/gzip"); err != nil {
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
} else if !ok {
|
||||
fail(http.StatusConflict, id+" already exists; overwrites are not allowed")
|
||||
return
|
||||
}
|
||||
entry, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
// The index entry gates visibility, so it is written last.
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, e.IndexPath(), entry, "application/json", false); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
slog.Info("local ansible collection published", "repo", remote.Name, "collection", id)
|
||||
repoBase := base + "/api/v1/local/" + url.PathEscape(remote.Name)
|
||||
writeAnsibleJSON(w, http.StatusAccepted, map[string]string{"task": repoBase + "/api/v3/imports/collections/" + e.TaskID() + "/"})
|
||||
}
|
||||
|
||||
// readAnsibleUpload returns the multipart "file" part, which ansible-galaxy
|
||||
// base64-encodes, and the optional "sha256" field sent with it.
|
||||
func readAnsibleUpload(r *http.Request) ([]byte, string, error) {
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
return nil, "", errors.New("expected a multipart/form-data upload")
|
||||
}
|
||||
var file []byte
|
||||
var sum string
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
switch part.FormName() {
|
||||
case "file":
|
||||
if file != nil {
|
||||
return nil, "", errors.New("multiple file parts")
|
||||
}
|
||||
var src io.Reader = part
|
||||
switch enc := strings.ToLower(part.Header.Get("Content-Transfer-Encoding")); enc {
|
||||
case "base64":
|
||||
src = base64.NewDecoder(base64.StdEncoding, part)
|
||||
case "", "binary", "7bit", "8bit":
|
||||
default:
|
||||
return nil, "", fmt.Errorf("unsupported Content-Transfer-Encoding %q", enc)
|
||||
}
|
||||
if file, err = io.ReadAll(src); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
case "sha256":
|
||||
b, err := io.ReadAll(io.LimitReader(part, 128))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sum = strings.TrimSpace(string(b))
|
||||
}
|
||||
}
|
||||
if file == nil {
|
||||
return nil, "", errors.New("missing file part")
|
||||
}
|
||||
return file, sum, nil
|
||||
}
|
||||
|
||||
// serveLocalAnsible answers the Galaxy v3 API of a local ansible repo.
|
||||
func (h *ProxyHandler) serveLocalAnsible(w http.ResponseWriter, r *http.Request, remote *models.Remote, ap ansible.APIPath) {
|
||||
base, ok := requestBaseURL(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
body, err := h.localAnsibleDoc(r.Context(), remote.Name, ap, r.URL.Query(), base+"/api/v1/local/"+url.PathEscape(remote.Name))
|
||||
if errors.Is(err, virtual.ErrNotFound) {
|
||||
ansibleError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
ansibleError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
writeAnsibleJSON(w, http.StatusOK, body)
|
||||
}
|
||||
|
||||
// LocalAnsible serves a local ansible repo's collection, full versions list
|
||||
// or version detail to ansible virtuals, with root-relative URLs.
|
||||
func (h *ProxyHandler) LocalAnsible(ctx context.Context, repo, path string) ([]byte, error) {
|
||||
doc, err := h.localAnsibleDoc(ctx, repo, ansible.ParseAPIPath(path), nil, "/api/v1/local/"+url.PathEscape(repo))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(doc)
|
||||
}
|
||||
|
||||
// localAnsibleDoc builds the document for ap; a nil q lists every version.
|
||||
func (h *ProxyHandler) localAnsibleDoc(ctx context.Context, repo string, ap ansible.APIPath, q url.Values, base string) (any, error) {
|
||||
switch ap.Kind {
|
||||
case ansible.APIRoot:
|
||||
return ansible.Root(), nil
|
||||
case ansible.APIImport:
|
||||
ns, rest, _ := strings.Cut(ap.TaskID, "-")
|
||||
name, version, _ := strings.Cut(rest, "-")
|
||||
entries, err := h.localAnsibleEntries(ctx, repo, ns, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Version == version {
|
||||
return ansible.ImportTask(e), nil
|
||||
}
|
||||
}
|
||||
case ansible.APICollection, ansible.APIVersions, ansible.APIVersion:
|
||||
entries, err := h.localAnsibleEntries(ctx, repo, ap.Namespace, ap.Name)
|
||||
if err != nil || len(entries) == 0 {
|
||||
return nil, cmp.Or(err, virtual.ErrNotFound)
|
||||
}
|
||||
refs := make([]ansible.VersionRef, len(entries))
|
||||
for i, e := range entries {
|
||||
refs[i] = e.Ref()
|
||||
}
|
||||
switch ap.Kind {
|
||||
case ansible.APICollection:
|
||||
return ansible.CollectionDoc(base, ap.Namespace, ap.Name, refs), nil
|
||||
case ansible.APIVersions:
|
||||
limit, offset := len(refs), 0
|
||||
if q != nil {
|
||||
limit, offset = ansible.Page(q)
|
||||
}
|
||||
return ansible.VersionsPage(base, ap.Namespace, ap.Name, refs, limit, offset), nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Version == ap.Version {
|
||||
return ansible.VersionDoc(base, e), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, virtual.ErrNotFound
|
||||
}
|
||||
|
||||
// localAnsibleEntries returns a collection's versions, newest publish first.
|
||||
func (h *ProxyHandler) localAnsibleEntries(ctx context.Context, repo, ns, name string) ([]ansible.Entry, error) {
|
||||
if !ansible.NameRe.MatchString(ns) || !ansible.NameRe.MatchString(name) {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := ansible.IndexPrefix + ns + "/" + name + "/"
|
||||
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(b.ID, a.ID) })
|
||||
var out []ansible.Entry
|
||||
for _, f := range files {
|
||||
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
|
||||
if !strings.HasPrefix(f.FilePath, prefix) {
|
||||
continue
|
||||
}
|
||||
raw, err := h.readLocalBlob(ctx, f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var e ansible.Entry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return nil, fmt.Errorf("decode %s: %w", f.FilePath, err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func testCollection(t *testing.T, ns, name, version string) []byte {
|
||||
t.Helper()
|
||||
manifest := `{"collection_info":{"namespace":"` + ns + `","name":"` + name + `","version":"` + version + `","dependencies":{"ansible.posix":">=1.0.0"}}}`
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "MANIFEST.json", Mode: 0o644, Size: int64(len(manifest))})
|
||||
_, _ = tw.Write([]byte(manifest))
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// ansiblePublish uploads like ansible-galaxy: the file part base64-encoded
|
||||
// whenever a sha256 field is sent, raw otherwise.
|
||||
func ansiblePublish(t *testing.T, url string, tarball []byte, sum string) (int, string) {
|
||||
t.Helper()
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
hdr := textproto.MIMEHeader{"Content-Disposition": {`form-data; name="file"; filename="c.tar.gz"`}}
|
||||
data := tarball
|
||||
if sum != "" {
|
||||
_ = mw.WriteField("sha256", sum)
|
||||
hdr.Set("Content-Transfer-Encoding", "base64")
|
||||
data = []byte(base64.StdEncoding.EncodeToString(tarball))
|
||||
}
|
||||
fw, _ := mw.CreatePart(hdr)
|
||||
_, _ = fw.Write(data)
|
||||
_ = mw.Close()
|
||||
req, _ := http.NewRequest(http.MethodPost, url, &body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
var b bytes.Buffer
|
||||
_, _ = b.ReadFrom(resp.Body)
|
||||
return resp.StatusCode, b.String()
|
||||
}
|
||||
|
||||
func getJSON(t *testing.T, url string, v any) {
|
||||
t.Helper()
|
||||
code, body := npmDo(t, http.MethodGet, url, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d: %s", url, code, body)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), v); err != nil {
|
||||
t.Fatalf("GET %s: %v: %s", url, err, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnsiblePublishAndServe(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageAnsible)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
publish := base + "api/v3/artifacts/collections/"
|
||||
|
||||
v1 := testCollection(t, "unkin", "e2e_hello", "1.0.0")
|
||||
code, body := ansiblePublish(t, publish, v1, sha256Hex(v1))
|
||||
if code != http.StatusAccepted {
|
||||
t.Fatalf("publish = %d: %s", code, body)
|
||||
}
|
||||
var task struct{ Task string }
|
||||
_ = json.Unmarshal([]byte(body), &task)
|
||||
if task.Task != base+"api/v3/imports/collections/unkin-e2e_hello-1.0.0/" {
|
||||
t.Fatalf("task = %q", task.Task)
|
||||
}
|
||||
var st map[string]any
|
||||
getJSON(t, task.Task, &st)
|
||||
if st["state"] != "completed" || st["finished_at"] == "" {
|
||||
t.Fatalf("task = %v", st)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"api/v3/imports/collections/unkin-e2e_hello-9.9.9/", ""); code != http.StatusNotFound {
|
||||
t.Errorf("unknown task = %d, want 404", code)
|
||||
}
|
||||
|
||||
for _, v := range []string{"1.1.0", "2.0.0"} {
|
||||
tb := testCollection(t, "unkin", "e2e_hello", v)
|
||||
if code, body := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
|
||||
t.Fatalf("publish %s = %d: %s", v, code, body)
|
||||
}
|
||||
}
|
||||
if code, body := ansiblePublish(t, publish, v1, ""); code != http.StatusConflict || !strings.Contains(body, "already exists") {
|
||||
t.Errorf("identical republish = %d: %s, want 409", code, body)
|
||||
}
|
||||
if code, _ := ansiblePublish(t, publish, testCollection(t, "unkin", "e2e_hello", "1.0.0")[:20], ""); code != http.StatusBadRequest {
|
||||
t.Errorf("truncated tarball = %d, want 400", code)
|
||||
}
|
||||
|
||||
var root map[string]map[string]string
|
||||
getJSON(t, base+"api/", &root)
|
||||
if root["available_versions"]["v3"] != "v3/" {
|
||||
t.Fatalf("root = %v", root)
|
||||
}
|
||||
|
||||
var page struct {
|
||||
Meta struct{ Count int }
|
||||
Links struct{ Next, Previous *string }
|
||||
Data []struct{ Version, Href string }
|
||||
}
|
||||
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/versions/?limit=2", &page)
|
||||
if page.Meta.Count != 3 || len(page.Data) != 2 || page.Data[0].Version != "2.0.0" || page.Links.Next == nil || page.Links.Previous != nil {
|
||||
t.Fatalf("page 1 = %+v", page)
|
||||
}
|
||||
next := *page.Links.Next
|
||||
if !strings.HasPrefix(next, "/api/v1/local/"+repo+"/") {
|
||||
t.Fatalf("next = %q, want root-relative", next)
|
||||
}
|
||||
next = srv.URL + next
|
||||
page.Links.Next = nil
|
||||
getJSON(t, next, &page)
|
||||
if len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next != nil {
|
||||
t.Fatalf("page 2 (%s) = %+v", next, page)
|
||||
}
|
||||
|
||||
var detail struct {
|
||||
DownloadURL string `json:"download_url"`
|
||||
Artifact struct{ Sha256 string }
|
||||
Metadata struct{ Dependencies map[string]string }
|
||||
Namespace struct{ Name string }
|
||||
Collection struct{ Name string }
|
||||
}
|
||||
getJSON(t, page.Data[0].Href, &detail)
|
||||
if detail.DownloadURL != base+"artifacts/unkin-e2e_hello-1.0.0.tar.gz" || detail.Artifact.Sha256 != sha256Hex(v1) ||
|
||||
detail.Metadata.Dependencies["ansible.posix"] != ">=1.0.0" || detail.Namespace.Name != "unkin" || detail.Collection.Name != "e2e_hello" {
|
||||
t.Fatalf("detail = %+v", detail)
|
||||
}
|
||||
if code, got := npmDo(t, http.MethodGet, detail.DownloadURL, ""); code != http.StatusOK || got != string(v1) {
|
||||
t.Fatalf("download = %d (%d bytes)", code, len(got))
|
||||
}
|
||||
|
||||
var coll map[string]any
|
||||
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/", &coll)
|
||||
if coll["versions_url"] != base+"api/v3/collections/unkin/e2e_hello/versions/" || coll["updated_at"] == "" {
|
||||
t.Fatalf("collection = %v", coll)
|
||||
}
|
||||
for _, p := range []string{"api/v3/collections/unkin/nope/", "api/v3/collections/unkin/e2e_hello/versions/9.0.0/", "_index/unkin/e2e_hello/1.0.0", "other"} {
|
||||
if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound {
|
||||
t.Errorf("GET %s = %d, want 404", p, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnsiblePublishRejects(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageAnsible)
|
||||
publish := srv.URL + "/api/v1/local/" + repo + "/api/v3/artifacts/collections/"
|
||||
tb := testCollection(t, "unkin", "rej", "1.0.0")
|
||||
|
||||
if code, _ := ansiblePublish(t, publish, tb, strings.Repeat("0", 64)); code != http.StatusBadRequest {
|
||||
t.Errorf("sha256 mismatch = %d, want 400", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPost, publish, "raw"); code != http.StatusBadRequest {
|
||||
t.Errorf("non-multipart = %d, want 400", code)
|
||||
}
|
||||
old := ansibleMaxPublishBytes
|
||||
ansibleMaxPublishBytes = 64
|
||||
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("oversize = %d, want 413", code)
|
||||
}
|
||||
ansibleMaxPublishBytes = old
|
||||
generic := npmRepo(t, h, models.PackageGeneric)
|
||||
if code, _ := ansiblePublish(t, srv.URL+"/api/v1/local/"+generic+"/api/v3/artifacts/collections/", tb, ""); code != http.StatusMethodNotAllowed {
|
||||
t.Errorf("publish to generic local = %d, want 405", code)
|
||||
}
|
||||
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
|
||||
t.Errorf("publish after rejects = %d, want 202", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnsibleLocalPerHost(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageAnsible)
|
||||
if code, b := ansiblePublish(t, srv.URL+"/api/v1/local/"+repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "host", "1.0.0"), ""); code != http.StatusAccepted {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
routes := h.Routes()
|
||||
path := "/local/" + repo + "/api/v3/collections/unkin/host/versions/1.0.0/"
|
||||
r := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
r.Host = "x@evil.com"
|
||||
w := httptest.NewRecorder()
|
||||
routes.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("bad Host = %d, want 400", w.Code)
|
||||
}
|
||||
r = httptest.NewRequest(http.MethodGet, path, nil)
|
||||
r.Host = "aa.example"
|
||||
r.Header.Set("X-Forwarded-Proto", "https")
|
||||
w = httptest.NewRecorder()
|
||||
routes.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"download_url":"https://aa.example/api/v1/local/`+repo+`/artifacts/unkin-host-1.0.0.tar.gz"`) {
|
||||
t.Fatalf("detail = %d: %s", w.Code, w.Body)
|
||||
}
|
||||
if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto" {
|
||||
t.Errorf("Cache-Control %q Vary %q", cc, vary)
|
||||
}
|
||||
}
|
||||
@@ -64,6 +64,10 @@ func (h *ProxyHandler) lockGemVersion(w http.ResponseWriter, r *http.Request, re
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) {
|
||||
if isAnsiblePublish(chi.URLParam(r, "*")) {
|
||||
h.handleAnsiblePublish(w, r)
|
||||
return
|
||||
}
|
||||
if chi.URLParam(r, "*") != "api/v1/gems" {
|
||||
gemsText(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
@@ -275,6 +276,17 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
if remote.PackageType == models.PackageAnsible {
|
||||
if ap := ansible.ParseAPIPath(path); ap.Kind != ansible.APIUnknown {
|
||||
h.serveLocalAnsible(w, r, remote, ap)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(path, ansible.ArtifactPrefix) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
|
||||
@@ -0,0 +1,347 @@
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Local ansible repos speak the Galaxy v3 publish and install API. Per version:
|
||||
// artifacts/<ns>-<name>-<version>.tar.gz collection tarball (immutable)
|
||||
// _index/<ns>/<name>/<version> Entry JSON; gates visibility
|
||||
|
||||
const (
|
||||
IndexPrefix = "_index/"
|
||||
ArtifactPrefix = "artifacts/"
|
||||
PublishPath = "api/v3/artifacts/collections/"
|
||||
)
|
||||
|
||||
var (
|
||||
NameRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
|
||||
VersionRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`)
|
||||
)
|
||||
|
||||
var (
|
||||
MaxManifestBytes int64 = 1 << 20
|
||||
MaxScanBytes int64 = 512 << 20
|
||||
)
|
||||
|
||||
// Entry is one published collection version.
|
||||
type Entry struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
Metadata json.RawMessage `json:"metadata"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (e Entry) Filename() string { return e.Namespace + "-" + e.Name + "-" + e.Version + ".tar.gz" }
|
||||
|
||||
// IndexPath is the entry's hidden index file.
|
||||
func (e Entry) IndexPath() string { return IndexPrefix + e.Namespace + "/" + e.Name + "/" + e.Version }
|
||||
|
||||
// TaskID names the import task returned on publish.
|
||||
func (e Entry) TaskID() string { return e.Namespace + "-" + e.Name + "-" + e.Version }
|
||||
|
||||
// ParseCollection reads collection_info from a collection tarball's top-level
|
||||
// MANIFEST.json. Every member is checked, so a tarball whose paths or links
|
||||
// escape the collection root is rejected even after the manifest.
|
||||
func ParseCollection(r io.Reader) (*Entry, error) {
|
||||
zr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not a gzip tarball: %w", err)
|
||||
}
|
||||
scan := &io.LimitedReader{R: zr, N: MaxScanBytes}
|
||||
tr := tar.NewReader(scan)
|
||||
var e *Entry
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if scan.N <= 0 {
|
||||
return nil, errors.New("collection exceeds decompressed size limit")
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read tarball: %w", err)
|
||||
}
|
||||
name, err := memberPath(hdr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name != "MANIFEST.json" || hdr.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
if e != nil {
|
||||
return nil, errors.New("duplicate MANIFEST.json")
|
||||
}
|
||||
if e, err = parseManifest(io.LimitReader(tr, MaxManifestBytes+1)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if e == nil {
|
||||
return nil, errors.New("no MANIFEST.json in collection tarball")
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// memberPath returns hdr's cleaned name, rejecting absolute or escaping
|
||||
// paths and links, and anything but files, directories and symlinks.
|
||||
func memberPath(hdr *tar.Header) (string, error) {
|
||||
name := path.Clean(strings.TrimPrefix(hdr.Name, "./"))
|
||||
if escapes(name) {
|
||||
return "", fmt.Errorf("tarball member %q escapes the collection root", hdr.Name)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeReg, tar.TypeDir:
|
||||
case tar.TypeSymlink:
|
||||
if path.IsAbs(hdr.Linkname) || escapes(path.Join(path.Dir(name), hdr.Linkname)) {
|
||||
return "", fmt.Errorf("tarball symlink %q escapes the collection root", hdr.Name)
|
||||
}
|
||||
default:
|
||||
return "", fmt.Errorf("tarball member %q has unsupported type %q", hdr.Name, hdr.Typeflag)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
func escapes(p string) bool {
|
||||
return path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") || strings.Contains(p, "\\")
|
||||
}
|
||||
|
||||
func parseManifest(r io.Reader) (*Entry, error) {
|
||||
raw, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read MANIFEST.json: %w", err)
|
||||
}
|
||||
if int64(len(raw)) > MaxManifestBytes {
|
||||
return nil, fmt.Errorf("MANIFEST.json exceeds %d bytes", MaxManifestBytes)
|
||||
}
|
||||
var doc struct {
|
||||
CollectionInfo map[string]json.RawMessage `json:"collection_info"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil || doc.CollectionInfo == nil {
|
||||
return nil, errors.New("MANIFEST.json has no collection_info")
|
||||
}
|
||||
info := doc.CollectionInfo
|
||||
e := &Entry{}
|
||||
for k, dst := range map[string]*string{"namespace": &e.Namespace, "name": &e.Name, "version": &e.Version} {
|
||||
if err := json.Unmarshal(info[k], dst); err != nil {
|
||||
return nil, fmt.Errorf("collection_info.%s is not a string", k)
|
||||
}
|
||||
}
|
||||
if !NameRe.MatchString(e.Namespace) || !NameRe.MatchString(e.Name) || !VersionRe.MatchString(e.Version) {
|
||||
return nil, fmt.Errorf("invalid collection %q.%q version %q", e.Namespace, e.Name, e.Version)
|
||||
}
|
||||
deps := map[string]string{}
|
||||
if d := info["dependencies"]; len(d) > 0 && string(d) != "null" {
|
||||
if err := json.Unmarshal(d, &deps); err != nil {
|
||||
return nil, errors.New("collection_info.dependencies must map collection names to version ranges")
|
||||
}
|
||||
}
|
||||
for k := range deps {
|
||||
ns, n, ok := strings.Cut(k, ".")
|
||||
if !ok || !NameRe.MatchString(ns) || !NameRe.MatchString(n) {
|
||||
return nil, fmt.Errorf("invalid dependency %q", k)
|
||||
}
|
||||
}
|
||||
info["dependencies"], _ = json.Marshal(deps)
|
||||
if e.Metadata, err = json.Marshal(info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// APIPath is a parsed Galaxy v3 request path below a repo.
|
||||
type APIPath struct {
|
||||
Kind APIKind
|
||||
Namespace, Name, Version string
|
||||
TaskID string
|
||||
}
|
||||
|
||||
type APIKind int
|
||||
|
||||
const (
|
||||
APIUnknown APIKind = iota
|
||||
APIRoot
|
||||
APICollection
|
||||
APIVersions
|
||||
APIVersion
|
||||
APIImport
|
||||
)
|
||||
|
||||
// ParseAPIPath parses api/, api/v3/collections/<ns>/<name>/[versions/[<v>/]]
|
||||
// and api/v3/imports/collections/<id>/, with or without trailing slashes.
|
||||
func ParseAPIPath(p string) APIPath {
|
||||
p = strings.TrimSuffix(p, "/")
|
||||
if p == "api" {
|
||||
return APIPath{Kind: APIRoot}
|
||||
}
|
||||
if id, ok := strings.CutPrefix(p, "api/v3/imports/collections/"); ok && id != "" && !strings.Contains(id, "/") {
|
||||
return APIPath{Kind: APIImport, TaskID: id}
|
||||
}
|
||||
rest, ok := strings.CutPrefix(p, "api/v3/collections/")
|
||||
if !ok {
|
||||
return APIPath{}
|
||||
}
|
||||
segs := strings.Split(rest, "/")
|
||||
if len(segs) < 2 || !NameRe.MatchString(segs[0]) || !NameRe.MatchString(segs[1]) {
|
||||
return APIPath{}
|
||||
}
|
||||
ap := APIPath{Namespace: segs[0], Name: segs[1]}
|
||||
switch {
|
||||
case len(segs) == 2:
|
||||
ap.Kind = APICollection
|
||||
case len(segs) == 3 && segs[2] == "versions":
|
||||
ap.Kind = APIVersions
|
||||
case len(segs) == 4 && segs[2] == "versions" && VersionRe.MatchString(segs[3]):
|
||||
ap.Kind, ap.Version = APIVersion, segs[3]
|
||||
default:
|
||||
return APIPath{}
|
||||
}
|
||||
return ap
|
||||
}
|
||||
|
||||
// Root is the API root ansible-galaxy discovers.
|
||||
func Root() map[string]any {
|
||||
return map[string]any{"available_versions": map[string]string{"v3": "v3/"}}
|
||||
}
|
||||
|
||||
func CollectionPath(ns, name string) string { return "/api/v3/collections/" + ns + "/" + name + "/" }
|
||||
|
||||
func VersionPath(ns, name, version string) string {
|
||||
return CollectionPath(ns, name) + "versions/" + url.PathEscape(version) + "/"
|
||||
}
|
||||
|
||||
// VersionRef is one entry of a versions list.
|
||||
type VersionRef struct {
|
||||
Version string `json:"version"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
UpdatedAt string `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
func (e Entry) Ref() VersionRef {
|
||||
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
|
||||
return VersionRef{Version: e.Version, CreatedAt: t, UpdatedAt: t}
|
||||
}
|
||||
|
||||
// CollectionDoc describes a collection from its versions; base is the repo
|
||||
// URL the hrefs are built on.
|
||||
func CollectionDoc(base, ns, name string, refs []VersionRef) map[string]any {
|
||||
var created, updated time.Time
|
||||
for _, r := range refs {
|
||||
if t, err := time.Parse(time.RFC3339Nano, r.CreatedAt); err == nil && (created.IsZero() || t.Before(created)) {
|
||||
created = t
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339Nano, r.UpdatedAt); err == nil && t.After(updated) {
|
||||
updated = t
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"href": base + CollectionPath(ns, name),
|
||||
"namespace": ns,
|
||||
"name": name,
|
||||
"deprecated": false,
|
||||
"versions_url": base + CollectionPath(ns, name) + "versions/",
|
||||
"created_at": formatTime(created),
|
||||
"updated_at": formatTime(updated),
|
||||
}
|
||||
}
|
||||
|
||||
func formatTime(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Page reads limit/offset as the remote forwards them, defaulting to 100/0.
|
||||
func Page(q url.Values) (limit, offset int) {
|
||||
limit, offset = maxPageLimit, 0
|
||||
if v, ok := pageLimit(q.Get("limit")); ok {
|
||||
limit, _ = strconv.Atoi(v)
|
||||
}
|
||||
if v, ok := pageOffset(q.Get("offset")); ok {
|
||||
offset, _ = strconv.Atoi(v)
|
||||
}
|
||||
return limit, offset
|
||||
}
|
||||
|
||||
// VersionsPage is one page of refs in Galaxy v3 list form. Links are
|
||||
// root-relative like Galaxy's, since ansible-galaxy splices them onto the
|
||||
// server URL.
|
||||
func VersionsPage(base, ns, name string, refs []VersionRef, limit, offset int) map[string]any {
|
||||
list := rootPath(base) + CollectionPath(ns, name) + "versions/"
|
||||
link := func(off int) any {
|
||||
return list + "?" + url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}.Encode()
|
||||
}
|
||||
offset = min(offset, len(refs))
|
||||
var previous, next any
|
||||
if offset > 0 {
|
||||
previous = link(max(offset-limit, 0))
|
||||
}
|
||||
if offset+limit < len(refs) {
|
||||
next = link(offset + limit)
|
||||
}
|
||||
data := []map[string]any{}
|
||||
for _, r := range refs[offset:min(offset+limit, len(refs))] {
|
||||
data = append(data, map[string]any{
|
||||
"version": r.Version,
|
||||
"href": base + VersionPath(ns, name, r.Version),
|
||||
"created_at": r.CreatedAt,
|
||||
"updated_at": r.UpdatedAt,
|
||||
"marks": []string{},
|
||||
})
|
||||
}
|
||||
return map[string]any{
|
||||
"meta": map[string]any{"count": len(refs)},
|
||||
"links": map[string]any{"first": link(0), "previous": previous, "next": next, "last": link(max(len(refs)-1, 0) / limit * limit)},
|
||||
"data": data,
|
||||
}
|
||||
}
|
||||
|
||||
// rootPath drops the scheme and host from base.
|
||||
func rootPath(base string) string {
|
||||
if u, err := url.Parse(base); err == nil {
|
||||
return u.EscapedPath()
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
// VersionDoc is a version's detail; the tarball is served from the repo at
|
||||
// artifacts/<filename>.
|
||||
func VersionDoc(base string, e Entry) map[string]any {
|
||||
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
|
||||
return map[string]any{
|
||||
"href": base + VersionPath(e.Namespace, e.Name, e.Version),
|
||||
"namespace": map[string]string{"name": e.Namespace},
|
||||
"collection": map[string]string{"name": e.Name},
|
||||
"name": e.Name,
|
||||
"version": e.Version,
|
||||
"download_url": base + "/" + ArtifactPrefix + e.Filename(),
|
||||
"artifact": map[string]any{"filename": e.Filename(), "sha256": e.SHA256, "size": e.Size},
|
||||
"metadata": e.Metadata,
|
||||
"signatures": []string{},
|
||||
"created_at": t,
|
||||
"updated_at": t,
|
||||
}
|
||||
}
|
||||
|
||||
// ImportTask is a finished import: publishing completes synchronously.
|
||||
func ImportTask(e Entry) map[string]any {
|
||||
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
|
||||
return map[string]any{
|
||||
"id": e.TaskID(), "state": "completed", "created_at": t, "started_at": t, "finished_at": t,
|
||||
"messages": []any{}, "error": nil,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type member struct {
|
||||
hdr tar.Header
|
||||
body string
|
||||
}
|
||||
|
||||
func collectionTar(t *testing.T, members ...member) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for _, m := range members {
|
||||
h := m.hdr
|
||||
if h.Typeflag == 0 {
|
||||
h.Typeflag = tar.TypeReg
|
||||
}
|
||||
if h.Typeflag == tar.TypeReg {
|
||||
h.Size = int64(len(m.body))
|
||||
}
|
||||
h.Mode = 0o644
|
||||
if err := tw.WriteHeader(&h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = tw.Write([]byte(m.body))
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func manifest(ns, name, version, deps string) member {
|
||||
return member{tar.Header{Name: "MANIFEST.json"}, `{"collection_info":{"namespace":"` + ns + `","name":"` + name +
|
||||
`","version":"` + version + `","authors":["a"],"dependencies":` + deps + `}}`}
|
||||
}
|
||||
|
||||
func TestParseCollection(t *testing.T) {
|
||||
e, err := ParseCollection(bytes.NewReader(collectionTar(t,
|
||||
member{tar.Header{Name: "plugins/", Typeflag: tar.TypeDir}, ""},
|
||||
member{tar.Header{Name: "plugins/modules/hello.py"}, "x"},
|
||||
member{tar.Header{Name: "docs/link", Typeflag: tar.TypeSymlink, Linkname: "../plugins/modules/hello.py"}, ""},
|
||||
manifest("unkin", "e2e_hello", "1.0.0-rc.1+b1", `{"ansible.posix":">=1.0.0"}`),
|
||||
)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Namespace != "unkin" || e.Name != "e2e_hello" || e.Version != "1.0.0-rc.1+b1" || e.Filename() != "unkin-e2e_hello-1.0.0-rc.1+b1.tar.gz" {
|
||||
t.Fatalf("entry = %+v", e)
|
||||
}
|
||||
var md struct {
|
||||
Authors []string `json:"authors"`
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Metadata, &md); err != nil || md.Dependencies["ansible.posix"] != ">=1.0.0" || len(md.Authors) != 1 {
|
||||
t.Fatalf("metadata = %s (%v)", e.Metadata, err)
|
||||
}
|
||||
|
||||
e, err = ParseCollection(bytes.NewReader(collectionTar(t, manifest("a", "b", "1.0.0", "null"))))
|
||||
if err != nil || !strings.Contains(string(e.Metadata), `"dependencies":{}`) {
|
||||
t.Fatalf("null dependencies = %s (%v), want {}", e.Metadata, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCollectionRejects(t *testing.T) {
|
||||
ok := manifest("a", "b", "1.0.0", "{}")
|
||||
tests := []struct {
|
||||
name string
|
||||
members []member
|
||||
want string
|
||||
}{
|
||||
{"no manifest", []member{{tar.Header{Name: "README.md"}, "x"}}, "no MANIFEST.json"},
|
||||
{"nested manifest only", []member{{tar.Header{Name: "x/MANIFEST.json"}, ok.body}}, "no MANIFEST.json"},
|
||||
{"dotdot", []member{ok, {tar.Header{Name: "../evil"}, "x"}}, "escapes"},
|
||||
{"dotdot after clean", []member{ok, {tar.Header{Name: "a/../../evil"}, "x"}}, "escapes"},
|
||||
{"absolute", []member{ok, {tar.Header{Name: "/etc/passwd"}, "x"}}, "escapes"},
|
||||
{"symlink escape", []member{ok, {tar.Header{Name: "a/l", Typeflag: tar.TypeSymlink, Linkname: "../../x"}, ""}}, "symlink"},
|
||||
{"absolute symlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeSymlink, Linkname: "/etc"}, ""}}, "symlink"},
|
||||
{"hardlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeLink, Linkname: "MANIFEST.json"}, ""}}, "unsupported type"},
|
||||
{"duplicate manifest", []member{ok, ok}, "duplicate"},
|
||||
{"bad namespace", []member{manifest("Bad", "b", "1.0.0", "{}")}, "invalid collection"},
|
||||
{"bad version", []member{manifest("a", "b", "1.0", "{}")}, "invalid collection"},
|
||||
{"bad dependency", []member{manifest("a", "b", "1.0.0", `{"nodot":"*"}`)}, "invalid dependency"},
|
||||
{"list dependencies", []member{manifest("a", "b", "1.0.0", `["x.y"]`)}, "dependencies must map"},
|
||||
{"huge manifest", []member{{tar.Header{Name: "MANIFEST.json"}, strings.Repeat(" ", int(MaxManifestBytes)+1)}}, "exceeds"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := ParseCollection(bytes.NewReader(collectionTar(t, tt.members...)))
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("err = %v, want %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := ParseCollection(strings.NewReader("not gzip")); err == nil {
|
||||
t.Error("non-gzip input accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCollectionScanLimit(t *testing.T) {
|
||||
old := MaxScanBytes
|
||||
MaxScanBytes = 4096
|
||||
defer func() { MaxScanBytes = old }()
|
||||
_, err := ParseCollection(bytes.NewReader(collectionTar(t,
|
||||
member{tar.Header{Name: "big"}, strings.Repeat("x", 8192)},
|
||||
manifest("a", "b", "1.0.0", "{}"),
|
||||
)))
|
||||
if err == nil || !strings.Contains(err.Error(), "decompressed size limit") {
|
||||
t.Fatalf("err = %v, want decompressed size limit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAPIPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want APIPath
|
||||
}{
|
||||
{"api", APIPath{Kind: APIRoot}},
|
||||
{"api/", APIPath{Kind: APIRoot}},
|
||||
{"api/v3/collections/unkin/e2e_hello/", APIPath{Kind: APICollection, Namespace: "unkin", Name: "e2e_hello"}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions", APIPath{Kind: APIVersions, Namespace: "unkin", Name: "e2e_hello"}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions/1.0.0/", APIPath{Kind: APIVersion, Namespace: "unkin", Name: "e2e_hello", Version: "1.0.0"}},
|
||||
{"api/v3/imports/collections/unkin-e2e_hello-1.0.0/", APIPath{Kind: APIImport, TaskID: "unkin-e2e_hello-1.0.0"}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions/1.0/", APIPath{}},
|
||||
{"api/v3/collections/../x/", APIPath{}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions/1.0.0/extra/", APIPath{}},
|
||||
{"artifacts/unkin-e2e_hello-1.0.0.tar.gz", APIPath{}},
|
||||
{"api/v3/imports/collections/a/b/", APIPath{}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := ParseAPIPath(tt.in); got != tt.want {
|
||||
t.Errorf("ParseAPIPath(%q) = %+v, want %+v", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionsPage(t *testing.T) {
|
||||
refs := []VersionRef{{Version: "3.0.0"}, {Version: "2.0.0"}, {Version: "1.0.0"}}
|
||||
list := "/r/api/v3/collections/a/b/versions/"
|
||||
links := func(p map[string]any) map[string]any { return p["links"].(map[string]any) }
|
||||
versions := func(p map[string]any) (out []string) {
|
||||
for _, d := range p["data"].([]map[string]any) {
|
||||
out = append(out, d["version"].(string))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
p := VersionsPage("https://h/r", "a", "b", refs, 2, 0)
|
||||
if got := versions(p); strings.Join(got, ",") != "3.0.0,2.0.0" {
|
||||
t.Fatalf("page 1 = %v", got)
|
||||
}
|
||||
if l := links(p); l["previous"] != nil || l["next"] != list+"?limit=2&offset=2" || l["last"] != list+"?limit=2&offset=2" {
|
||||
t.Fatalf("page 1 links = %v", l)
|
||||
}
|
||||
if p["meta"].(map[string]any)["count"] != 3 {
|
||||
t.Fatalf("count = %v", p["meta"])
|
||||
}
|
||||
|
||||
p = VersionsPage("https://h/r", "a", "b", refs, 2, 2)
|
||||
if got := versions(p); strings.Join(got, ",") != "1.0.0" {
|
||||
t.Fatalf("page 2 = %v", got)
|
||||
}
|
||||
if l := links(p); l["next"] != nil || l["previous"] != list+"?limit=2&offset=0" {
|
||||
t.Fatalf("page 2 links = %v", l)
|
||||
}
|
||||
|
||||
if got := versions(VersionsPage("", "a", "b", refs, 2, 99)); len(got) != 0 {
|
||||
t.Fatalf("offset past end = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVersionsPageFollow pages through more than 100 versions the way
|
||||
// ansible-galaxy does: old clients splice next onto the versions URL in place
|
||||
// of its path, new ones urljoin it onto the server.
|
||||
func TestVersionsPageFollow(t *testing.T) {
|
||||
var refs []VersionRef
|
||||
for i := 104; i >= 0; i-- {
|
||||
refs = append(refs, VersionRef{Version: fmt.Sprintf("1.0.%d", i)})
|
||||
}
|
||||
server := "https://h:8443/api/v1/local/r%2Bx/"
|
||||
base := strings.TrimSuffix(server, "/")
|
||||
versionsURL := server + "api/v3/collections/a/b/versions/"
|
||||
for _, follow := range map[string]func(string) string{
|
||||
"replace": func(link string) string {
|
||||
u, _ := url.Parse(versionsURL)
|
||||
return strings.Replace(versionsURL, u.EscapedPath(), link, 1)
|
||||
},
|
||||
"urljoin": func(link string) string {
|
||||
s, _ := url.Parse(server)
|
||||
l, _ := url.Parse(link)
|
||||
return s.ResolveReference(l).String()
|
||||
},
|
||||
} {
|
||||
var got []string
|
||||
next := versionsURL + "?limit=100"
|
||||
for pages := 0; next != ""; pages++ {
|
||||
if pages > 3 {
|
||||
t.Fatal("pagination does not terminate")
|
||||
}
|
||||
u, err := url.Parse(next)
|
||||
if err != nil || u.Host != "h:8443" || !strings.HasPrefix(u.EscapedPath(), "/api/v1/local/r%2Bx/api/v3/") {
|
||||
t.Fatalf("followed link %q is not on the server", next)
|
||||
}
|
||||
limit, offset := Page(u.Query())
|
||||
p := VersionsPage(base, "a", "b", refs, limit, offset)
|
||||
for _, d := range p["data"].([]map[string]any) {
|
||||
got = append(got, d["version"].(string))
|
||||
}
|
||||
next = ""
|
||||
if l, ok := p["links"].(map[string]any)["next"].(string); ok {
|
||||
next = follow(l)
|
||||
}
|
||||
}
|
||||
if len(got) != 105 || got[0] != "1.0.104" || got[104] != "1.0.0" {
|
||||
t.Fatalf("collected %d versions, first %v", len(got), got[:1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionDocTimes(t *testing.T) {
|
||||
d := CollectionDoc("", "a", "b", []VersionRef{
|
||||
{CreatedAt: "2026-01-01T00:00:05Z", UpdatedAt: "2026-01-01T00:00:05Z"},
|
||||
{CreatedAt: "2026-01-01T00:00:05.5Z", UpdatedAt: "2026-01-01T00:00:05.5Z"},
|
||||
{CreatedAt: "2026-01-01T00:00:04.9Z", UpdatedAt: "2026-01-01T00:00:04.9Z"},
|
||||
})
|
||||
if d["created_at"] != "2026-01-01T00:00:04.9Z" || d["updated_at"] != "2026-01-01T00:00:05.5Z" {
|
||||
t.Fatalf("created/updated = %v/%v", d["created_at"], d["updated_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPage(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
q string
|
||||
limit, offset int
|
||||
}{
|
||||
{"", 100, 0},
|
||||
{"limit=5&offset=10", 5, 10},
|
||||
{"limit=5000", 100, 0},
|
||||
{"limit=0&offset=-1", 100, 0},
|
||||
} {
|
||||
q, _ := url.ParseQuery(tt.q)
|
||||
if l, o := Page(q); l != tt.limit || o != tt.offset {
|
||||
t.Errorf("Page(%q) = %d,%d want %d,%d", tt.q, l, o, tt.limit, tt.offset)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,19 @@ import type { PackageTypeUI } from '../types';
|
||||
|
||||
const def: PackageTypeUI = {
|
||||
color: 'red',
|
||||
snippets: ({ proxy }) => {
|
||||
snippets: ({ name, url, proxy, isLocal }) => {
|
||||
if (isLocal) {
|
||||
const src = `${url}/api/v1/local/${name}/`;
|
||||
return [
|
||||
{
|
||||
title: 'Publish and install',
|
||||
language: 'bash',
|
||||
code: `ansible-galaxy collection publish <ns>-<name>-<version>.tar.gz --server ${src} --api-key unused
|
||||
ansible-galaxy collection install <ns>.<name> --server ${src}`,
|
||||
note: 'publish needs an API key; any value works. Publishing an existing version is rejected.',
|
||||
},
|
||||
];
|
||||
}
|
||||
return [
|
||||
{
|
||||
title: 'Install collections through this Galaxy proxy',
|
||||
|
||||
Reference in New Issue
Block a user