artifactapi/internal/virtual/ansible_merger.go

package virtual

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
	"git.unkin.net/unkin/artifactapi/internal/proxy"
	"git.unkin.net/unkin/artifactapi/pkg/models"
)

const (
	maxAnsibleCollections = 1024
	ansibleMemberPage     = 100
	maxAnsibleMemberPages = 100
)

// AnsibleMember is one member's versions of a collection; Found is false when
// the member does not have it.
type AnsibleMember struct {
	RemoteName string
	Found      bool
	Refs       []ansible.VersionRef
}

// AnsibleDoc is a collection's merged versions; Owners maps each version to
// the member serving it.
type AnsibleDoc struct {
	Refs   []ansible.VersionRef
	Owners map[string]string
	body   []byte
}

func (d *AnsibleDoc) index() []byte            { return d.body }
func (d *AnsibleDoc) files() map[string][]byte { return nil }

// MergeAnsibleVersions concatenates member versions in member order; the
// first member wins a duplicate version. ErrNotFound means no member has the
// collection.
func MergeAnsibleVersions(members []AnsibleMember) (*AnsibleDoc, error) {
	d := &AnsibleDoc{Refs: []ansible.VersionRef{}, Owners: map[string]string{}}
	found := false
	for _, m := range members {
		found = found || m.Found
		for _, r := range m.Refs {
			if _, dup := d.Owners[r.Version]; dup {
				continue
			}
			d.Owners[r.Version] = m.RemoteName
			d.Refs = append(d.Refs, r)
		}
	}
	if !found {
		return nil, ErrNotFound
	}
	var err error
	d.body, err = json.Marshal(d.Refs)
	return d, err
}

// AnsibleVersions returns the merged versions of ns.name across members.
func (e *Engine) AnsibleVersions(ctx context.Context, virt models.Virtual, ns, name string) (*AnsibleDoc, error) {
	prefix := "ansible\x00" + virt.Name + "\x00"
	inserted := false
	repo, err := e.merged(ctx, prefix+ns+"/"+name, func(ctx context.Context) (mergedRepo, error) {
		inserted = true
		members, err := fetchMembers(virt, func(m string) (*AnsibleMember, error) { return e.ansibleMember(ctx, m, ns, name) })
		if err != nil {
			return nil, err
		}
		return MergeAnsibleVersions(members)
	})
	if err != nil {
		return nil, err
	}
	if inserted {
		e.evictOldest(prefix, maxAnsibleCollections)
	}
	return repo.(*AnsibleDoc), nil
}

// AnsibleVersion serves the owning member's version detail with href and
// download_url pointing at the virtual; the download 302s to the member.
// Member documents carry root-relative URLs, so anything else is rejected.
func (e *Engine) AnsibleVersion(ctx context.Context, virt models.Virtual, ns, name, version, proxyBaseURL string) ([]byte, error) {
	d, err := e.AnsibleVersions(ctx, virt, ns, name)
	if err != nil {
		return nil, err
	}
	owner, ok := d.Owners[version]
	if !ok {
		return nil, ErrNotFound
	}
	remote, err := e.getRemote(ctx, owner)
	if err != nil {
		return nil, fmt.Errorf("remote %q: %w", owner, err)
	}
	body, err := e.ansibleDoc(ctx, *remote, ansible.VersionPath(ns, name, version)[1:])
	if err != nil {
		return nil, fmt.Errorf("member %q: %v", owner, err)
	}
	var detail map[string]any
	if err := json.Unmarshal(body, &detail); err != nil {
		return nil, fmt.Errorf("member %q version detail: %v", owner, err)
	}
	dl, _ := detail["download_url"].(string)
	u, err := url.Parse(dl)
	if err != nil {
		return nil, fmt.Errorf("member %q download_url %q: %v", owner, dl, err)
	}
	rest, ok := strings.CutPrefix(u.EscapedPath(), "/api/v1/"+string(remote.RepoType)+"/"+url.PathEscape(owner)+"/")
	if !ok || u.Scheme != "" || u.Host != "" {
		return nil, fmt.Errorf("member %q download_url %q is not served by the member", owner, dl)
	}
	base := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/virtual/" + url.PathEscape(virt.Name)
	detail["download_url"] = base + "/" + url.PathEscape(owner) + "/" + rest
	detail["href"] = base + ansible.VersionPath(ns, name, version)
	return json.Marshal(detail)
}

// ansibleMember reads every version a member lists for ns.name, following
// the member's pagination for remotes.
func (e *Engine) ansibleMember(ctx context.Context, member, ns, name string) (*AnsibleMember, error) {
	remote, err := e.getRemote(ctx, member)
	if err != nil {
		return nil, fmt.Errorf("remote %q: %w", member, err)
	}
	m := &AnsibleMember{RemoteName: member}
	list := ansible.CollectionPath(ns, name)[1:] + "versions/"
	for page := 0; page < maxAnsibleMemberPages; page++ {
		// Locals list every version at once; remote pages match the keys the
		// remote caches client requests under.
		path := list
		if remote.RepoType != models.RepoTypeLocal {
			path += "?limit=" + strconv.Itoa(ansibleMemberPage)
			if page > 0 {
				path += "&offset=" + strconv.Itoa(len(m.Refs))
			}
		}
		body, err := e.ansibleDoc(ctx, *remote, path)
		var pe *proxy.ProxyError
		if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
			return m, nil
		}
		if err != nil {
			return nil, err
		}
		var doc struct {
			Meta struct{ Count int }
			Data []ansible.VersionRef
		}
		if err := json.Unmarshal(body, &doc); err != nil {
			return nil, fmt.Errorf("%s/%s: %v", member, path, err)
		}
		m.Found = true
		m.Refs = append(m.Refs, doc.Data...)
		if remote.RepoType == models.RepoTypeLocal || len(m.Refs) >= doc.Meta.Count {
			return m, nil
		}
		if len(doc.Data) == 0 {
			return nil, fmt.Errorf("%s/%s: empty page after %d of %d versions", member, path, len(m.Refs), doc.Meta.Count)
		}
	}
	return nil, fmt.Errorf("%s: more than %d pages of %s.%s versions", member, maxAnsibleMemberPages, ns, name)
}

// ansibleMemberDoc reads a Galaxy API document from a member.
func (e *Engine) ansibleMemberDoc(ctx context.Context, remote models.Remote, path string) ([]byte, error) {
	if remote.RepoType != models.RepoTypeLocal {
		return e.fetchMemberPath(ctx, remote, path)
	}
	if e.AnsibleLocal == nil {
		return nil, errors.New("local ansible indexes are not wired")
	}
	return e.AnsibleLocal(ctx, remote.Name, path)
}