Add alpine virtual repositories (#164)

Virtual repos cover rpm, deb, npm and others but not alpine, so a local alpine repo and github_alpine remotes cannot be served as one apk repository.

- merge members' `<dir>/APKINDEX.tar.gz` per directory; member order wins duplicate (P, V, A); a member 404 is skipped
- 302 `<dir>/<P>-<V>.apk` downloads to the member whose index listed the package
- serve the merged index unsigned, like local and github_alpine indexes
- cache merges per directory on the shared virtual cache; member errors fail with 502
- add docs/alpine/virtual.md, unit tests and an apk e2e with abuild-built packages

Requires unkin/artifactapi#163.

Reviewed-on: #164
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

internal/virtual/alpine_merger.go

@@ -0,0 +1,90 @@
+package virtual
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+)
+
+// AlpineMember is one member's decompressed APKINDEX for a directory; Index is
+// nil when the member does not serve that directory.
+type AlpineMember struct {
+	RemoteName string
+	Index      []byte
+}
+
+// AlpineRepo is a merged directory: its APKINDEX.tar.gz plus, per
+// "<name>-<version>.apk" apk will request, the member that owns it.
+type AlpineRepo struct {
+	Index  []byte
+	Owners map[string][]byte
+}
+
+func (r *AlpineRepo) index() []byte            { return r.Index }
+func (r *AlpineRepo) files() map[string][]byte { return r.Owners }
+
+// MergeAlpine merges members' APKINDEX records. Members are in priority order:
+// the first to carry a (P, V, A) wins. The index is unsigned, like the local
+// and github_alpine indexes it merges.
+func MergeAlpine(members []AlpineMember) *AlpineRepo {
+	repo := &AlpineRepo{Owners: map[string][]byte{}}
+	seen := map[string]bool{}
+	var b bytes.Buffer
+	for _, m := range members {
+		for _, rec := range splitStanzas(m.Index) {
+			p, v, a := stanzaField(rec, "P"), stanzaField(rec, "V"), stanzaField(rec, "A")
+			key := p + "\x00" + v + "\x00" + a
+			if p == "" || v == "" || seen[key] {
+				continue
+			}
+			seen[key] = true
+			if _, ok := repo.Owners[p+"-"+v+".apk"]; !ok {
+				repo.Owners[p+"-"+v+".apk"] = []byte(m.RemoteName)
+			}
+			b.WriteString(rec)
+			b.WriteString("\n\n")
+		}
+	}
+	var t bytes.Buffer
+	tw := tar.NewWriter(&t)
+	_ = tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(b.Len()), Typeflag: tar.TypeReg, ModTime: time.Unix(0, 0)})
+	_, _ = tw.Write(b.Bytes())
+	_ = tw.Close()
+	repo.Index = gzipDeterministic(t.Bytes())
+	return repo
+}
+
+// apkIndexText extracts the APKINDEX entry from an APKINDEX.tar.gz, skipping a
+// leading signature stream if present.
+func apkIndexText(body []byte) ([]byte, error) {
+	zr, err := gzip.NewReader(bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	tr := tar.NewReader(zr)
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			return nil, fmt.Errorf("no APKINDEX entry")
+		}
+		if err != nil {
+			return nil, err
+		}
+		if hdr.Name == "APKINDEX" {
+			return io.ReadAll(tr)
+		}
+	}
+}
+
+// alpinePath splits a request into its directory and file, or ok is false for
+// anything but an APKINDEX.tar.gz or .apk.
+func alpinePath(p string) (dir, file string, ok bool) {
+	i := strings.LastIndex(p, "/")
+	dir, file = p[:i+1], p[i+1:]
+	return dir, file, dir != "" && (file == "APKINDEX.tar.gz" || strings.HasSuffix(file, ".apk"))
+}

internal/virtual/alpine_merger_test.go

@@ -0,0 +1,140 @@
+package virtual
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"errors"
+	"strings"
+	"testing"
+
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func apkRec(p, v, a string) string {
+	return "C:Q1" + p + v + "\nP:" + p + "\nV:" + v + "\nA:" + a + "\nS:10\n"
+}
+
+func apkTarGz(t *testing.T, name string, body []byte, closeTar bool) []byte {
+	t.Helper()
+	var tb bytes.Buffer
+	tw := tar.NewWriter(&tb)
+	if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil {
+		t.Fatal(err)
+	}
+	tw.Write(body)
+	if closeTar {
+		tw.Close()
+	} else {
+		tw.Flush()
+	}
+	var gb bytes.Buffer
+	gz := gzip.NewWriter(&gb)
+	gz.Write(tb.Bytes())
+	gz.Close()
+	return gb.Bytes()
+}
+
+func TestMergeAlpine(t *testing.T) {
+	repo := MergeAlpine([]AlpineMember{
+		{RemoteName: "a", Index: []byte(apkRec("foo", "1.0-r0", "x86_64") + "\n" + apkRec("bar", "1-r0", "noarch") + "\nC:Q1x\nV:9\n")},
+		{RemoteName: "b"},
+		{RemoteName: "c", Index: []byte(apkRec("foo", "1.0-r0", "x86_64") + "\n" + apkRec("foo", "2.0-r0", "x86_64") + "\n" + apkRec("bar", "1-r0", "x86_64"))},
+	})
+	idx, err := apkIndexText(repo.Index)
+	if err != nil {
+		t.Fatal(err)
+	}
+	recs := splitStanzas(idx)
+	if len(recs) != 4 || strings.Count(string(idx), "P:foo\nV:1.0-r0\n") != 1 {
+		t.Fatalf("want foo 1.0 once, foo 2.0 and bar for both arches, record without P dropped:\n%s", idx)
+	}
+	for file, want := range map[string]string{"foo-1.0-r0.apk": "a", "foo-2.0-r0.apk": "c", "bar-1-r0.apk": "a"} {
+		if got := string(repo.Owners[file]); got != want {
+			t.Errorf("%s owned by %q, want %q", file, got, want)
+		}
+	}
+	if again := MergeAlpine([]AlpineMember{{RemoteName: "a", Index: idx}}); !bytes.Equal(again.Index, MergeAlpine([]AlpineMember{{RemoteName: "a", Index: idx}}).Index) {
+		t.Fatal("merged index must be byte-stable across merges")
+	}
+}
+
+func TestApkIndexTextSkipsSignature(t *testing.T) {
+	signed := append(apkTarGz(t, ".SIGN.RSA.alpine.rsa.pub", []byte("sig"), false), apkTarGz(t, "APKINDEX", []byte(apkRec("foo", "1", "x86_64")), true)...)
+	idx, err := apkIndexText(signed)
+	if err != nil || !strings.Contains(string(idx), "P:foo\n") {
+		t.Fatalf("signed index: %v\n%s", err, idx)
+	}
+	if _, err := apkIndexText(apkTarGz(t, "DESCRIPTION", []byte("x"), true)); err == nil {
+		t.Fatal("index without APKINDEX entry should fail")
+	}
+}
+
+func alpineEngine(members map[string]string) *Engine {
+	e := fakeEngine(nil)
+	e.apkMember = func(_ context.Context, name, dir string) (*AlpineMember, error) {
+		idx, ok := members[name]
+		if !ok {
+			return nil, errors.New("upstream down")
+		}
+		if dir != "x86_64/" {
+			return &AlpineMember{RemoteName: name}, nil
+		}
+		return &AlpineMember{RemoteName: name, Index: []byte(idx)}, nil
+	}
+	return e
+}
+
+func alpineVirt(members ...string) models.Virtual {
+	return models.Virtual{Name: "v", PackageType: models.PackageAlpine, Members: members}
+}
+
+func TestFetchAlpineIndexAndRedirect(t *testing.T) {
+	e := alpineEngine(map[string]string{"loc": apkRec("foo", "1-r0", "x86_64"), "gh": apkRec("foo", "1-r0", "x86_64") + "\n" + apkRec("tool", "2-r0", "x86_64")})
+	virt := alpineVirt("loc", "gh")
+
+	body, ct, err := e.Fetch(context.Background(), virt, "./x86_64/APKINDEX.tar.gz", "")
+	if err != nil || ct != "application/gzip" {
+		t.Fatalf("fetch index: %v %q", err, ct)
+	}
+	if idx, _ := apkIndexText(body); strings.Count(string(idx), "P:") != 2 {
+		t.Fatalf("merged index:\n%s", idx)
+	}
+	for _, tc := range []struct {
+		path, want string
+		ok         bool
+	}{
+		{path: "x86_64/foo-1-r0.apk", ok: true, want: "http://h/api/v1/remote/loc/x86_64/foo-1-r0.apk"},
+		{path: "./x86_64/tool-2-r0.apk", ok: true, want: "http://h/api/v1/remote/gh/x86_64/tool-2-r0.apk"},
+		{path: "x86_64/../x86_64/tool-2-r0.apk", ok: true, want: "http://h/api/v1/remote/gh/x86_64/tool-2-r0.apk"},
+		{path: "x86_64/missing-1-r0.apk"},
+		{path: "aarch64/foo-1-r0.apk"},
+		{path: "x86_64/APKINDEX.tar.gz"},
+		{path: "foo-1-r0.apk"},
+	} {
+		got, ok, err := e.MemberRedirect(context.Background(), virt, tc.path, "", "http://h/")
+		if got != tc.want || ok != tc.ok || err != nil {
+			t.Errorf("%s: got (%q, %v, %v), want (%q, %v)", tc.path, got, ok, err, tc.want, tc.ok)
+		}
+	}
+	for _, p := range []string{"x86_64/missing-1-r0.apk", "x86_64/DESCRIPTION", "APKINDEX.tar.gz"} {
+		if _, _, err := e.Fetch(context.Background(), virt, p, ""); !errors.Is(err, ErrNotFound) {
+			t.Errorf("%s: want not found, got %v", p, err)
+		}
+	}
+}
+
+func TestFetchAlpineFailsClosed(t *testing.T) {
+	e := alpineEngine(map[string]string{"a": apkRec("foo", "1-r0", "x86_64")})
+	virt := alpineVirt("a", "down")
+	if _, _, err := e.Fetch(context.Background(), virt, "x86_64/APKINDEX.tar.gz", ""); err == nil || errors.Is(err, ErrNotFound) {
+		t.Fatalf("index with a member down: want upstream error, got %v", err)
+	}
+	if _, ok, err := e.MemberRedirect(context.Background(), virt, "x86_64/foo-1-r0.apk", "", "http://h/"); ok || err != nil {
+		t.Fatalf("redirect with a member down must defer to Fetch: ok %v err %v", ok, err)
+	}
+	if _, _, err := e.Fetch(context.Background(), virt, "x86_64/foo-1-r0.apk", ""); err == nil || errors.Is(err, ErrNotFound) {
+		t.Fatalf("download with a member down: want upstream error, got %v", err)
+	}
+}

internal/virtual/engine.go

@@ -28,6 +28,7 @@ type Engine struct {
 	getRemote   func(context.Context, string) (*models.Remote, error)
 	rpmMember   func(context.Context, string) (*RPMMember, error)
 	debMember   func(ctx context.Context, name, base string) (*DebMember, error)
+	apkMember   func(ctx context.Context, name, dir string) (*AlpineMember, error)
 	npmMember   func(ctx context.Context, name, pkg string) (*NPMMember, error)
 
 	// NPMLocal returns a local npm repo's packument with tarball paths relative
@@ -76,13 +77,15 @@ type repoGen struct {
 
 const rpmMergeTTL = 60 * time.Second
 
-// maxDebSuites bounds the client-chosen suites cached per deb virtual.
+// maxDebSuites bounds the client-chosen deb suites or alpine directories
+// cached per virtual.
 const maxDebSuites = 32
 
 func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
 	e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL}
 	e.rpmMember = e.fetchRPMMember
 	e.debMember = e.fetchDebMember
+	e.apkMember = e.fetchAlpineMember
 	e.npmMember = e.fetchNPMMember
 	e.gemsMember = e.fetchGemsMember
 	e.ansibleDoc = e.ansibleMemberDoc
@@ -97,6 +100,8 @@ func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, pr
 		return e.fetchRPM(ctx, virt, path)
 	case models.PackageDeb:
 		return e.fetchDeb(ctx, virt, path)
+	case models.PackageAlpine:
+		return e.fetchAlpine(ctx, virt, path)
 	}
 
 	merger, err := GetMerger(virt.PackageType)
@@ -216,8 +221,8 @@ var (
 )
 
 // MemberRedirect maps an rpm, deb, npm or ansible virtual package path
-// (<member>/<href>, as written by MergeRPM/MergeDeb/MergeNPM/AnsibleVersion)
-// to the owning member's route. ok is false for any other path; ErrBadPath
+// (<member>/<href>, as written by MergeRPM/MergeDeb/MergeNPM/AnsibleVersion),
+// or an alpine <dir>/<name>-<version>.apk, to the owning member's route. ok is false for any other path; ErrBadPath
 // means the path is absolute or escapes the member.
 func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) {
 	switch virt.PackageType {
@@ -231,6 +236,8 @@ func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path,
 		if strings.HasPrefix(path, "dists/") {
 			return "", false, nil
 		}
+	case models.PackageAlpine:
+		return e.alpineRedirect(ctx, virt, path, rawQuery, proxyBaseURL)
 	case models.PackageNPM, models.PackageAnsible:
 	default:
 		return "", false, nil
@@ -445,6 +452,78 @@ func (e *Engine) fetchDebMember(ctx context.Context, name, base string) (*DebMem
 	return m, nil
 }
 
+func (e *Engine) mergedAlpine(ctx context.Context, virt models.Virtual, dir string) (mergedRepo, error) {
+	repo, err := e.merged(ctx, "alpine\x00"+virt.Name+"\x00"+dir, func(ctx context.Context) (mergedRepo, error) {
+		members, err := fetchMembers(virt, func(name string) (*AlpineMember, error) { return e.apkMember(ctx, name, dir) })
+		if err != nil {
+			return nil, err
+		}
+		return MergeAlpine(members), nil
+	})
+	if err == nil {
+		e.evictLRU("alpine\x00"+virt.Name+"\x00", maxDebSuites)
+	}
+	return repo, err
+}
+
+func (e *Engine) fetchAlpine(ctx context.Context, virt models.Virtual, reqPath string) ([]byte, string, error) {
+	dir, file, ok := alpinePath(strings.TrimPrefix(path.Clean("/"+reqPath), "/"))
+	if !ok {
+		return nil, "", ErrNotFound
+	}
+	repo, err := e.mergedAlpine(ctx, virt, dir)
+	if err != nil {
+		return nil, "", err
+	}
+	if file != "APKINDEX.tar.gz" {
+		return nil, "", ErrNotFound
+	}
+	return repo.index(), "application/gzip", nil
+}
+
+// alpineRedirect finds the member whose index lists the requested .apk; apk
+// builds the download path itself, so the index cannot carry a member prefix.
+// A failed merge returns ok=false so Fetch re-merges and fails with 502.
+func (e *Engine) alpineRedirect(ctx context.Context, virt models.Virtual, reqPath, rawQuery, proxyBaseURL string) (string, bool, error) {
+	dir, file, ok := alpinePath(strings.TrimPrefix(path.Clean("/"+reqPath), "/"))
+	if !ok || file == "APKINDEX.tar.gz" {
+		return "", false, nil
+	}
+	owner, ok := e.cachedFile("alpine\x00"+virt.Name+"\x00"+dir, file)
+	if !ok {
+		repo, err := e.mergedAlpine(ctx, virt, dir)
+		if err != nil {
+			return "", false, nil
+		}
+		if owner, ok = repo.files()[file]; !ok {
+			return "", false, nil
+		}
+	}
+	return e.memberLocation(ctx, string(owner), dir+file, rawQuery, proxyBaseURL)
+}
+
+// fetchAlpineMember reads a member's APKINDEX under dir; a member without that
+// directory contributes nothing.
+func (e *Engine) fetchAlpineMember(ctx context.Context, name, dir string) (*AlpineMember, error) {
+	remote, err := e.getRemote(ctx, name)
+	if err != nil {
+		return nil, fmt.Errorf("remote %q: %w", name, err)
+	}
+	raw, err := e.fetchMemberPath(ctx, *remote, dir+"APKINDEX.tar.gz")
+	var pe *proxy.ProxyError
+	if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
+		return &AlpineMember{RemoteName: name}, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	idx, err := apkIndexText(raw)
+	if err != nil {
+		return nil, fmt.Errorf("%s/%sAPKINDEX.tar.gz: %w", name, dir, err)
+	}
+	return &AlpineMember{RemoteName: name, Index: idx}, nil
+}
+
 // evictLRU drops the least recently used merges under prefix beyond keep.
 func (e *Engine) evictLRU(prefix string, keep int) {
 	e.mu.Lock()