From f88678360f48d7ac54971e47dfbf03e82ac00613 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 24 Oct 2026 17:30:40 +1100 Subject: [PATCH 1/2] Drop local files and metadata with their remote --- e2e-docker/local_test.go | 54 +++++++++++++++++------------- internal/database/database_test.go | 28 ++++++++++++++++ internal/database/remotes.go | 15 +++++++-- 3 files changed, 71 insertions(+), 26 deletions(-) diff --git a/e2e-docker/local_test.go b/e2e-docker/local_test.go index 562fa68..6d90eed 100644 --- a/e2e-docker/local_test.go +++ b/e2e-docker/local_test.go @@ -150,31 +150,15 @@ func TestLocalAlpineIndex(t *testing.T) { apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64") uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive") - // The index is generated asynchronously after upload; poll for it. - resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second) - if resp.StatusCode != http.StatusOK { - t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body) - } - - zr, err := gzip.NewReader(bytes.NewReader(body)) - if err != nil { - t.Fatalf("APKINDEX not gzip: %v", err) - } - tarBytes, _ := io.ReadAll(zr) - tr := tar.NewReader(bytes.NewReader(tarBytes)) + // The index is served (empty) at once but lists the package only once the + // upload's metadata is parsed asynchronously; poll until it does. var index string - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - t.Fatalf("APKINDEX not tar: %v", err) - } - if hdr.Name == "APKINDEX" { - b, _ := io.ReadAll(tr) - index = string(b) + for deadline := time.Now().Add(15 * time.Second); !strings.Contains(index, "P:e2e-testpkg") && time.Now().Before(deadline); time.Sleep(250 * time.Millisecond) { + resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), nil, "") + if resp.StatusCode != http.StatusOK { + t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body) } + index = apkIndexText(t, body) } for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} { if !strings.Contains(index, want) { @@ -183,7 +167,7 @@ func TestLocalAlpineIndex(t *testing.T) { } // The .apk downloads back byte-identical from its arch path. - resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "") + resp, body := doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "") if resp.StatusCode != http.StatusOK { t.Fatalf("download apk: status %d: %s", resp.StatusCode, body) } @@ -191,3 +175,25 @@ func TestLocalAlpineIndex(t *testing.T) { t.Fatalf("apk content mismatch") } } + +func apkIndexText(t *testing.T, body []byte) string { + t.Helper() + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatalf("APKINDEX not gzip: %v", err) + } + tr := tar.NewReader(zr) + for { + hdr, err := tr.Next() + if err == io.EOF { + return "" + } + if err != nil { + t.Fatalf("APKINDEX not tar: %v", err) + } + if hdr.Name == "APKINDEX" { + b, _ := io.ReadAll(tr) + return string(b) + } + } +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 2f11049..ef0e8a9 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -307,6 +307,34 @@ func TestLocalFiles(t *testing.T) { } } +// A repo recreated under a deleted repo's name must not inherit its files or +// metadata, or re-uploading the same path 409s. +func TestDeleteRemoteDropsLocalFiles(t *testing.T) { + requireDB(t) + seedRemote(t, "r-recreate") + hash := seedBlob(t, "recreatehash") + if err := testDB.CreateLocalFile(ctx(), "r-recreate", "x86_64/a-1.0-r0.apk", hash); err != nil { + t.Fatal(err) + } + if err := testDB.InsertAlpineMetadata(ctx(), &provider.AlpineMetadata{RepoName: "r-recreate", FilePath: "x86_64/a-1.0-r0.apk", ContentHash: hash, Name: "a", Version: "1.0-r0", Arch: "x86_64"}); err != nil { + t.Fatal(err) + } + if err := testDB.DeleteRemote(ctx(), "r-recreate"); err != nil { + t.Fatal(err) + } + seedRemote(t, "r-recreate") + defer testDB.DeleteRemote(ctx(), "r-recreate") + if f, err := testDB.GetLocalFile(ctx(), "r-recreate", "x86_64/a-1.0-r0.apk"); err != nil || f != nil { + t.Fatalf("recreated repo inherited local file: %v %v", f, err) + } + if rows, err := testDB.ListAlpineMetadataEntries(ctx(), "r-recreate"); err != nil || len(rows) != 0 { + t.Fatalf("recreated repo inherited alpine metadata: %v %v", rows, err) + } + if err := testDB.CreateLocalFile(ctx(), "r-recreate", "x86_64/a-1.0-r0.apk", hash); err != nil { + t.Fatalf("re-upload after recreate: %v", err) + } +} + func TestVirtualsCRUD(t *testing.T) { requireDB(t) if err := testDB.CreateVirtual(ctx(), &models.Virtual{ diff --git a/internal/database/remotes.go b/internal/database/remotes.go index 76afa06..44b2eb5 100644 --- a/internal/database/remotes.go +++ b/internal/database/remotes.go @@ -3,6 +3,8 @@ package database import ( "context" + "github.com/jackc/pgx/v5" + "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -116,7 +118,16 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error { return err } +// DeleteRemote drops a remote with its local files and package metadata, so a +// repo recreated under the same name starts empty. func (db *DB) DeleteRemote(ctx context.Context, name string) error { - _, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name) - return err + return pgx.BeginFunc(ctx, db.Pool, func(tx pgx.Tx) error { + for _, table := range []string{"local_files", "rpm_metadata", "deb_metadata", "alpine_metadata", "puppet_metadata", "helm_metadata"} { + if _, err := tx.Exec(ctx, `DELETE FROM `+table+` WHERE repo_name = $1`, name); err != nil { + return err + } + } + _, err := tx.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name) + return err + }) } -- 2.47.3 From 501beaffa195b5811730d3db475e10db15e88d2b Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 24 Oct 2026 17:41:58 +1100 Subject: [PATCH 2/2] Add alpine virtual repositories --- README.md | 2 +- docs/alpine/virtual.md | 30 ++++++ e2e-docker/alpine_virtual_test.go | 105 +++++++++++++++++++ internal/virtual/alpine_merger.go | 90 ++++++++++++++++ internal/virtual/alpine_merger_test.go | 140 +++++++++++++++++++++++++ internal/virtual/engine.go | 85 ++++++++++++++- 6 files changed, 448 insertions(+), 4 deletions(-) create mode 100644 docs/alpine/virtual.md create mode 100644 e2e-docker/alpine_virtual_test.go create mode 100644 internal/virtual/alpine_merger.go create mode 100644 internal/virtual/alpine_merger_test.go diff --git a/README.md b/README.md index 616dbb5..7ca3d60 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Classification precedence, TTLs and eviction for every remote: [docs/common/cach Each type's paths, Terraform examples and local/virtual behaviour are in `docs//`: -- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md) +- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md), [virtual](docs/alpine/virtual.md) - `ansible`: [remote](docs/ansible/remote.md), [local](docs/ansible/local.md), [virtual](docs/ansible/virtual.md) - `cargo`: [remote](docs/cargo/remote.md) - `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md), [virtual](docs/deb/virtual.md) diff --git a/docs/alpine/virtual.md b/docs/alpine/virtual.md new file mode 100644 index 0000000..743b569 --- /dev/null +++ b/docs/alpine/virtual.md @@ -0,0 +1,30 @@ +# `alpine` virtual + +An alpine virtual serves one apk repository over local and remote alpine +members at `/api/v1/virtual//`. + +- `/APKINDEX.tar.gz` merges each member's index for the same `` + (e.g. `x86_64/`); on a duplicate (P, V, A) the earlier member wins. A member + that 404s for that directory is skipped. +- apk builds download paths itself (`/

-.apk`), so the virtual 302s + each download to the member whose index listed it. +- The merged index is unsigned, like local and `github_alpine` indexes: clients + need `--allow-untrusted` (packages keep their own signatures). +- Any other member error fails the request with 502. Merges are cached for 60s + per directory, up to 32 directories per virtual. + +```hcl +resource "artifactapi_virtual" "alpine" { + name = "alpine" + package_type = "alpine" + members = [artifactapi_local_alpine.internal.name, artifactapi_remote_github_alpine.tool.name] +} +``` + +```sh +echo https://artifactapi.example.com/api/v1/virtual/alpine >> /etc/apk/repositories +apk update --allow-untrusted +apk add --allow-untrusted mytool +``` + +Upload to the local member directly; the virtual is read-only. diff --git a/e2e-docker/alpine_virtual_test.go b/e2e-docker/alpine_virtual_test.go new file mode 100644 index 0000000..f4d6410 --- /dev/null +++ b/e2e-docker/alpine_virtual_test.go @@ -0,0 +1,105 @@ +//go:build dockere2e + +package e2edocker + +import ( + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// abuildScript builds a real, abuild-signed $N-$V-r0.apk and writes it to stdout. +const abuildScript = `set -eu +apk add -q abuild >&2 +adduser -D b && addgroup b abuild +su b -c 'set -eu; abuild-keygen -an >&2; mkdir -p ~/p && cd ~/p +printf "%s\n" "pkgname=$N" "pkgver=$V" pkgrel=0 "pkgdesc=\"e2e\"" "url=\"https://example.com\"" "arch=\"x86_64\"" "license=\"MIT\"" "options=\"!check\"" \ + "package() { mkdir -p \"\$pkgdir/usr/share/$N\"; echo ok > \"\$pkgdir/usr/share/$N/ok\"; }" > APKBUILD +abuild -d >&2 || true' +cat /home/b/packages/*/x86_64/$N-$V-r0.apk` + +func buildApk(t *testing.T, name, version string) []byte { + t.Helper() + cmd := exec.Command("docker", "run", "--rm", "-e", "N="+name, "-e", "V="+version, "alpine:3.20", "sh", "-c", abuildScript) + var stderr strings.Builder + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil || len(out) == 0 { + t.Fatalf("abuild %s-%s: %v\n%s", name, version, err, stderr.String()) + } + return out +} + +// TestVirtualAlpineMerge merges two local alpine repos carrying the same +// abuild-built package: the first member wins the duplicate and its download, +// and a real apk installs packages from both members through the virtual. +func TestVirtualAlpineMerge(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := os.Getenv("ARTIFACTAPI_INTERNAL") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; skipping real apk") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + one, two := buildApk(t, "vapk-one", "1.0"), buildApk(t, "vapk-two", "2.0") + + createRepo(t, `{"name":"vapk-a","package_type":"alpine","repo_type":"local"}`) + createRepo(t, `{"name":"vapk-b","package_type":"alpine","repo_type":"local"}`) + defer deleteRepo(t, "vapk-a") + defer deleteRepo(t, "vapk-b") + + const apkType = "application/vnd.android.package-archive" + uploadFile(t, "vapk-a", "x86_64/vapk-one-1.0-r0.apk", one, apkType) + uploadFile(t, "vapk-b", "x86_64/vapk-one-1.0-r0.apk", one, apkType) + uploadFile(t, "vapk-b", "x86_64/vapk-two-2.0-r0.apk", two, apkType) + + // Local indexes list uploads once their metadata is parsed asynchronously, + // and the virtual caches its merge, so merge only once both members list all. + for _, m := range []struct{ repo, pkg string }{{"vapk-a", "P:vapk-one\n"}, {"vapk-b", "P:vapk-two\n"}} { + var idx string + for deadline := time.Now().Add(15 * time.Second); !strings.Contains(idx, m.pkg) && time.Now().Before(deadline); time.Sleep(250 * time.Millisecond) { + _, body := doRequest(t, http.MethodGet, api("/api/v1/local/"+m.repo+"/x86_64/APKINDEX.tar.gz"), nil, "") + idx = apkIndexText(t, body) + } + } + + createVirtual(t, `{"name":"vapk","package_type":"alpine","members":["vapk-a","vapk-b"]}`) + defer deleteVirtual(t, "vapk") + + resp, body := doRequest(t, http.MethodGet, api("/api/v1/virtual/vapk/x86_64/APKINDEX.tar.gz"), nil, "") + if resp.StatusCode != http.StatusOK { + t.Fatalf("virtual APKINDEX: status %d: %s", resp.StatusCode, body) + } + index := apkIndexText(t, body) + if strings.Count(index, "P:vapk-one\n") != 1 || !strings.Contains(index, "P:vapk-two\n") { + t.Fatalf("merged APKINDEX should hold vapk-one once and vapk-two:\n%s", index) + } + + noRedirect := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + for pkg, member := range map[string]string{"vapk-one-1.0-r0.apk": "vapk-a", "vapk-two-2.0-r0.apk": "vapk-b"} { + resp, err := noRedirect.Get(api("/api/v1/virtual/vapk/x86_64/" + pkg)) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if loc := resp.Header.Get("Location"); resp.StatusCode != http.StatusFound || !strings.HasSuffix(loc, "/api/v1/local/"+member+"/x86_64/"+pkg) { + t.Fatalf("%s: want 302 to %s, got %d %q", pkg, member, resp.StatusCode, loc) + } + } + + script := "set -eu; " + + "echo \"$URL\" > /etc/apk/repositories; " + + "apk update --allow-untrusted; " + + "apk add --allow-untrusted vapk-one vapk-two; " + + "test -f /usr/share/vapk-one/ok && test -f /usr/share/vapk-two/ok" + out, err := exec.Command("docker", "run", "--rm", "--network", network, + "-e", "URL="+strings.TrimRight(internal, "/")+"/api/v1/virtual/vapk", + "alpine:3.20", "sh", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("real apk against alpine virtual failed: %v\n%s", err, out) + } +} diff --git a/internal/virtual/alpine_merger.go b/internal/virtual/alpine_merger.go new file mode 100644 index 0000000..b02a5cc --- /dev/null +++ b/internal/virtual/alpine_merger.go @@ -0,0 +1,90 @@ +package virtual + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "strings" + "time" +) + +// AlpineMember is one member's decompressed APKINDEX for a directory; Index is +// nil when the member does not serve that directory. +type AlpineMember struct { + RemoteName string + Index []byte +} + +// AlpineRepo is a merged directory: its APKINDEX.tar.gz plus, per +// "-.apk" apk will request, the member that owns it. +type AlpineRepo struct { + Index []byte + Owners map[string][]byte +} + +func (r *AlpineRepo) index() []byte { return r.Index } +func (r *AlpineRepo) files() map[string][]byte { return r.Owners } + +// MergeAlpine merges members' APKINDEX records. Members are in priority order: +// the first to carry a (P, V, A) wins. The index is unsigned, like the local +// and github_alpine indexes it merges. +func MergeAlpine(members []AlpineMember) *AlpineRepo { + repo := &AlpineRepo{Owners: map[string][]byte{}} + seen := map[string]bool{} + var b bytes.Buffer + for _, m := range members { + for _, rec := range splitStanzas(m.Index) { + p, v, a := stanzaField(rec, "P"), stanzaField(rec, "V"), stanzaField(rec, "A") + key := p + "\x00" + v + "\x00" + a + if p == "" || v == "" || seen[key] { + continue + } + seen[key] = true + if _, ok := repo.Owners[p+"-"+v+".apk"]; !ok { + repo.Owners[p+"-"+v+".apk"] = []byte(m.RemoteName) + } + b.WriteString(rec) + b.WriteString("\n\n") + } + } + var t bytes.Buffer + tw := tar.NewWriter(&t) + _ = tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(b.Len()), Typeflag: tar.TypeReg, ModTime: time.Unix(0, 0)}) + _, _ = tw.Write(b.Bytes()) + _ = tw.Close() + repo.Index = gzipDeterministic(t.Bytes()) + return repo +} + +// apkIndexText extracts the APKINDEX entry from an APKINDEX.tar.gz, skipping a +// leading signature stream if present. +func apkIndexText(body []byte) ([]byte, error) { + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + tr := tar.NewReader(zr) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf("no APKINDEX entry") + } + if err != nil { + return nil, err + } + if hdr.Name == "APKINDEX" { + return io.ReadAll(tr) + } + } +} + +// alpinePath splits a request into its directory and file, or ok is false for +// anything but an APKINDEX.tar.gz or .apk. +func alpinePath(p string) (dir, file string, ok bool) { + i := strings.LastIndex(p, "/") + dir, file = p[:i+1], p[i+1:] + return dir, file, dir != "" && (file == "APKINDEX.tar.gz" || strings.HasSuffix(file, ".apk")) +} diff --git a/internal/virtual/alpine_merger_test.go b/internal/virtual/alpine_merger_test.go new file mode 100644 index 0000000..933cca0 --- /dev/null +++ b/internal/virtual/alpine_merger_test.go @@ -0,0 +1,140 @@ +package virtual + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func apkRec(p, v, a string) string { + return "C:Q1" + p + v + "\nP:" + p + "\nV:" + v + "\nA:" + a + "\nS:10\n" +} + +func apkTarGz(t *testing.T, name string, body []byte, closeTar bool) []byte { + t.Helper() + var tb bytes.Buffer + tw := tar.NewWriter(&tb) + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + tw.Write(body) + if closeTar { + tw.Close() + } else { + tw.Flush() + } + var gb bytes.Buffer + gz := gzip.NewWriter(&gb) + gz.Write(tb.Bytes()) + gz.Close() + return gb.Bytes() +} + +func TestMergeAlpine(t *testing.T) { + repo := MergeAlpine([]AlpineMember{ + {RemoteName: "a", Index: []byte(apkRec("foo", "1.0-r0", "x86_64") + "\n" + apkRec("bar", "1-r0", "noarch") + "\nC:Q1x\nV:9\n")}, + {RemoteName: "b"}, + {RemoteName: "c", Index: []byte(apkRec("foo", "1.0-r0", "x86_64") + "\n" + apkRec("foo", "2.0-r0", "x86_64") + "\n" + apkRec("bar", "1-r0", "x86_64"))}, + }) + idx, err := apkIndexText(repo.Index) + if err != nil { + t.Fatal(err) + } + recs := splitStanzas(idx) + if len(recs) != 4 || strings.Count(string(idx), "P:foo\nV:1.0-r0\n") != 1 { + t.Fatalf("want foo 1.0 once, foo 2.0 and bar for both arches, record without P dropped:\n%s", idx) + } + for file, want := range map[string]string{"foo-1.0-r0.apk": "a", "foo-2.0-r0.apk": "c", "bar-1-r0.apk": "a"} { + if got := string(repo.Owners[file]); got != want { + t.Errorf("%s owned by %q, want %q", file, got, want) + } + } + if again := MergeAlpine([]AlpineMember{{RemoteName: "a", Index: idx}}); !bytes.Equal(again.Index, MergeAlpine([]AlpineMember{{RemoteName: "a", Index: idx}}).Index) { + t.Fatal("merged index must be byte-stable across merges") + } +} + +func TestApkIndexTextSkipsSignature(t *testing.T) { + signed := append(apkTarGz(t, ".SIGN.RSA.alpine.rsa.pub", []byte("sig"), false), apkTarGz(t, "APKINDEX", []byte(apkRec("foo", "1", "x86_64")), true)...) + idx, err := apkIndexText(signed) + if err != nil || !strings.Contains(string(idx), "P:foo\n") { + t.Fatalf("signed index: %v\n%s", err, idx) + } + if _, err := apkIndexText(apkTarGz(t, "DESCRIPTION", []byte("x"), true)); err == nil { + t.Fatal("index without APKINDEX entry should fail") + } +} + +func alpineEngine(members map[string]string) *Engine { + e := fakeEngine(nil) + e.apkMember = func(_ context.Context, name, dir string) (*AlpineMember, error) { + idx, ok := members[name] + if !ok { + return nil, errors.New("upstream down") + } + if dir != "x86_64/" { + return &AlpineMember{RemoteName: name}, nil + } + return &AlpineMember{RemoteName: name, Index: []byte(idx)}, nil + } + return e +} + +func alpineVirt(members ...string) models.Virtual { + return models.Virtual{Name: "v", PackageType: models.PackageAlpine, Members: members} +} + +func TestFetchAlpineIndexAndRedirect(t *testing.T) { + e := alpineEngine(map[string]string{"loc": apkRec("foo", "1-r0", "x86_64"), "gh": apkRec("foo", "1-r0", "x86_64") + "\n" + apkRec("tool", "2-r0", "x86_64")}) + virt := alpineVirt("loc", "gh") + + body, ct, err := e.Fetch(context.Background(), virt, "./x86_64/APKINDEX.tar.gz", "") + if err != nil || ct != "application/gzip" { + t.Fatalf("fetch index: %v %q", err, ct) + } + if idx, _ := apkIndexText(body); strings.Count(string(idx), "P:") != 2 { + t.Fatalf("merged index:\n%s", idx) + } + for _, tc := range []struct { + path, want string + ok bool + }{ + {path: "x86_64/foo-1-r0.apk", ok: true, want: "http://h/api/v1/remote/loc/x86_64/foo-1-r0.apk"}, + {path: "./x86_64/tool-2-r0.apk", ok: true, want: "http://h/api/v1/remote/gh/x86_64/tool-2-r0.apk"}, + {path: "x86_64/../x86_64/tool-2-r0.apk", ok: true, want: "http://h/api/v1/remote/gh/x86_64/tool-2-r0.apk"}, + {path: "x86_64/missing-1-r0.apk"}, + {path: "aarch64/foo-1-r0.apk"}, + {path: "x86_64/APKINDEX.tar.gz"}, + {path: "foo-1-r0.apk"}, + } { + got, ok, err := e.MemberRedirect(context.Background(), virt, tc.path, "", "http://h/") + if got != tc.want || ok != tc.ok || err != nil { + t.Errorf("%s: got (%q, %v, %v), want (%q, %v)", tc.path, got, ok, err, tc.want, tc.ok) + } + } + for _, p := range []string{"x86_64/missing-1-r0.apk", "x86_64/DESCRIPTION", "APKINDEX.tar.gz"} { + if _, _, err := e.Fetch(context.Background(), virt, p, ""); !errors.Is(err, ErrNotFound) { + t.Errorf("%s: want not found, got %v", p, err) + } + } +} + +func TestFetchAlpineFailsClosed(t *testing.T) { + e := alpineEngine(map[string]string{"a": apkRec("foo", "1-r0", "x86_64")}) + virt := alpineVirt("a", "down") + if _, _, err := e.Fetch(context.Background(), virt, "x86_64/APKINDEX.tar.gz", ""); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("index with a member down: want upstream error, got %v", err) + } + if _, ok, err := e.MemberRedirect(context.Background(), virt, "x86_64/foo-1-r0.apk", "", "http://h/"); ok || err != nil { + t.Fatalf("redirect with a member down must defer to Fetch: ok %v err %v", ok, err) + } + if _, _, err := e.Fetch(context.Background(), virt, "x86_64/foo-1-r0.apk", ""); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("download with a member down: want upstream error, got %v", err) + } +} diff --git a/internal/virtual/engine.go b/internal/virtual/engine.go index c062bc6..9ab1905 100644 --- a/internal/virtual/engine.go +++ b/internal/virtual/engine.go @@ -28,6 +28,7 @@ type Engine struct { getRemote func(context.Context, string) (*models.Remote, error) rpmMember func(context.Context, string) (*RPMMember, error) debMember func(ctx context.Context, name, base string) (*DebMember, error) + apkMember func(ctx context.Context, name, dir string) (*AlpineMember, error) npmMember func(ctx context.Context, name, pkg string) (*NPMMember, error) // NPMLocal returns a local npm repo's packument with tarball paths relative @@ -76,13 +77,15 @@ type repoGen struct { const rpmMergeTTL = 60 * time.Second -// maxDebSuites bounds the client-chosen suites cached per deb virtual. +// maxDebSuites bounds the client-chosen deb suites or alpine directories +// cached per virtual. const maxDebSuites = 32 func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine { e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL} e.rpmMember = e.fetchRPMMember e.debMember = e.fetchDebMember + e.apkMember = e.fetchAlpineMember e.npmMember = e.fetchNPMMember e.gemsMember = e.fetchGemsMember e.ansibleDoc = e.ansibleMemberDoc @@ -97,6 +100,8 @@ func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, pr return e.fetchRPM(ctx, virt, path) case models.PackageDeb: return e.fetchDeb(ctx, virt, path) + case models.PackageAlpine: + return e.fetchAlpine(ctx, virt, path) } merger, err := GetMerger(virt.PackageType) @@ -216,8 +221,8 @@ var ( ) // MemberRedirect maps an rpm, deb, npm or ansible virtual package path -// (/, as written by MergeRPM/MergeDeb/MergeNPM/AnsibleVersion) -// to the owning member's route. ok is false for any other path; ErrBadPath +// (/, as written by MergeRPM/MergeDeb/MergeNPM/AnsibleVersion), +// or an alpine

/-.apk, to the owning member's route. ok is false for any other path; ErrBadPath // means the path is absolute or escapes the member. func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) { switch virt.PackageType { @@ -231,6 +236,8 @@ func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, if strings.HasPrefix(path, "dists/") { return "", false, nil } + case models.PackageAlpine: + return e.alpineRedirect(ctx, virt, path, rawQuery, proxyBaseURL) case models.PackageNPM, models.PackageAnsible: default: return "", false, nil @@ -445,6 +452,78 @@ func (e *Engine) fetchDebMember(ctx context.Context, name, base string) (*DebMem return m, nil } +func (e *Engine) mergedAlpine(ctx context.Context, virt models.Virtual, dir string) (mergedRepo, error) { + repo, err := e.merged(ctx, "alpine\x00"+virt.Name+"\x00"+dir, func(ctx context.Context) (mergedRepo, error) { + members, err := fetchMembers(virt, func(name string) (*AlpineMember, error) { return e.apkMember(ctx, name, dir) }) + if err != nil { + return nil, err + } + return MergeAlpine(members), nil + }) + if err == nil { + e.evictLRU("alpine\x00"+virt.Name+"\x00", maxDebSuites) + } + return repo, err +} + +func (e *Engine) fetchAlpine(ctx context.Context, virt models.Virtual, reqPath string) ([]byte, string, error) { + dir, file, ok := alpinePath(strings.TrimPrefix(path.Clean("/"+reqPath), "/")) + if !ok { + return nil, "", ErrNotFound + } + repo, err := e.mergedAlpine(ctx, virt, dir) + if err != nil { + return nil, "", err + } + if file != "APKINDEX.tar.gz" { + return nil, "", ErrNotFound + } + return repo.index(), "application/gzip", nil +} + +// alpineRedirect finds the member whose index lists the requested .apk; apk +// builds the download path itself, so the index cannot carry a member prefix. +// A failed merge returns ok=false so Fetch re-merges and fails with 502. +func (e *Engine) alpineRedirect(ctx context.Context, virt models.Virtual, reqPath, rawQuery, proxyBaseURL string) (string, bool, error) { + dir, file, ok := alpinePath(strings.TrimPrefix(path.Clean("/"+reqPath), "/")) + if !ok || file == "APKINDEX.tar.gz" { + return "", false, nil + } + owner, ok := e.cachedFile("alpine\x00"+virt.Name+"\x00"+dir, file) + if !ok { + repo, err := e.mergedAlpine(ctx, virt, dir) + if err != nil { + return "", false, nil + } + if owner, ok = repo.files()[file]; !ok { + return "", false, nil + } + } + return e.memberLocation(ctx, string(owner), dir+file, rawQuery, proxyBaseURL) +} + +// fetchAlpineMember reads a member's APKINDEX under dir; a member without that +// directory contributes nothing. +func (e *Engine) fetchAlpineMember(ctx context.Context, name, dir string) (*AlpineMember, error) { + remote, err := e.getRemote(ctx, name) + if err != nil { + return nil, fmt.Errorf("remote %q: %w", name, err) + } + raw, err := e.fetchMemberPath(ctx, *remote, dir+"APKINDEX.tar.gz") + var pe *proxy.ProxyError + if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound { + return &AlpineMember{RemoteName: name}, nil + } + if err != nil { + return nil, err + } + idx, err := apkIndexText(raw) + if err != nil { + return nil, fmt.Errorf("%s/%sAPKINDEX.tar.gz: %w", name, dir, err) + } + return &AlpineMember{RemoteName: name, Index: idx}, nil +} + // evictLRU drops the least recently used merges under prefix beyond keep. func (e *Engine) evictLRU(prefix string, keep int) { e.mu.Lock() -- 2.47.3