Add Alpine/apk local repository support #114

Merged
benvin merged 1 commits from benvin/apk-local into master 2026-08-12 20:39:27 +10:00
7 changed files with 917 additions and 0 deletions
+55
View File
@@ -3,7 +3,10 @@
package e2edocker
import (
"archive/tar"
"bytes"
"compress/gzip"
"io"
"net/http"
"strings"
"testing"
@@ -136,3 +139,55 @@ func TestLocalDebRepo(t *testing.T) {
t.Fatalf("deb content mismatch")
}
}
// TestLocalAlpineIndex uploads an .apk to an alpine local repo and validates
// that a per-arch APKINDEX.tar.gz is generated automatically from the parsed
// .PKGINFO (the apk-local analog of rpm repodata / deb Packages generation).
func TestLocalAlpineIndex(t *testing.T) {
createRepo(t, `{"name":"local-alpine","package_type":"alpine","repo_type":"local"}`)
defer deleteRepo(t, "local-alpine")
apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64")
uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive")
// The index is generated asynchronously after upload; poll for it.
resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second)
if resp.StatusCode != http.StatusOK {
t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body)
}
zr, err := gzip.NewReader(bytes.NewReader(body))
if err != nil {
t.Fatalf("APKINDEX not gzip: %v", err)
}
tarBytes, _ := io.ReadAll(zr)
tr := tar.NewReader(bytes.NewReader(tarBytes))
var index string
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("APKINDEX not tar: %v", err)
}
if hdr.Name == "APKINDEX" {
b, _ := io.ReadAll(tr)
index = string(b)
}
}
for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} {
if !strings.Contains(index, want) {
t.Fatalf("APKINDEX missing %q:\n%s", want, index)
}
}
// The .apk downloads back byte-identical from its arch path.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download apk: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, apk) {
t.Fatalf("apk content mismatch")
}
}
+70
View File
@@ -0,0 +1,70 @@
package database
import (
"context"
"strings"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertAlpineMetadata(ctx context.Context, meta *provider.AlpineMetadata) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO alpine_metadata (
repo_name, file_path, content_hash, checksum,
name, version, arch, download_size, installed_size,
description, url, license, origin, maintainer,
build_time, commit_hash, provider_priority,
depends, provides, install_if
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)
ON CONFLICT (repo_name, file_path) DO NOTHING
`,
meta.RepoName, meta.FilePath, meta.ContentHash, meta.Checksum,
meta.Name, meta.Version, meta.Arch, meta.DownloadSize, meta.InstalledSize,
meta.Description, meta.URL, meta.License, meta.Origin, meta.Maintainer,
meta.BuildTime, meta.Commit, meta.ProviderPriority,
strings.Join(meta.Depends, " "), strings.Join(meta.Provides, " "), strings.Join(meta.InstallIf, " "),
)
return err
}
func (db *DB) DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM alpine_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]provider.AlpineMetadata, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash, checksum,
name, version, arch, download_size, installed_size,
description, url, license, origin, maintainer,
build_time, commit_hash, provider_priority,
depends, provides, install_if
FROM alpine_metadata
WHERE repo_name = $1
ORDER BY name, version, arch
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []provider.AlpineMetadata
for rows.Next() {
var m provider.AlpineMetadata
var depends, provides, installIf string
if err := rows.Scan(
&m.RepoName, &m.FilePath, &m.ContentHash, &m.Checksum,
&m.Name, &m.Version, &m.Arch, &m.DownloadSize, &m.InstalledSize,
&m.Description, &m.URL, &m.License, &m.Origin, &m.Maintainer,
&m.BuildTime, &m.Commit, &m.ProviderPriority,
&depends, &provides, &installIf,
); err != nil {
return nil, err
}
m.Depends = strings.Fields(depends)
m.Provides = strings.Fields(provides)
m.InstallIf = strings.Fields(installIf)
result = append(result, m)
}
return result, rows.Err()
}
+29
View File
@@ -182,6 +182,35 @@ func (db *DB) migrate() error {
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
CREATE TABLE IF NOT EXISTS alpine_metadata (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL,
checksum TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
arch TEXT NOT NULL,
download_size BIGINT DEFAULT 0,
installed_size BIGINT DEFAULT 0,
description TEXT DEFAULT '',
url TEXT DEFAULT '',
license TEXT DEFAULT '',
origin TEXT DEFAULT '',
maintainer TEXT DEFAULT '',
build_time BIGINT DEFAULT 0,
commit_hash TEXT DEFAULT '',
provider_priority TEXT DEFAULT '',
depends TEXT DEFAULT '',
provides TEXT DEFAULT '',
install_if TEXT DEFAULT '',
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
remote_name TEXT PRIMARY KEY,
etag TEXT DEFAULT '',
+357
View File
@@ -1,12 +1,26 @@
package alpine
import (
"bufio"
"bytes"
"compress/gzip"
"context"
"crypto/sha1"
"encoding/base64"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"path"
"strconv"
"strings"
"archive/tar"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -46,3 +60,346 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte,
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
// --- LocalUploader: hosting real .apk packages -----------------------------
// ValidateUpload accepts any *.apk and preserves the client-supplied directory
// (the arch prefix) as the storage path, since arch cannot be parsed from the
// filename alone and the generic uploader hands us only the path. apk clients
// fetch packages at <arch>/<file>.apk, so publishers upload to that same path;
// AfterUpload records the true arch (from .PKGINFO) for index filtering.
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
clean := strings.TrimPrefix(path.Clean("/"+filePath), "/")
filename := clean
if i := strings.LastIndex(clean, "/"); i >= 0 {
filename = clean[i+1:]
}
if !strings.HasSuffix(strings.ToLower(filename), ".apk") {
return "", "", fmt.Errorf("file must be a .apk package")
}
return clean, "application/vnd.android.package-archive", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
filename := storagePath
if i := strings.LastIndex(storagePath, "/"); i >= 0 {
filename = storagePath[i+1:]
}
return map[string]any{
"filename": filename,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
reader, blobSize, err := blobs.Download(ctx, s3Key)
if err != nil {
slog.Error("alpine metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer reader.Close()
raw, err := io.ReadAll(reader)
if err != nil {
slog.Error("alpine metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
return
}
meta, err := parseApk(raw)
if err != nil {
slog.Error("alpine metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
meta.RepoName = repoName
meta.FilePath = storagePath
meta.ContentHash = contentHash
meta.DownloadSize = blobSize
if meta.Name == "" || meta.Arch == "" {
slog.Error("alpine metadata: .PKGINFO missing pkgname/arch", "repo", repoName, "path", storagePath)
return
}
store, ok := db.(provider.AlpineMetadataStore)
if !ok {
slog.Error("alpine metadata: store does not support alpine metadata", "repo", repoName)
return
}
if err := store.InsertAlpineMetadata(ctx, meta); err != nil {
slog.Error("alpine metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("alpine metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
deleter, ok := db.(provider.AlpineMetadataDeleter)
if !ok {
return nil
}
if err := deleter.DeleteAlpineMetadata(ctx, repoName, storagePath); err != nil {
slog.Error("alpine metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
return err
}
slog.Info("alpine metadata: deleted", "repo", repoName, "path", storagePath)
return nil
}
// --- LocalIndexer: generating a per-arch APKINDEX.tar.gz -------------------
// normalizeIndexPath collapses apk's dot-segment prefix: an /etc/apk/repositories
// line of "<url>/api/v1/local/<name>" makes apk request "./<arch>/APKINDEX.tar.gz".
// Mirrors deb's flat-repo normalization.
func normalizeIndexPath(p string) string {
return strings.TrimPrefix(path.Clean("/"+p), "/")
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
clean := normalizeIndexPath(reqPath)
if !strings.HasSuffix(clean, "APKINDEX.tar.gz") {
return false
}
arch := strings.TrimSuffix(clean, "APKINDEX.tar.gz")
arch = strings.Trim(arch, "/")
if arch == "" || strings.Contains(arch, "/") {
http.Error(w, "APKINDEX must be requested per-arch: <arch>/APKINDEX.tar.gz", http.StatusNotFound)
return true
}
reader, ok := files.(provider.AlpineMetadataReader)
if !ok {
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
return true
}
metas, err := reader.ListAlpineMetadataEntries(r.Context(), repoName)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
slog.Warn("alpine: metadata read canceled", "repo", repoName, "error", err)
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
return true
}
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
var filtered []provider.AlpineMetadata
for _, m := range metas {
if m.Arch == arch {
filtered = append(filtered, m)
}
}
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(generateAPKIndex(filtered))
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("alpine local index generation for virtual repos not supported")
}
// --- pure-Go .apk parsing --------------------------------------------------
// parseApk reads an .apk (up to three concatenated, independently gzipped tar
// streams: optional signature, control, data). It locates the control stream by
// its .PKGINFO member, computes the apk pull checksum C: = "Q1" +
// base64(sha1(<control gzip stream bytes>)), and reads the .PKGINFO fields.
func parseApk(raw []byte) (*provider.AlpineMetadata, error) {
members, err := gzipMembers(raw)
if err != nil {
return nil, err
}
for _, m := range members {
pkginfo, ok := pkginfoFromTar(m.tar)
if !ok {
continue
}
meta := parsePkginfo(pkginfo)
sum := sha1.Sum(m.raw)
meta.Checksum = "Q1" + base64.StdEncoding.EncodeToString(sum[:])
return meta, nil
}
return nil, errors.New("no .PKGINFO found in any .apk gzip stream")
}
type gzMember struct {
raw []byte // the raw bytes of this gzip stream (for the Q1 checksum)
tar []byte // the decompressed tar payload
}
// gzipMembers splits the concatenated gzip streams, returning each stream's raw
// bytes alongside its decompressed tar. It relies on bytes.Reader being an
// io.ByteReader (so compress/gzip does not over-read past a member's trailer)
// to recover exact stream boundaries via Multistream(false)+Reset.
func gzipMembers(data []byte) ([]gzMember, error) {
br := bytes.NewReader(data)
zr, err := gzip.NewReader(br)
if err != nil {
return nil, err
}
var members []gzMember
prev := 0
for {
zr.Multistream(false)
out, err := io.ReadAll(zr)
if err != nil {
return nil, err
}
end := len(data) - br.Len()
members = append(members, gzMember{raw: data[prev:end], tar: out})
prev = end
if err := zr.Reset(br); err != nil {
if err == io.EOF {
break
}
return nil, err
}
}
return members, nil
}
func pkginfoFromTar(tarBytes []byte) (string, bool) {
tr := tar.NewReader(bytes.NewReader(tarBytes))
for {
hdr, err := tr.Next()
if err != nil {
return "", false
}
if strings.TrimPrefix(hdr.Name, "./") == ".PKGINFO" {
b, err := io.ReadAll(tr)
if err != nil {
return "", false
}
return string(b), true
}
}
}
// parsePkginfo reads the "key = value" .PKGINFO text, collecting the repeated
// depend/provides/install_if keys into slices.
func parsePkginfo(text string) *provider.AlpineMetadata {
m := &provider.AlpineMetadata{}
sc := bufio.NewScanner(strings.NewReader(text))
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
idx := strings.Index(line, "=")
if idx < 0 {
continue
}
key := strings.TrimSpace(line[:idx])
val := strings.TrimSpace(line[idx+1:])
switch key {
case "pkgname":
m.Name = val
case "pkgver":
m.Version = val
case "arch":
m.Arch = val
case "pkgdesc":
m.Description = val
case "url":
m.URL = val
case "license":
m.License = val
case "origin":
m.Origin = val
case "maintainer":
m.Maintainer = val
case "builddate":
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
m.BuildTime = n
}
case "commit":
m.Commit = val
case "size":
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
m.InstalledSize = n
}
case "provider_priority":
m.ProviderPriority = val
case "depend":
if val != "" {
m.Depends = append(m.Depends, val)
}
case "provides":
if val != "" {
m.Provides = append(m.Provides, val)
}
case "install_if":
if val != "" {
m.InstallIf = append(m.InstallIf, val)
}
}
}
return m
}
// generateAPKIndex builds the APKINDEX.tar.gz = gzip(tar(APKINDEX)) for the
// given (already arch-filtered) rows. Records are blank-line separated; fields
// follow the canonical C/P/V/A/S/I/T/U/L/o/m/t/c/k/D/p/i order and empties are
// omitted. Unsigned (clients use --allow-untrusted), matching rpm gpgcheck=0.
func generateAPKIndex(metas []provider.AlpineMetadata) []byte {
var idx bytes.Buffer
for i, m := range metas {
if i > 0 {
idx.WriteString("\n")
}
writeField(&idx, "C", m.Checksum)
writeField(&idx, "P", m.Name)
writeField(&idx, "V", m.Version)
writeField(&idx, "A", m.Arch)
writeField(&idx, "S", intField(m.DownloadSize))
writeField(&idx, "I", intField(m.InstalledSize))
writeField(&idx, "T", m.Description)
writeField(&idx, "U", m.URL)
writeField(&idx, "L", m.License)
writeField(&idx, "o", m.Origin)
writeField(&idx, "m", m.Maintainer)
writeField(&idx, "t", intField(m.BuildTime))
writeField(&idx, "c", m.Commit)
writeField(&idx, "k", m.ProviderPriority)
writeField(&idx, "D", strings.Join(m.Depends, " "))
writeField(&idx, "p", strings.Join(m.Provides, " "))
writeField(&idx, "i", strings.Join(m.InstallIf, " "))
}
var tarBuf bytes.Buffer
tw := tar.NewWriter(&tarBuf)
body := idx.Bytes()
tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
tw.Write(body)
tw.Close()
var gzBuf bytes.Buffer
gz := gzip.NewWriter(&gzBuf)
gz.Write(tarBuf.Bytes())
gz.Close()
return gzBuf.Bytes()
}
func writeField(b *bytes.Buffer, key, val string) {
if val == "" {
return
}
b.WriteString(key)
b.WriteString(":")
b.WriteString(val)
b.WriteString("\n")
}
func intField(n int64) string {
if n == 0 {
return ""
}
return strconv.FormatInt(n, 10)
}
@@ -0,0 +1,324 @@
package alpine
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/sha1"
"encoding/base64"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
type fakeBlobReader struct{ data []byte }
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
}
type errBlobReader struct{}
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
return nil, 0, io.ErrUnexpectedEOF
}
// fakeAlpineStore satisfies provider.MetadataStore (shared) and
// provider.AlpineMetadataStore, recording the row AfterUpload writes.
type fakeAlpineStore struct{ inserted *provider.AlpineMetadata }
func (f *fakeAlpineStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
func (f *fakeAlpineStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
func (f *fakeAlpineStore) InsertAlpineMetadata(_ context.Context, m *provider.AlpineMetadata) error {
f.inserted = m
return nil
}
// fakeAlpineDeleter satisfies provider.MetadataDeleter and AlpineMetadataDeleter.
type fakeAlpineDeleter struct{ deleted bool }
func (f *fakeAlpineDeleter) DeleteRPMMetadata(context.Context, string, string) error { return nil }
func (f *fakeAlpineDeleter) DeleteDebMetadata(context.Context, string, string) error { return nil }
func (f *fakeAlpineDeleter) DeleteAlpineMetadata(context.Context, string, string) error {
f.deleted = true
return nil
}
// fakeAlpineReader is a FileStore that also serves alpine metadata rows.
type fakeAlpineReader struct{ metas []provider.AlpineMetadata }
func (f fakeAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
return f.metas, nil
}
func (f fakeAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (f fakeAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
type errAlpineReader struct{}
func (errAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
return nil, io.ErrUnexpectedEOF
}
func (errAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (errAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestAlpineValidateUpload(t *testing.T) {
p := &Provider{}
sp, ct, err := p.ValidateUpload("x86_64/foo-1.0-r0.apk")
if err != nil || sp != "x86_64/foo-1.0-r0.apk" || ct != "application/vnd.android.package-archive" {
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
}
// Dot-segment prefix is normalized away.
if sp, _, err := p.ValidateUpload("./aarch64/bar-2.0-r1.apk"); err != nil || sp != "aarch64/bar-2.0-r1.apk" {
t.Errorf("dot-seg: sp=%q err=%v", sp, err)
}
if _, _, err := p.ValidateUpload("foo.rpm"); err == nil {
t.Error("expected error for non-apk")
}
resp := p.UploadResponse("x86_64/foo-1.0-r0.apk", "sha256:abc", 42)
if resp["filename"] != "foo-1.0-r0.apk" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) {
t.Errorf("upload response %v", resp)
}
}
func TestAlpineAfterUpload(t *testing.T) {
data := testsupport.MinimalApk("hello", "1.0-r0", "x86_64")
store := &fakeAlpineStore{}
(&Provider{}).AfterUpload(context.Background(), "myrepo", "x86_64/hello-1.0-r0.apk",
"sha256:deadbeef", fakeBlobReader{data: data}, store)
m := store.inserted
if m == nil {
t.Fatal("no metadata inserted")
}
if m.Name != "hello" || m.Version != "1.0-r0" || m.Arch != "x86_64" {
t.Errorf("unexpected metadata: %+v", m)
}
if m.DownloadSize != int64(len(data)) {
t.Errorf("DownloadSize = %d, want %d", m.DownloadSize, len(data))
}
if m.InstalledSize != 4 {
t.Errorf("InstalledSize = %d, want 4", m.InstalledSize)
}
if m.License != "MIT" || m.Origin != "hello" || !strings.HasPrefix(m.Maintainer, "e2e") {
t.Errorf("scalar fields not parsed: %+v", m)
}
if len(m.Depends) != 1 || m.Depends[0] != "so:libc.musl-x86_64.so.1" {
t.Errorf("Depends = %v", m.Depends)
}
if len(m.Provides) != 1 || m.Provides[0] != "cmd:hello=1.0-r0" {
t.Errorf("Provides = %v", m.Provides)
}
// The Q1 checksum is the sha1 of the CONTROL gzip stream (the member whose
// tar carries .PKGINFO), not of the whole file.
controlRaw := controlStreamBytes(t, data)
sum := sha1.Sum(controlRaw)
want := "Q1" + base64.StdEncoding.EncodeToString(sum[:])
if m.Checksum != want {
t.Errorf("Checksum = %q, want %q (sha1 of control stream)", m.Checksum, want)
}
// And explicitly NOT the sha1 of the whole apk.
whole := sha1.Sum(data)
if m.Checksum == "Q1"+base64.StdEncoding.EncodeToString(whole[:]) {
t.Error("Checksum was computed over the whole file, not the control stream")
}
}
func TestAlpineAfterUploadErrors(t *testing.T) {
store := &fakeAlpineStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", errBlobReader{}, store)
if store.inserted != nil {
t.Error("no metadata should be inserted on download error")
}
store2 := &fakeAlpineStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", fakeBlobReader{data: []byte("not an apk")}, store2)
if store2.inserted != nil {
t.Error("no metadata should be inserted on parse error")
}
}
func TestAlpineAfterDelete(t *testing.T) {
d := &fakeAlpineDeleter{}
if err := (&Provider{}).AfterDelete(context.Background(), "r", "x86_64/p.apk", d); err != nil {
t.Fatalf("AfterDelete: %v", err)
}
if !d.deleted {
t.Error("DeleteAlpineMetadata not called")
}
}
func TestAlpineServeLocalIndex(t *testing.T) {
p := &Provider{}
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10,
Description: "pkg aaa", URL: "https://a", License: "MIT", Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}},
{Name: "bbb", Version: "2.0-r0", Arch: "aarch64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
}}
// x86_64 index contains only aaa, with its fields, and not bbb.
w := serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz")
if w.Code != 200 {
t.Fatalf("code %d", w.Code)
}
idx := untarIndex(t, w.Body.Bytes())
for _, want := range []string{"C:Q1aaa", "P:aaa", "V:1.0-r0", "A:x86_64", "S:100", "I:10", "T:pkg aaa", "U:https://a", "L:MIT", "D:so:libc", "p:cmd:aaa"} {
if !strings.Contains(idx, want) {
t.Errorf("x86_64 APKINDEX missing %q:\n%s", want, idx)
}
}
if strings.Contains(idx, "P:bbb") {
t.Errorf("x86_64 APKINDEX leaked aarch64 package:\n%s", idx)
}
// aarch64 index contains only bbb.
w = serveIndex(t, p, reader, "aarch64/APKINDEX.tar.gz")
idx = untarIndex(t, w.Body.Bytes())
if !strings.Contains(idx, "P:bbb") || strings.Contains(idx, "P:aaa") {
t.Errorf("aarch64 filtering wrong:\n%s", idx)
}
// Non-index and .apk paths are not owned by the indexer.
for _, path := range []string{"x86_64/foo-1.0-r0.apk", "x86_64/", "README"} {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
if p.ServeLocalIndex(w, r, reader, "repo", path) {
t.Errorf("ServeLocalIndex should return false for %q", path)
}
}
}
// Empty fields are omitted from the record (bbb has no description/url).
func TestAlpineIndexOmitsEmptyFields(t *testing.T) {
p := &Provider{}
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
{Name: "bbb", Version: "2.0-r0", Arch: "x86_64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
}}
idx := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
for _, absent := range []string{"T:", "U:", "L:", "D:", "p:", "i:", "o:", "m:", "c:", "k:"} {
if strings.Contains(idx, absent) {
t.Errorf("empty field %q should be omitted:\n%s", absent, idx)
}
}
}
// apk requests "./<arch>/APKINDEX.tar.gz" for a bare repo base URL; the
// dot-segment must be collapsed and yield the same bytes as the plain path.
func TestAlpineServeLocalIndexDotSegment(t *testing.T) {
p := &Provider{}
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10},
}}
plain := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
dotted := untarIndex(t, serveIndex(t, p, reader, "./x86_64/APKINDEX.tar.gz").Body.Bytes())
if plain != dotted {
t.Errorf("dot-segment path differs:\nplain=%q\ndotted=%q", plain, dotted)
}
}
func TestAlpineServeLocalIndexArchRequired(t *testing.T) {
p := &Provider{}
reader := fakeAlpineReader{}
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/APKINDEX.tar.gz", nil)
if !p.ServeLocalIndex(w, r, reader, "repo", "APKINDEX.tar.gz") {
t.Fatal("bare APKINDEX should be owned (and rejected) by the indexer")
}
if w.Code != http.StatusNotFound {
t.Errorf("bare APKINDEX code = %d, want 404", w.Code)
}
}
func TestAlpineServeMetadataError(t *testing.T) {
p := &Provider{}
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/x86_64/APKINDEX.tar.gz", nil)
p.ServeLocalIndex(w, r, errAlpineReader{}, "repo", "x86_64/APKINDEX.tar.gz")
if w.Code != 500 {
t.Errorf("failing reader code = %d, want 500", w.Code)
}
}
func TestAlpineGenerateLocalIndexUnsupported(t *testing.T) {
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeAlpineReader{}, "r", "x86_64/APKINDEX.tar.gz"); err == nil {
t.Error("expected unsupported error")
}
}
func serveIndex(t *testing.T, p *Provider, files provider.FileStore, path string) *httptest.ResponseRecorder {
t.Helper()
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
if !p.ServeLocalIndex(w, r, files, "repo", path) {
t.Fatalf("ServeLocalIndex returned false for %q", path)
}
return w
}
// untarIndex un-gzips and un-tars an APKINDEX.tar.gz and returns the APKINDEX text.
func untarIndex(t *testing.T, gzTar []byte) string {
t.Helper()
zr, err := gzip.NewReader(bytes.NewReader(gzTar))
if err != nil {
t.Fatalf("APKINDEX not gzip: %v", err)
}
tarBytes, _ := io.ReadAll(zr)
tr := tar.NewReader(bytes.NewReader(tarBytes))
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatalf("APKINDEX not tar: %v", err)
}
if hdr.Name == "APKINDEX" {
b, _ := io.ReadAll(tr)
return string(b)
}
}
t.Fatal("no APKINDEX member in tarball")
return ""
}
// controlStreamBytes returns the raw bytes of the gzip stream whose tar carries
// .PKGINFO, so the test can independently compute the expected Q1 checksum.
func controlStreamBytes(t *testing.T, apk []byte) []byte {
t.Helper()
br := bytes.NewReader(apk)
zr, err := gzip.NewReader(br)
if err != nil {
t.Fatalf("gzip: %v", err)
}
prev := 0
for {
zr.Multistream(false)
out, _ := io.ReadAll(zr)
end := len(apk) - br.Len()
tr := tar.NewReader(bytes.NewReader(out))
for {
h, err := tr.Next()
if err != nil {
break
}
if strings.TrimPrefix(h.Name, "./") == ".PKGINFO" {
return apk[prev:end]
}
}
prev = end
if err := zr.Reset(br); err != nil {
break
}
}
t.Fatal("no control stream found")
return nil
}
+44
View File
@@ -115,6 +115,50 @@ type DebMetadata struct {
SHA256 string
}
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
// Alpine-specific persistence surfaces. They are kept separate from the shared
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
// satisfies all three.
type AlpineMetadataStore interface {
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
}
type AlpineMetadataDeleter interface {
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
}
type AlpineMetadataReader interface {
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
}
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
// sha1 of the control gzip stream) and the download/installed sizes.
type AlpineMetadata struct {
RepoName string
FilePath string
ContentHash string
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
Name string // P:
Version string // V:
Arch string // A:
DownloadSize int64 // S: on-disk .apk size
InstalledSize int64 // I: unpacked size from .PKGINFO
Description string // T:
URL string // U:
License string // L:
Origin string // o:
Maintainer string // m:
BuildTime int64 // t:
Commit string // c:
ProviderPriority string // k:
Depends []string // D:
Provides []string // p:
InstallIf []string // i:
}
type RPMMetadata struct {
RepoName string
FilePath string
+38
View File
@@ -0,0 +1,38 @@
package testsupport
import (
"bytes"
"fmt"
)
// MinimalApk builds a valid-enough Alpine package in pure Go (no committed
// binary fixture, no abuild): two concatenated, independently gzipped tar
// streams -- a control stream carrying .PKGINFO and a data stream carrying a
// single payload file. It mirrors MinimalDeb/MinimalRPM and is parseable by the
// alpine provider (which derives arch/name/version and the Q1 pull checksum from
// the control stream).
func MinimalApk(name, version, arch string) []byte {
pkginfo := fmt.Sprintf(
"# generated by testsupport\n"+
"pkgname = %s\n"+
"pkgver = %s\n"+
"arch = %s\n"+
"pkgdesc = minimal test package\n"+
"url = https://example.com/%s\n"+
"license = MIT\n"+
"origin = %s\n"+
"maintainer = e2e <e2e@example.com>\n"+
"builddate = 1700000000\n"+
"size = 4\n"+
"depend = so:libc.musl-x86_64.so.1\n"+
"provides = cmd:%s=%s\n",
name, version, arch, name, name, name, version)
control := gzipBytes(tarSingle(".PKGINFO", []byte(pkginfo)))
data := gzipBytes(tarSingle("usr/bin/"+name, []byte("body")))
var buf bytes.Buffer
buf.Write(control)
buf.Write(data)
return buf.Bytes()
}