Add Alpine/apk local repository support #114
@@ -3,7 +3,10 @@
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -136,3 +139,55 @@ func TestLocalDebRepo(t *testing.T) {
|
||||
t.Fatalf("deb content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalAlpineIndex uploads an .apk to an alpine local repo and validates
|
||||
// that a per-arch APKINDEX.tar.gz is generated automatically from the parsed
|
||||
// .PKGINFO (the apk-local analog of rpm repodata / deb Packages generation).
|
||||
func TestLocalAlpineIndex(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-alpine","package_type":"alpine","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-alpine")
|
||||
|
||||
apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64")
|
||||
uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive")
|
||||
|
||||
// The index is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
zr, err := gzip.NewReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not gzip: %v", err)
|
||||
}
|
||||
tarBytes, _ := io.ReadAll(zr)
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
var index string
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not tar: %v", err)
|
||||
}
|
||||
if hdr.Name == "APKINDEX" {
|
||||
b, _ := io.ReadAll(tr)
|
||||
index = string(b)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} {
|
||||
if !strings.Contains(index, want) {
|
||||
t.Fatalf("APKINDEX missing %q:\n%s", want, index)
|
||||
}
|
||||
}
|
||||
|
||||
// The .apk downloads back byte-identical from its arch path.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download apk: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, apk) {
|
||||
t.Fatalf("apk content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertAlpineMetadata(ctx context.Context, meta *provider.AlpineMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO alpine_metadata (
|
||||
repo_name, file_path, content_hash, checksum,
|
||||
name, version, arch, download_size, installed_size,
|
||||
description, url, license, origin, maintainer,
|
||||
build_time, commit_hash, provider_priority,
|
||||
depends, provides, install_if
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash, meta.Checksum,
|
||||
meta.Name, meta.Version, meta.Arch, meta.DownloadSize, meta.InstalledSize,
|
||||
meta.Description, meta.URL, meta.License, meta.Origin, meta.Maintainer,
|
||||
meta.BuildTime, meta.Commit, meta.ProviderPriority,
|
||||
strings.Join(meta.Depends, " "), strings.Join(meta.Provides, " "), strings.Join(meta.InstallIf, " "),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM alpine_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]provider.AlpineMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, checksum,
|
||||
name, version, arch, download_size, installed_size,
|
||||
description, url, license, origin, maintainer,
|
||||
build_time, commit_hash, provider_priority,
|
||||
depends, provides, install_if
|
||||
FROM alpine_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.AlpineMetadata
|
||||
for rows.Next() {
|
||||
var m provider.AlpineMetadata
|
||||
var depends, provides, installIf string
|
||||
if err := rows.Scan(
|
||||
&m.RepoName, &m.FilePath, &m.ContentHash, &m.Checksum,
|
||||
&m.Name, &m.Version, &m.Arch, &m.DownloadSize, &m.InstalledSize,
|
||||
&m.Description, &m.URL, &m.License, &m.Origin, &m.Maintainer,
|
||||
&m.BuildTime, &m.Commit, &m.ProviderPriority,
|
||||
&depends, &provides, &installIf,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Depends = strings.Fields(depends)
|
||||
m.Provides = strings.Fields(provides)
|
||||
m.InstallIf = strings.Fields(installIf)
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -182,6 +182,35 @@ func (db *DB) migrate() error {
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alpine_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
download_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
description TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
license TEXT DEFAULT '',
|
||||
origin TEXT DEFAULT '',
|
||||
maintainer TEXT DEFAULT '',
|
||||
build_time BIGINT DEFAULT 0,
|
||||
commit_hash TEXT DEFAULT '',
|
||||
provider_priority TEXT DEFAULT '',
|
||||
depends TEXT DEFAULT '',
|
||||
provides TEXT DEFAULT '',
|
||||
install_if TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
|
||||
@@ -1,12 +1,26 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"archive/tar"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -46,3 +60,346 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte,
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
// --- LocalUploader: hosting real .apk packages -----------------------------
|
||||
|
||||
// ValidateUpload accepts any *.apk and preserves the client-supplied directory
|
||||
// (the arch prefix) as the storage path, since arch cannot be parsed from the
|
||||
// filename alone and the generic uploader hands us only the path. apk clients
|
||||
// fetch packages at <arch>/<file>.apk, so publishers upload to that same path;
|
||||
// AfterUpload records the true arch (from .PKGINFO) for index filtering.
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
clean := strings.TrimPrefix(path.Clean("/"+filePath), "/")
|
||||
filename := clean
|
||||
if i := strings.LastIndex(clean, "/"); i >= 0 {
|
||||
filename = clean[i+1:]
|
||||
}
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".apk") {
|
||||
return "", "", fmt.Errorf("file must be a .apk package")
|
||||
}
|
||||
return clean, "application/vnd.android.package-archive", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := storagePath
|
||||
if i := strings.LastIndex(storagePath, "/"); i >= 0 {
|
||||
filename = storagePath[i+1:]
|
||||
}
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
raw, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
meta, err := parseApk(raw)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
meta.RepoName = repoName
|
||||
meta.FilePath = storagePath
|
||||
meta.ContentHash = contentHash
|
||||
meta.DownloadSize = blobSize
|
||||
|
||||
if meta.Name == "" || meta.Arch == "" {
|
||||
slog.Error("alpine metadata: .PKGINFO missing pkgname/arch", "repo", repoName, "path", storagePath)
|
||||
return
|
||||
}
|
||||
|
||||
store, ok := db.(provider.AlpineMetadataStore)
|
||||
if !ok {
|
||||
slog.Error("alpine metadata: store does not support alpine metadata", "repo", repoName)
|
||||
return
|
||||
}
|
||||
if err := store.InsertAlpineMetadata(ctx, meta); err != nil {
|
||||
slog.Error("alpine metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("alpine metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
deleter, ok := db.(provider.AlpineMetadataDeleter)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err := deleter.DeleteAlpineMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("alpine metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("alpine metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- LocalIndexer: generating a per-arch APKINDEX.tar.gz -------------------
|
||||
|
||||
// normalizeIndexPath collapses apk's dot-segment prefix: an /etc/apk/repositories
|
||||
// line of "<url>/api/v1/local/<name>" makes apk request "./<arch>/APKINDEX.tar.gz".
|
||||
// Mirrors deb's flat-repo normalization.
|
||||
func normalizeIndexPath(p string) string {
|
||||
return strings.TrimPrefix(path.Clean("/"+p), "/")
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
clean := normalizeIndexPath(reqPath)
|
||||
if !strings.HasSuffix(clean, "APKINDEX.tar.gz") {
|
||||
return false
|
||||
}
|
||||
arch := strings.TrimSuffix(clean, "APKINDEX.tar.gz")
|
||||
arch = strings.Trim(arch, "/")
|
||||
if arch == "" || strings.Contains(arch, "/") {
|
||||
http.Error(w, "APKINDEX must be requested per-arch: <arch>/APKINDEX.tar.gz", http.StatusNotFound)
|
||||
return true
|
||||
}
|
||||
|
||||
reader, ok := files.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
metas, err := reader.ListAlpineMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
slog.Warn("alpine: metadata read canceled", "repo", repoName, "error", err)
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return true
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
var filtered []provider.AlpineMetadata
|
||||
for _, m := range metas {
|
||||
if m.Arch == arch {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateAPKIndex(filtered))
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("alpine local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
// --- pure-Go .apk parsing --------------------------------------------------
|
||||
|
||||
// parseApk reads an .apk (up to three concatenated, independently gzipped tar
|
||||
// streams: optional signature, control, data). It locates the control stream by
|
||||
// its .PKGINFO member, computes the apk pull checksum C: = "Q1" +
|
||||
// base64(sha1(<control gzip stream bytes>)), and reads the .PKGINFO fields.
|
||||
func parseApk(raw []byte) (*provider.AlpineMetadata, error) {
|
||||
members, err := gzipMembers(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, m := range members {
|
||||
pkginfo, ok := pkginfoFromTar(m.tar)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
meta := parsePkginfo(pkginfo)
|
||||
sum := sha1.Sum(m.raw)
|
||||
meta.Checksum = "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
return meta, nil
|
||||
}
|
||||
return nil, errors.New("no .PKGINFO found in any .apk gzip stream")
|
||||
}
|
||||
|
||||
type gzMember struct {
|
||||
raw []byte // the raw bytes of this gzip stream (for the Q1 checksum)
|
||||
tar []byte // the decompressed tar payload
|
||||
}
|
||||
|
||||
// gzipMembers splits the concatenated gzip streams, returning each stream's raw
|
||||
// bytes alongside its decompressed tar. It relies on bytes.Reader being an
|
||||
// io.ByteReader (so compress/gzip does not over-read past a member's trailer)
|
||||
// to recover exact stream boundaries via Multistream(false)+Reset.
|
||||
func gzipMembers(data []byte) ([]gzMember, error) {
|
||||
br := bytes.NewReader(data)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var members []gzMember
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
end := len(data) - br.Len()
|
||||
members = append(members, gzMember{raw: data[prev:end], tar: out})
|
||||
prev = end
|
||||
if err := zr.Reset(br); err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func pkginfoFromTar(tarBytes []byte) (string, bool) {
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if strings.TrimPrefix(hdr.Name, "./") == ".PKGINFO" {
|
||||
b, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return string(b), true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parsePkginfo reads the "key = value" .PKGINFO text, collecting the repeated
|
||||
// depend/provides/install_if keys into slices.
|
||||
func parsePkginfo(text string) *provider.AlpineMetadata {
|
||||
m := &provider.AlpineMetadata{}
|
||||
sc := bufio.NewScanner(strings.NewReader(text))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(line, "=")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
val := strings.TrimSpace(line[idx+1:])
|
||||
switch key {
|
||||
case "pkgname":
|
||||
m.Name = val
|
||||
case "pkgver":
|
||||
m.Version = val
|
||||
case "arch":
|
||||
m.Arch = val
|
||||
case "pkgdesc":
|
||||
m.Description = val
|
||||
case "url":
|
||||
m.URL = val
|
||||
case "license":
|
||||
m.License = val
|
||||
case "origin":
|
||||
m.Origin = val
|
||||
case "maintainer":
|
||||
m.Maintainer = val
|
||||
case "builddate":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
m.BuildTime = n
|
||||
}
|
||||
case "commit":
|
||||
m.Commit = val
|
||||
case "size":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
m.InstalledSize = n
|
||||
}
|
||||
case "provider_priority":
|
||||
m.ProviderPriority = val
|
||||
case "depend":
|
||||
if val != "" {
|
||||
m.Depends = append(m.Depends, val)
|
||||
}
|
||||
case "provides":
|
||||
if val != "" {
|
||||
m.Provides = append(m.Provides, val)
|
||||
}
|
||||
case "install_if":
|
||||
if val != "" {
|
||||
m.InstallIf = append(m.InstallIf, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// generateAPKIndex builds the APKINDEX.tar.gz = gzip(tar(APKINDEX)) for the
|
||||
// given (already arch-filtered) rows. Records are blank-line separated; fields
|
||||
// follow the canonical C/P/V/A/S/I/T/U/L/o/m/t/c/k/D/p/i order and empties are
|
||||
// omitted. Unsigned (clients use --allow-untrusted), matching rpm gpgcheck=0.
|
||||
func generateAPKIndex(metas []provider.AlpineMetadata) []byte {
|
||||
var idx bytes.Buffer
|
||||
for i, m := range metas {
|
||||
if i > 0 {
|
||||
idx.WriteString("\n")
|
||||
}
|
||||
writeField(&idx, "C", m.Checksum)
|
||||
writeField(&idx, "P", m.Name)
|
||||
writeField(&idx, "V", m.Version)
|
||||
writeField(&idx, "A", m.Arch)
|
||||
writeField(&idx, "S", intField(m.DownloadSize))
|
||||
writeField(&idx, "I", intField(m.InstalledSize))
|
||||
writeField(&idx, "T", m.Description)
|
||||
writeField(&idx, "U", m.URL)
|
||||
writeField(&idx, "L", m.License)
|
||||
writeField(&idx, "o", m.Origin)
|
||||
writeField(&idx, "m", m.Maintainer)
|
||||
writeField(&idx, "t", intField(m.BuildTime))
|
||||
writeField(&idx, "c", m.Commit)
|
||||
writeField(&idx, "k", m.ProviderPriority)
|
||||
writeField(&idx, "D", strings.Join(m.Depends, " "))
|
||||
writeField(&idx, "p", strings.Join(m.Provides, " "))
|
||||
writeField(&idx, "i", strings.Join(m.InstallIf, " "))
|
||||
}
|
||||
|
||||
var tarBuf bytes.Buffer
|
||||
tw := tar.NewWriter(&tarBuf)
|
||||
body := idx.Bytes()
|
||||
tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
|
||||
tw.Write(body)
|
||||
tw.Close()
|
||||
|
||||
var gzBuf bytes.Buffer
|
||||
gz := gzip.NewWriter(&gzBuf)
|
||||
gz.Write(tarBuf.Bytes())
|
||||
gz.Close()
|
||||
return gzBuf.Bytes()
|
||||
}
|
||||
|
||||
func writeField(b *bytes.Buffer, key, val string) {
|
||||
if val == "" {
|
||||
return
|
||||
}
|
||||
b.WriteString(key)
|
||||
b.WriteString(":")
|
||||
b.WriteString(val)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
func intField(n int64) string {
|
||||
if n == 0 {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatInt(n, 10)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
type fakeBlobReader struct{ data []byte }
|
||||
|
||||
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type errBlobReader struct{}
|
||||
|
||||
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return nil, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// fakeAlpineStore satisfies provider.MetadataStore (shared) and
|
||||
// provider.AlpineMetadataStore, recording the row AfterUpload writes.
|
||||
type fakeAlpineStore struct{ inserted *provider.AlpineMetadata }
|
||||
|
||||
func (f *fakeAlpineStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeAlpineStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeAlpineStore) InsertAlpineMetadata(_ context.Context, m *provider.AlpineMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeAlpineDeleter satisfies provider.MetadataDeleter and AlpineMetadataDeleter.
|
||||
type fakeAlpineDeleter struct{ deleted bool }
|
||||
|
||||
func (f *fakeAlpineDeleter) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeAlpineDeleter) DeleteDebMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeAlpineDeleter) DeleteAlpineMetadata(context.Context, string, string) error {
|
||||
f.deleted = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeAlpineReader is a FileStore that also serves alpine metadata rows.
|
||||
type fakeAlpineReader struct{ metas []provider.AlpineMetadata }
|
||||
|
||||
func (f fakeAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f fakeAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
type errAlpineReader struct{}
|
||||
|
||||
func (errAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
func (errAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (errAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestAlpineValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("x86_64/foo-1.0-r0.apk")
|
||||
if err != nil || sp != "x86_64/foo-1.0-r0.apk" || ct != "application/vnd.android.package-archive" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
// Dot-segment prefix is normalized away.
|
||||
if sp, _, err := p.ValidateUpload("./aarch64/bar-2.0-r1.apk"); err != nil || sp != "aarch64/bar-2.0-r1.apk" {
|
||||
t.Errorf("dot-seg: sp=%q err=%v", sp, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("foo.rpm"); err == nil {
|
||||
t.Error("expected error for non-apk")
|
||||
}
|
||||
resp := p.UploadResponse("x86_64/foo-1.0-r0.apk", "sha256:abc", 42)
|
||||
if resp["filename"] != "foo-1.0-r0.apk" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) {
|
||||
t.Errorf("upload response %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterUpload(t *testing.T) {
|
||||
data := testsupport.MinimalApk("hello", "1.0-r0", "x86_64")
|
||||
store := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "myrepo", "x86_64/hello-1.0-r0.apk",
|
||||
"sha256:deadbeef", fakeBlobReader{data: data}, store)
|
||||
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("no metadata inserted")
|
||||
}
|
||||
if m.Name != "hello" || m.Version != "1.0-r0" || m.Arch != "x86_64" {
|
||||
t.Errorf("unexpected metadata: %+v", m)
|
||||
}
|
||||
if m.DownloadSize != int64(len(data)) {
|
||||
t.Errorf("DownloadSize = %d, want %d", m.DownloadSize, len(data))
|
||||
}
|
||||
if m.InstalledSize != 4 {
|
||||
t.Errorf("InstalledSize = %d, want 4", m.InstalledSize)
|
||||
}
|
||||
if m.License != "MIT" || m.Origin != "hello" || !strings.HasPrefix(m.Maintainer, "e2e") {
|
||||
t.Errorf("scalar fields not parsed: %+v", m)
|
||||
}
|
||||
if len(m.Depends) != 1 || m.Depends[0] != "so:libc.musl-x86_64.so.1" {
|
||||
t.Errorf("Depends = %v", m.Depends)
|
||||
}
|
||||
if len(m.Provides) != 1 || m.Provides[0] != "cmd:hello=1.0-r0" {
|
||||
t.Errorf("Provides = %v", m.Provides)
|
||||
}
|
||||
|
||||
// The Q1 checksum is the sha1 of the CONTROL gzip stream (the member whose
|
||||
// tar carries .PKGINFO), not of the whole file.
|
||||
controlRaw := controlStreamBytes(t, data)
|
||||
sum := sha1.Sum(controlRaw)
|
||||
want := "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
if m.Checksum != want {
|
||||
t.Errorf("Checksum = %q, want %q (sha1 of control stream)", m.Checksum, want)
|
||||
}
|
||||
// And explicitly NOT the sha1 of the whole apk.
|
||||
whole := sha1.Sum(data)
|
||||
if m.Checksum == "Q1"+base64.StdEncoding.EncodeToString(whole[:]) {
|
||||
t.Error("Checksum was computed over the whole file, not the control stream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterUploadErrors(t *testing.T) {
|
||||
store := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", errBlobReader{}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("no metadata should be inserted on download error")
|
||||
}
|
||||
store2 := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", fakeBlobReader{data: []byte("not an apk")}, store2)
|
||||
if store2.inserted != nil {
|
||||
t.Error("no metadata should be inserted on parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterDelete(t *testing.T) {
|
||||
d := &fakeAlpineDeleter{}
|
||||
if err := (&Provider{}).AfterDelete(context.Background(), "r", "x86_64/p.apk", d); err != nil {
|
||||
t.Fatalf("AfterDelete: %v", err)
|
||||
}
|
||||
if !d.deleted {
|
||||
t.Error("DeleteAlpineMetadata not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10,
|
||||
Description: "pkg aaa", URL: "https://a", License: "MIT", Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}},
|
||||
{Name: "bbb", Version: "2.0-r0", Arch: "aarch64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
|
||||
}}
|
||||
|
||||
// x86_64 index contains only aaa, with its fields, and not bbb.
|
||||
w := serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("code %d", w.Code)
|
||||
}
|
||||
idx := untarIndex(t, w.Body.Bytes())
|
||||
for _, want := range []string{"C:Q1aaa", "P:aaa", "V:1.0-r0", "A:x86_64", "S:100", "I:10", "T:pkg aaa", "U:https://a", "L:MIT", "D:so:libc", "p:cmd:aaa"} {
|
||||
if !strings.Contains(idx, want) {
|
||||
t.Errorf("x86_64 APKINDEX missing %q:\n%s", want, idx)
|
||||
}
|
||||
}
|
||||
if strings.Contains(idx, "P:bbb") {
|
||||
t.Errorf("x86_64 APKINDEX leaked aarch64 package:\n%s", idx)
|
||||
}
|
||||
|
||||
// aarch64 index contains only bbb.
|
||||
w = serveIndex(t, p, reader, "aarch64/APKINDEX.tar.gz")
|
||||
idx = untarIndex(t, w.Body.Bytes())
|
||||
if !strings.Contains(idx, "P:bbb") || strings.Contains(idx, "P:aaa") {
|
||||
t.Errorf("aarch64 filtering wrong:\n%s", idx)
|
||||
}
|
||||
|
||||
// Non-index and .apk paths are not owned by the indexer.
|
||||
for _, path := range []string{"x86_64/foo-1.0-r0.apk", "x86_64/", "README"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if p.ServeLocalIndex(w, r, reader, "repo", path) {
|
||||
t.Errorf("ServeLocalIndex should return false for %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Empty fields are omitted from the record (bbb has no description/url).
|
||||
func TestAlpineIndexOmitsEmptyFields(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "bbb", Version: "2.0-r0", Arch: "x86_64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
|
||||
}}
|
||||
idx := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
for _, absent := range []string{"T:", "U:", "L:", "D:", "p:", "i:", "o:", "m:", "c:", "k:"} {
|
||||
if strings.Contains(idx, absent) {
|
||||
t.Errorf("empty field %q should be omitted:\n%s", absent, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// apk requests "./<arch>/APKINDEX.tar.gz" for a bare repo base URL; the
|
||||
// dot-segment must be collapsed and yield the same bytes as the plain path.
|
||||
func TestAlpineServeLocalIndexDotSegment(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10},
|
||||
}}
|
||||
plain := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
dotted := untarIndex(t, serveIndex(t, p, reader, "./x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
if plain != dotted {
|
||||
t.Errorf("dot-segment path differs:\nplain=%q\ndotted=%q", plain, dotted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeLocalIndexArchRequired(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{}
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "repo", "APKINDEX.tar.gz") {
|
||||
t.Fatal("bare APKINDEX should be owned (and rejected) by the indexer")
|
||||
}
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("bare APKINDEX code = %d, want 404", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeMetadataError(t *testing.T) {
|
||||
p := &Provider{}
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/x86_64/APKINDEX.tar.gz", nil)
|
||||
p.ServeLocalIndex(w, r, errAlpineReader{}, "repo", "x86_64/APKINDEX.tar.gz")
|
||||
if w.Code != 500 {
|
||||
t.Errorf("failing reader code = %d, want 500", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeAlpineReader{}, "r", "x86_64/APKINDEX.tar.gz"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
|
||||
func serveIndex(t *testing.T, p *Provider, files provider.FileStore, path string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, files, "repo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// untarIndex un-gzips and un-tars an APKINDEX.tar.gz and returns the APKINDEX text.
|
||||
func untarIndex(t *testing.T, gzTar []byte) string {
|
||||
t.Helper()
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gzTar))
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not gzip: %v", err)
|
||||
}
|
||||
tarBytes, _ := io.ReadAll(zr)
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not tar: %v", err)
|
||||
}
|
||||
if hdr.Name == "APKINDEX" {
|
||||
b, _ := io.ReadAll(tr)
|
||||
return string(b)
|
||||
}
|
||||
}
|
||||
t.Fatal("no APKINDEX member in tarball")
|
||||
return ""
|
||||
}
|
||||
|
||||
// controlStreamBytes returns the raw bytes of the gzip stream whose tar carries
|
||||
// .PKGINFO, so the test can independently compute the expected Q1 checksum.
|
||||
func controlStreamBytes(t *testing.T, apk []byte) []byte {
|
||||
t.Helper()
|
||||
br := bytes.NewReader(apk)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, _ := io.ReadAll(zr)
|
||||
end := len(apk) - br.Len()
|
||||
tr := tar.NewReader(bytes.NewReader(out))
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if strings.TrimPrefix(h.Name, "./") == ".PKGINFO" {
|
||||
return apk[prev:end]
|
||||
}
|
||||
}
|
||||
prev = end
|
||||
if err := zr.Reset(br); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
t.Fatal("no control stream found")
|
||||
return nil
|
||||
}
|
||||
@@ -115,6 +115,50 @@ type DebMetadata struct {
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
|
||||
// Alpine-specific persistence surfaces. They are kept separate from the shared
|
||||
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
|
||||
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
|
||||
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
|
||||
// satisfies all three.
|
||||
type AlpineMetadataStore interface {
|
||||
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
|
||||
}
|
||||
|
||||
type AlpineMetadataDeleter interface {
|
||||
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type AlpineMetadataReader interface {
|
||||
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
|
||||
}
|
||||
|
||||
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
|
||||
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
|
||||
// sha1 of the control gzip stream) and the download/installed sizes.
|
||||
type AlpineMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
|
||||
Name string // P:
|
||||
Version string // V:
|
||||
Arch string // A:
|
||||
DownloadSize int64 // S: on-disk .apk size
|
||||
InstalledSize int64 // I: unpacked size from .PKGINFO
|
||||
Description string // T:
|
||||
URL string // U:
|
||||
License string // L:
|
||||
Origin string // o:
|
||||
Maintainer string // m:
|
||||
BuildTime int64 // t:
|
||||
Commit string // c:
|
||||
ProviderPriority string // k:
|
||||
Depends []string // D:
|
||||
Provides []string // p:
|
||||
InstallIf []string // i:
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package testsupport
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// MinimalApk builds a valid-enough Alpine package in pure Go (no committed
|
||||
// binary fixture, no abuild): two concatenated, independently gzipped tar
|
||||
// streams -- a control stream carrying .PKGINFO and a data stream carrying a
|
||||
// single payload file. It mirrors MinimalDeb/MinimalRPM and is parseable by the
|
||||
// alpine provider (which derives arch/name/version and the Q1 pull checksum from
|
||||
// the control stream).
|
||||
func MinimalApk(name, version, arch string) []byte {
|
||||
pkginfo := fmt.Sprintf(
|
||||
"# generated by testsupport\n"+
|
||||
"pkgname = %s\n"+
|
||||
"pkgver = %s\n"+
|
||||
"arch = %s\n"+
|
||||
"pkgdesc = minimal test package\n"+
|
||||
"url = https://example.com/%s\n"+
|
||||
"license = MIT\n"+
|
||||
"origin = %s\n"+
|
||||
"maintainer = e2e <e2e@example.com>\n"+
|
||||
"builddate = 1700000000\n"+
|
||||
"size = 4\n"+
|
||||
"depend = so:libc.musl-x86_64.so.1\n"+
|
||||
"provides = cmd:%s=%s\n",
|
||||
name, version, arch, name, name, name, version)
|
||||
|
||||
control := gzipBytes(tarSingle(".PKGINFO", []byte(pkginfo)))
|
||||
data := gzipBytes(tarSingle("usr/bin/"+name, []byte("body")))
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.Write(control)
|
||||
buf.Write(data)
|
||||
return buf.Bytes()
|
||||
}
|
||||
Reference in New Issue
Block a user