From 7f77666709dae876ba87e39bbcaf6f63d45e4a47 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Wed, 12 Aug 2026 20:39:27 +1000 Subject: [PATCH] Add Alpine/apk local repository support (#114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The alpine provider only supported remote (proxy) repositories, so there was no way to publish first-party `.apk` packages the way `rpm-local` and `deb-local` already allow. This extends the existing alpine provider into a real apk repository: uploaded `.apk` files are parsed in pure Go and a per-arch `APKINDEX.tar.gz` is generated on demand, at parity with rpm repodata and deb Packages generation. (The metadata-only `github_alpine` type is a separate follow-up and is not part of this PR.) ## How - Implements `LocalUploader` / `LocalIndexer` / `PostUploadHook` / `PostDeleteHook` on the existing `alpine` provider, leaving the remote proxy methods (`UpstreamURL`/`ContentType`/`AuthHeaders`/`RewriteResponse`/`Classify`) intact. - Parses the `.apk` (up to three concatenated, independently gzipped tar streams) in pure Go: locates the control stream by its `.PKGINFO` member, reads the `key = value` fields, and computes the apk pull checksum `C:` = `Q1` + base64(sha1(**control gzip stream bytes**)) — the sha1 of the second gzip member, not of the whole file. - Derives arch from `.PKGINFO` and records download size (`S:` blob size) and installed size (`I:` from `.PKGINFO size`). - Generates an **unsigned** per-arch `APKINDEX.tar.gz` = gzip(tar(`APKINDEX`)) filtered by requested arch (clients use `--allow-untrusted`, matching rpm `gpgcheck=0` / deb `[trusted=yes]`), applying the same dot-segment normalization as deb so `.//APKINDEX.tar.gz` resolves. Non-index / `.apk` paths return `false` so the generic file streamer serves the stored blob. - Adds `AlpineMetadata` plus **separate** `AlpineMetadataStore` / `AlpineMetadataReader` / `AlpineMetadataDeleter` interfaces (type-asserted from the generic hooks) so the shared rpm/deb metadata interfaces and their test doubles are untouched. - Adds the `alpine_metadata` table (keyed by `repo_name` + `file_path`, per-arch index) and its `Insert`/`Delete`/`List` DB methods. - Adds `testsupport.MinimalApk`, unit tests (`.PKGINFO` parse, Q1 checksum over the control stream, per-arch filtering, empty-field omission, `./` dot-segment handling, ValidateUpload accept/reject), and a `dockere2e` `TestLocalAlpineIndex`. ## Consumption `/etc/apk/repositories` line = `/api/v1/local/` (apk appends `//APKINDEX.tar.gz`); `apk update --allow-untrusted && apk add --allow-untrusted `. Packages live at `/api/v1/local///.apk`. ## Verification `go build ./...`, `go vet ./...` (incl. `-tags dockere2e`), `go mod tidy` (no change), `make test` (`-race`), and `pre-commit run --all-files` all pass. Reviewed-on: https://git.unkin.net/unkin/artifactapi/pulls/114 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- e2e-docker/local_test.go | 55 +++ internal/database/alpine_metadata.go | 70 ++++ internal/database/postgres.go | 29 ++ internal/provider/alpine/alpine.go | 357 ++++++++++++++++++ internal/provider/alpine/alpine_local_test.go | 324 ++++++++++++++++ internal/provider/provider.go | 44 +++ internal/testsupport/apk.go | 38 ++ 7 files changed, 917 insertions(+) create mode 100644 internal/database/alpine_metadata.go create mode 100644 internal/provider/alpine/alpine_local_test.go create mode 100644 internal/testsupport/apk.go diff --git a/e2e-docker/local_test.go b/e2e-docker/local_test.go index 11daa22..562fa68 100644 --- a/e2e-docker/local_test.go +++ b/e2e-docker/local_test.go @@ -3,7 +3,10 @@ package e2edocker import ( + "archive/tar" "bytes" + "compress/gzip" + "io" "net/http" "strings" "testing" @@ -136,3 +139,55 @@ func TestLocalDebRepo(t *testing.T) { t.Fatalf("deb content mismatch") } } + +// TestLocalAlpineIndex uploads an .apk to an alpine local repo and validates +// that a per-arch APKINDEX.tar.gz is generated automatically from the parsed +// .PKGINFO (the apk-local analog of rpm repodata / deb Packages generation). +func TestLocalAlpineIndex(t *testing.T) { + createRepo(t, `{"name":"local-alpine","package_type":"alpine","repo_type":"local"}`) + defer deleteRepo(t, "local-alpine") + + apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64") + uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive") + + // The index is generated asynchronously after upload; poll for it. + resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second) + if resp.StatusCode != http.StatusOK { + t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body) + } + + zr, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + t.Fatalf("APKINDEX not gzip: %v", err) + } + tarBytes, _ := io.ReadAll(zr) + tr := tar.NewReader(bytes.NewReader(tarBytes)) + var index string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("APKINDEX not tar: %v", err) + } + if hdr.Name == "APKINDEX" { + b, _ := io.ReadAll(tr) + index = string(b) + } + } + for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} { + if !strings.Contains(index, want) { + t.Fatalf("APKINDEX missing %q:\n%s", want, index) + } + } + + // The .apk downloads back byte-identical from its arch path. + resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "") + if resp.StatusCode != http.StatusOK { + t.Fatalf("download apk: status %d: %s", resp.StatusCode, body) + } + if !bytes.Equal(body, apk) { + t.Fatalf("apk content mismatch") + } +} diff --git a/internal/database/alpine_metadata.go b/internal/database/alpine_metadata.go new file mode 100644 index 0000000..ecf19d1 --- /dev/null +++ b/internal/database/alpine_metadata.go @@ -0,0 +1,70 @@ +package database + +import ( + "context" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func (db *DB) InsertAlpineMetadata(ctx context.Context, meta *provider.AlpineMetadata) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO alpine_metadata ( + repo_name, file_path, content_hash, checksum, + name, version, arch, download_size, installed_size, + description, url, license, origin, maintainer, + build_time, commit_hash, provider_priority, + depends, provides, install_if + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20) + ON CONFLICT (repo_name, file_path) DO NOTHING + `, + meta.RepoName, meta.FilePath, meta.ContentHash, meta.Checksum, + meta.Name, meta.Version, meta.Arch, meta.DownloadSize, meta.InstalledSize, + meta.Description, meta.URL, meta.License, meta.Origin, meta.Maintainer, + meta.BuildTime, meta.Commit, meta.ProviderPriority, + strings.Join(meta.Depends, " "), strings.Join(meta.Provides, " "), strings.Join(meta.InstallIf, " "), + ) + return err +} + +func (db *DB) DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM alpine_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} + +func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]provider.AlpineMetadata, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, content_hash, checksum, + name, version, arch, download_size, installed_size, + description, url, license, origin, maintainer, + build_time, commit_hash, provider_priority, + depends, provides, install_if + FROM alpine_metadata + WHERE repo_name = $1 + ORDER BY name, version, arch + `, repoName) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.AlpineMetadata + for rows.Next() { + var m provider.AlpineMetadata + var depends, provides, installIf string + if err := rows.Scan( + &m.RepoName, &m.FilePath, &m.ContentHash, &m.Checksum, + &m.Name, &m.Version, &m.Arch, &m.DownloadSize, &m.InstalledSize, + &m.Description, &m.URL, &m.License, &m.Origin, &m.Maintainer, + &m.BuildTime, &m.Commit, &m.ProviderPriority, + &depends, &provides, &installIf, + ); err != nil { + return nil, err + } + m.Depends = strings.Fields(depends) + m.Provides = strings.Fields(provides) + m.InstallIf = strings.Fields(installIf) + result = append(result, m) + } + return result, rows.Err() +} diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 0f6c030..a1679d6 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -182,6 +182,35 @@ func (db *DB) migrate() error { CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name); + CREATE TABLE IF NOT EXISTS alpine_metadata ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL, + checksum TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + arch TEXT NOT NULL, + download_size BIGINT DEFAULT 0, + installed_size BIGINT DEFAULT 0, + description TEXT DEFAULT '', + url TEXT DEFAULT '', + license TEXT DEFAULT '', + origin TEXT DEFAULT '', + maintainer TEXT DEFAULT '', + build_time BIGINT DEFAULT 0, + commit_hash TEXT DEFAULT '', + provider_priority TEXT DEFAULT '', + depends TEXT DEFAULT '', + provides TEXT DEFAULT '', + install_if TEXT DEFAULT '', + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) + ); + + CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name); + CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch); + CREATE TABLE IF NOT EXISTS github_rpm_sync_state ( remote_name TEXT PRIMARY KEY, etag TEXT DEFAULT '', diff --git a/internal/provider/alpine/alpine.go b/internal/provider/alpine/alpine.go index 3c5e3e9..b3198e2 100644 --- a/internal/provider/alpine/alpine.go +++ b/internal/provider/alpine/alpine.go @@ -1,12 +1,26 @@ package alpine import ( + "bufio" + "bytes" + "compress/gzip" "context" + "crypto/sha1" + "encoding/base64" + "errors" + "fmt" + "io" + "log/slog" "net/http" + "path" + "strconv" "strings" + "archive/tar" + "git.unkin.net/unkin/artifactapi/internal/auth" "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -46,3 +60,346 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { return auth.BasicHeaders(remote), nil } + +// --- LocalUploader: hosting real .apk packages ----------------------------- + +// ValidateUpload accepts any *.apk and preserves the client-supplied directory +// (the arch prefix) as the storage path, since arch cannot be parsed from the +// filename alone and the generic uploader hands us only the path. apk clients +// fetch packages at /.apk, so publishers upload to that same path; +// AfterUpload records the true arch (from .PKGINFO) for index filtering. +func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { + clean := strings.TrimPrefix(path.Clean("/"+filePath), "/") + filename := clean + if i := strings.LastIndex(clean, "/"); i >= 0 { + filename = clean[i+1:] + } + if !strings.HasSuffix(strings.ToLower(filename), ".apk") { + return "", "", fmt.Errorf("file must be a .apk package") + } + return clean, "application/vnd.android.package-archive", nil +} + +func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any { + filename := storagePath + if i := strings.LastIndex(storagePath, "/"); i >= 0 { + filename = storagePath[i+1:] + } + return map[string]any{ + "filename": filename, + "content_hash": contentHash, + "size_bytes": sizeBytes, + } +} + +func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) { + s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")) + + reader, blobSize, err := blobs.Download(ctx, s3Key) + if err != nil { + slog.Error("alpine metadata: download failed", "repo", repoName, "path", storagePath, "error", err) + return + } + defer reader.Close() + + raw, err := io.ReadAll(reader) + if err != nil { + slog.Error("alpine metadata: read failed", "repo", repoName, "path", storagePath, "error", err) + return + } + + meta, err := parseApk(raw) + if err != nil { + slog.Error("alpine metadata: parse failed", "repo", repoName, "path", storagePath, "error", err) + return + } + meta.RepoName = repoName + meta.FilePath = storagePath + meta.ContentHash = contentHash + meta.DownloadSize = blobSize + + if meta.Name == "" || meta.Arch == "" { + slog.Error("alpine metadata: .PKGINFO missing pkgname/arch", "repo", repoName, "path", storagePath) + return + } + + store, ok := db.(provider.AlpineMetadataStore) + if !ok { + slog.Error("alpine metadata: store does not support alpine metadata", "repo", repoName) + return + } + if err := store.InsertAlpineMetadata(ctx, meta); err != nil { + slog.Error("alpine metadata: insert failed", "repo", repoName, "path", storagePath, "error", err) + return + } + slog.Info("alpine metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch) +} + +func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error { + deleter, ok := db.(provider.AlpineMetadataDeleter) + if !ok { + return nil + } + if err := deleter.DeleteAlpineMetadata(ctx, repoName, storagePath); err != nil { + slog.Error("alpine metadata: delete failed", "repo", repoName, "path", storagePath, "error", err) + return err + } + slog.Info("alpine metadata: deleted", "repo", repoName, "path", storagePath) + return nil +} + +// --- LocalIndexer: generating a per-arch APKINDEX.tar.gz ------------------- + +// normalizeIndexPath collapses apk's dot-segment prefix: an /etc/apk/repositories +// line of "/api/v1/local/" makes apk request ".//APKINDEX.tar.gz". +// Mirrors deb's flat-repo normalization. +func normalizeIndexPath(p string) string { + return strings.TrimPrefix(path.Clean("/"+p), "/") +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool { + clean := normalizeIndexPath(reqPath) + if !strings.HasSuffix(clean, "APKINDEX.tar.gz") { + return false + } + arch := strings.TrimSuffix(clean, "APKINDEX.tar.gz") + arch = strings.Trim(arch, "/") + if arch == "" || strings.Contains(arch, "/") { + http.Error(w, "APKINDEX must be requested per-arch: /APKINDEX.tar.gz", http.StatusNotFound) + return true + } + + reader, ok := files.(provider.AlpineMetadataReader) + if !ok { + http.Error(w, "alpine metadata not available", http.StatusInternalServerError) + return true + } + + metas, err := reader.ListAlpineMetadataEntries(r.Context(), repoName) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + slog.Warn("alpine: metadata read canceled", "repo", repoName, "error", err) + http.Error(w, "metadata read canceled", http.StatusServiceUnavailable) + return true + } + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + + var filtered []provider.AlpineMetadata + for _, m := range metas { + if m.Arch == arch { + filtered = append(filtered, m) + } + } + + w.Header().Set("Content-Type", "application/gzip") + w.WriteHeader(http.StatusOK) + w.Write(generateAPKIndex(filtered)) + return true +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) { + return nil, fmt.Errorf("alpine local index generation for virtual repos not supported") +} + +// --- pure-Go .apk parsing -------------------------------------------------- + +// parseApk reads an .apk (up to three concatenated, independently gzipped tar +// streams: optional signature, control, data). It locates the control stream by +// its .PKGINFO member, computes the apk pull checksum C: = "Q1" + +// base64(sha1()), and reads the .PKGINFO fields. +func parseApk(raw []byte) (*provider.AlpineMetadata, error) { + members, err := gzipMembers(raw) + if err != nil { + return nil, err + } + + for _, m := range members { + pkginfo, ok := pkginfoFromTar(m.tar) + if !ok { + continue + } + meta := parsePkginfo(pkginfo) + sum := sha1.Sum(m.raw) + meta.Checksum = "Q1" + base64.StdEncoding.EncodeToString(sum[:]) + return meta, nil + } + return nil, errors.New("no .PKGINFO found in any .apk gzip stream") +} + +type gzMember struct { + raw []byte // the raw bytes of this gzip stream (for the Q1 checksum) + tar []byte // the decompressed tar payload +} + +// gzipMembers splits the concatenated gzip streams, returning each stream's raw +// bytes alongside its decompressed tar. It relies on bytes.Reader being an +// io.ByteReader (so compress/gzip does not over-read past a member's trailer) +// to recover exact stream boundaries via Multistream(false)+Reset. +func gzipMembers(data []byte) ([]gzMember, error) { + br := bytes.NewReader(data) + zr, err := gzip.NewReader(br) + if err != nil { + return nil, err + } + var members []gzMember + prev := 0 + for { + zr.Multistream(false) + out, err := io.ReadAll(zr) + if err != nil { + return nil, err + } + end := len(data) - br.Len() + members = append(members, gzMember{raw: data[prev:end], tar: out}) + prev = end + if err := zr.Reset(br); err != nil { + if err == io.EOF { + break + } + return nil, err + } + } + return members, nil +} + +func pkginfoFromTar(tarBytes []byte) (string, bool) { + tr := tar.NewReader(bytes.NewReader(tarBytes)) + for { + hdr, err := tr.Next() + if err != nil { + return "", false + } + if strings.TrimPrefix(hdr.Name, "./") == ".PKGINFO" { + b, err := io.ReadAll(tr) + if err != nil { + return "", false + } + return string(b), true + } + } +} + +// parsePkginfo reads the "key = value" .PKGINFO text, collecting the repeated +// depend/provides/install_if keys into slices. +func parsePkginfo(text string) *provider.AlpineMetadata { + m := &provider.AlpineMetadata{} + sc := bufio.NewScanner(strings.NewReader(text)) + sc.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + idx := strings.Index(line, "=") + if idx < 0 { + continue + } + key := strings.TrimSpace(line[:idx]) + val := strings.TrimSpace(line[idx+1:]) + switch key { + case "pkgname": + m.Name = val + case "pkgver": + m.Version = val + case "arch": + m.Arch = val + case "pkgdesc": + m.Description = val + case "url": + m.URL = val + case "license": + m.License = val + case "origin": + m.Origin = val + case "maintainer": + m.Maintainer = val + case "builddate": + if n, err := strconv.ParseInt(val, 10, 64); err == nil { + m.BuildTime = n + } + case "commit": + m.Commit = val + case "size": + if n, err := strconv.ParseInt(val, 10, 64); err == nil { + m.InstalledSize = n + } + case "provider_priority": + m.ProviderPriority = val + case "depend": + if val != "" { + m.Depends = append(m.Depends, val) + } + case "provides": + if val != "" { + m.Provides = append(m.Provides, val) + } + case "install_if": + if val != "" { + m.InstallIf = append(m.InstallIf, val) + } + } + } + return m +} + +// generateAPKIndex builds the APKINDEX.tar.gz = gzip(tar(APKINDEX)) for the +// given (already arch-filtered) rows. Records are blank-line separated; fields +// follow the canonical C/P/V/A/S/I/T/U/L/o/m/t/c/k/D/p/i order and empties are +// omitted. Unsigned (clients use --allow-untrusted), matching rpm gpgcheck=0. +func generateAPKIndex(metas []provider.AlpineMetadata) []byte { + var idx bytes.Buffer + for i, m := range metas { + if i > 0 { + idx.WriteString("\n") + } + writeField(&idx, "C", m.Checksum) + writeField(&idx, "P", m.Name) + writeField(&idx, "V", m.Version) + writeField(&idx, "A", m.Arch) + writeField(&idx, "S", intField(m.DownloadSize)) + writeField(&idx, "I", intField(m.InstalledSize)) + writeField(&idx, "T", m.Description) + writeField(&idx, "U", m.URL) + writeField(&idx, "L", m.License) + writeField(&idx, "o", m.Origin) + writeField(&idx, "m", m.Maintainer) + writeField(&idx, "t", intField(m.BuildTime)) + writeField(&idx, "c", m.Commit) + writeField(&idx, "k", m.ProviderPriority) + writeField(&idx, "D", strings.Join(m.Depends, " ")) + writeField(&idx, "p", strings.Join(m.Provides, " ")) + writeField(&idx, "i", strings.Join(m.InstallIf, " ")) + } + + var tarBuf bytes.Buffer + tw := tar.NewWriter(&tarBuf) + body := idx.Bytes() + tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}) + tw.Write(body) + tw.Close() + + var gzBuf bytes.Buffer + gz := gzip.NewWriter(&gzBuf) + gz.Write(tarBuf.Bytes()) + gz.Close() + return gzBuf.Bytes() +} + +func writeField(b *bytes.Buffer, key, val string) { + if val == "" { + return + } + b.WriteString(key) + b.WriteString(":") + b.WriteString(val) + b.WriteString("\n") +} + +func intField(n int64) string { + if n == 0 { + return "" + } + return strconv.FormatInt(n, 10) +} diff --git a/internal/provider/alpine/alpine_local_test.go b/internal/provider/alpine/alpine_local_test.go new file mode 100644 index 0000000..1316d53 --- /dev/null +++ b/internal/provider/alpine/alpine_local_test.go @@ -0,0 +1,324 @@ +package alpine + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha1" + "encoding/base64" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/testsupport" +) + +type fakeBlobReader struct{ data []byte } + +func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil +} + +type errBlobReader struct{} + +func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) { + return nil, 0, io.ErrUnexpectedEOF +} + +// fakeAlpineStore satisfies provider.MetadataStore (shared) and +// provider.AlpineMetadataStore, recording the row AfterUpload writes. +type fakeAlpineStore struct{ inserted *provider.AlpineMetadata } + +func (f *fakeAlpineStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil } +func (f *fakeAlpineStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil } +func (f *fakeAlpineStore) InsertAlpineMetadata(_ context.Context, m *provider.AlpineMetadata) error { + f.inserted = m + return nil +} + +// fakeAlpineDeleter satisfies provider.MetadataDeleter and AlpineMetadataDeleter. +type fakeAlpineDeleter struct{ deleted bool } + +func (f *fakeAlpineDeleter) DeleteRPMMetadata(context.Context, string, string) error { return nil } +func (f *fakeAlpineDeleter) DeleteDebMetadata(context.Context, string, string) error { return nil } +func (f *fakeAlpineDeleter) DeleteAlpineMetadata(context.Context, string, string) error { + f.deleted = true + return nil +} + +// fakeAlpineReader is a FileStore that also serves alpine metadata rows. +type fakeAlpineReader struct{ metas []provider.AlpineMetadata } + +func (f fakeAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) { + return f.metas, nil +} +func (f fakeAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (f fakeAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +type errAlpineReader struct{} + +func (errAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) { + return nil, io.ErrUnexpectedEOF +} +func (errAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (errAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestAlpineValidateUpload(t *testing.T) { + p := &Provider{} + sp, ct, err := p.ValidateUpload("x86_64/foo-1.0-r0.apk") + if err != nil || sp != "x86_64/foo-1.0-r0.apk" || ct != "application/vnd.android.package-archive" { + t.Errorf("sp=%q ct=%q err=%v", sp, ct, err) + } + // Dot-segment prefix is normalized away. + if sp, _, err := p.ValidateUpload("./aarch64/bar-2.0-r1.apk"); err != nil || sp != "aarch64/bar-2.0-r1.apk" { + t.Errorf("dot-seg: sp=%q err=%v", sp, err) + } + if _, _, err := p.ValidateUpload("foo.rpm"); err == nil { + t.Error("expected error for non-apk") + } + resp := p.UploadResponse("x86_64/foo-1.0-r0.apk", "sha256:abc", 42) + if resp["filename"] != "foo-1.0-r0.apk" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) { + t.Errorf("upload response %v", resp) + } +} + +func TestAlpineAfterUpload(t *testing.T) { + data := testsupport.MinimalApk("hello", "1.0-r0", "x86_64") + store := &fakeAlpineStore{} + (&Provider{}).AfterUpload(context.Background(), "myrepo", "x86_64/hello-1.0-r0.apk", + "sha256:deadbeef", fakeBlobReader{data: data}, store) + + m := store.inserted + if m == nil { + t.Fatal("no metadata inserted") + } + if m.Name != "hello" || m.Version != "1.0-r0" || m.Arch != "x86_64" { + t.Errorf("unexpected metadata: %+v", m) + } + if m.DownloadSize != int64(len(data)) { + t.Errorf("DownloadSize = %d, want %d", m.DownloadSize, len(data)) + } + if m.InstalledSize != 4 { + t.Errorf("InstalledSize = %d, want 4", m.InstalledSize) + } + if m.License != "MIT" || m.Origin != "hello" || !strings.HasPrefix(m.Maintainer, "e2e") { + t.Errorf("scalar fields not parsed: %+v", m) + } + if len(m.Depends) != 1 || m.Depends[0] != "so:libc.musl-x86_64.so.1" { + t.Errorf("Depends = %v", m.Depends) + } + if len(m.Provides) != 1 || m.Provides[0] != "cmd:hello=1.0-r0" { + t.Errorf("Provides = %v", m.Provides) + } + + // The Q1 checksum is the sha1 of the CONTROL gzip stream (the member whose + // tar carries .PKGINFO), not of the whole file. + controlRaw := controlStreamBytes(t, data) + sum := sha1.Sum(controlRaw) + want := "Q1" + base64.StdEncoding.EncodeToString(sum[:]) + if m.Checksum != want { + t.Errorf("Checksum = %q, want %q (sha1 of control stream)", m.Checksum, want) + } + // And explicitly NOT the sha1 of the whole apk. + whole := sha1.Sum(data) + if m.Checksum == "Q1"+base64.StdEncoding.EncodeToString(whole[:]) { + t.Error("Checksum was computed over the whole file, not the control stream") + } +} + +func TestAlpineAfterUploadErrors(t *testing.T) { + store := &fakeAlpineStore{} + (&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", errBlobReader{}, store) + if store.inserted != nil { + t.Error("no metadata should be inserted on download error") + } + store2 := &fakeAlpineStore{} + (&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", fakeBlobReader{data: []byte("not an apk")}, store2) + if store2.inserted != nil { + t.Error("no metadata should be inserted on parse error") + } +} + +func TestAlpineAfterDelete(t *testing.T) { + d := &fakeAlpineDeleter{} + if err := (&Provider{}).AfterDelete(context.Background(), "r", "x86_64/p.apk", d); err != nil { + t.Fatalf("AfterDelete: %v", err) + } + if !d.deleted { + t.Error("DeleteAlpineMetadata not called") + } +} + +func TestAlpineServeLocalIndex(t *testing.T) { + p := &Provider{} + reader := fakeAlpineReader{metas: []provider.AlpineMetadata{ + {Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10, + Description: "pkg aaa", URL: "https://a", License: "MIT", Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}}, + {Name: "bbb", Version: "2.0-r0", Arch: "aarch64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20}, + }} + + // x86_64 index contains only aaa, with its fields, and not bbb. + w := serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz") + if w.Code != 200 { + t.Fatalf("code %d", w.Code) + } + idx := untarIndex(t, w.Body.Bytes()) + for _, want := range []string{"C:Q1aaa", "P:aaa", "V:1.0-r0", "A:x86_64", "S:100", "I:10", "T:pkg aaa", "U:https://a", "L:MIT", "D:so:libc", "p:cmd:aaa"} { + if !strings.Contains(idx, want) { + t.Errorf("x86_64 APKINDEX missing %q:\n%s", want, idx) + } + } + if strings.Contains(idx, "P:bbb") { + t.Errorf("x86_64 APKINDEX leaked aarch64 package:\n%s", idx) + } + + // aarch64 index contains only bbb. + w = serveIndex(t, p, reader, "aarch64/APKINDEX.tar.gz") + idx = untarIndex(t, w.Body.Bytes()) + if !strings.Contains(idx, "P:bbb") || strings.Contains(idx, "P:aaa") { + t.Errorf("aarch64 filtering wrong:\n%s", idx) + } + + // Non-index and .apk paths are not owned by the indexer. + for _, path := range []string{"x86_64/foo-1.0-r0.apk", "x86_64/", "README"} { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/"+path, nil) + if p.ServeLocalIndex(w, r, reader, "repo", path) { + t.Errorf("ServeLocalIndex should return false for %q", path) + } + } +} + +// Empty fields are omitted from the record (bbb has no description/url). +func TestAlpineIndexOmitsEmptyFields(t *testing.T) { + p := &Provider{} + reader := fakeAlpineReader{metas: []provider.AlpineMetadata{ + {Name: "bbb", Version: "2.0-r0", Arch: "x86_64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20}, + }} + idx := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes()) + for _, absent := range []string{"T:", "U:", "L:", "D:", "p:", "i:", "o:", "m:", "c:", "k:"} { + if strings.Contains(idx, absent) { + t.Errorf("empty field %q should be omitted:\n%s", absent, idx) + } + } +} + +// apk requests ".//APKINDEX.tar.gz" for a bare repo base URL; the +// dot-segment must be collapsed and yield the same bytes as the plain path. +func TestAlpineServeLocalIndexDotSegment(t *testing.T) { + p := &Provider{} + reader := fakeAlpineReader{metas: []provider.AlpineMetadata{ + {Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10}, + }} + plain := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes()) + dotted := untarIndex(t, serveIndex(t, p, reader, "./x86_64/APKINDEX.tar.gz").Body.Bytes()) + if plain != dotted { + t.Errorf("dot-segment path differs:\nplain=%q\ndotted=%q", plain, dotted) + } +} + +func TestAlpineServeLocalIndexArchRequired(t *testing.T) { + p := &Provider{} + reader := fakeAlpineReader{} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/APKINDEX.tar.gz", nil) + if !p.ServeLocalIndex(w, r, reader, "repo", "APKINDEX.tar.gz") { + t.Fatal("bare APKINDEX should be owned (and rejected) by the indexer") + } + if w.Code != http.StatusNotFound { + t.Errorf("bare APKINDEX code = %d, want 404", w.Code) + } +} + +func TestAlpineServeMetadataError(t *testing.T) { + p := &Provider{} + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/x86_64/APKINDEX.tar.gz", nil) + p.ServeLocalIndex(w, r, errAlpineReader{}, "repo", "x86_64/APKINDEX.tar.gz") + if w.Code != 500 { + t.Errorf("failing reader code = %d, want 500", w.Code) + } +} + +func TestAlpineGenerateLocalIndexUnsupported(t *testing.T) { + if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeAlpineReader{}, "r", "x86_64/APKINDEX.tar.gz"); err == nil { + t.Error("expected unsupported error") + } +} + +func serveIndex(t *testing.T, p *Provider, files provider.FileStore, path string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/"+path, nil) + if !p.ServeLocalIndex(w, r, files, "repo", path) { + t.Fatalf("ServeLocalIndex returned false for %q", path) + } + return w +} + +// untarIndex un-gzips and un-tars an APKINDEX.tar.gz and returns the APKINDEX text. +func untarIndex(t *testing.T, gzTar []byte) string { + t.Helper() + zr, err := gzip.NewReader(bytes.NewReader(gzTar)) + if err != nil { + t.Fatalf("APKINDEX not gzip: %v", err) + } + tarBytes, _ := io.ReadAll(zr) + tr := tar.NewReader(bytes.NewReader(tarBytes)) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("APKINDEX not tar: %v", err) + } + if hdr.Name == "APKINDEX" { + b, _ := io.ReadAll(tr) + return string(b) + } + } + t.Fatal("no APKINDEX member in tarball") + return "" +} + +// controlStreamBytes returns the raw bytes of the gzip stream whose tar carries +// .PKGINFO, so the test can independently compute the expected Q1 checksum. +func controlStreamBytes(t *testing.T, apk []byte) []byte { + t.Helper() + br := bytes.NewReader(apk) + zr, err := gzip.NewReader(br) + if err != nil { + t.Fatalf("gzip: %v", err) + } + prev := 0 + for { + zr.Multistream(false) + out, _ := io.ReadAll(zr) + end := len(apk) - br.Len() + tr := tar.NewReader(bytes.NewReader(out)) + for { + h, err := tr.Next() + if err != nil { + break + } + if strings.TrimPrefix(h.Name, "./") == ".PKGINFO" { + return apk[prev:end] + } + } + prev = end + if err := zr.Reset(br); err != nil { + break + } + } + t.Fatal("no control stream found") + return nil +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 20cb537..f8bab30 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -115,6 +115,50 @@ type DebMetadata struct { SHA256 string } +// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the +// Alpine-specific persistence surfaces. They are kept separate from the shared +// RPM/Deb metadata interfaces so the apk provider can type-assert the generic +// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and +// thus perturbing the test doubles of) the rpm and deb providers. *database.DB +// satisfies all three. +type AlpineMetadataStore interface { + InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error +} + +type AlpineMetadataDeleter interface { + DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error +} + +type AlpineMetadataReader interface { + ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error) +} + +// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding +// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the +// sha1 of the control gzip stream) and the download/installed sizes. +type AlpineMetadata struct { + RepoName string + FilePath string + ContentHash string + Checksum string // C: "Q1" + base64(sha1(control gzip stream)) + Name string // P: + Version string // V: + Arch string // A: + DownloadSize int64 // S: on-disk .apk size + InstalledSize int64 // I: unpacked size from .PKGINFO + Description string // T: + URL string // U: + License string // L: + Origin string // o: + Maintainer string // m: + BuildTime int64 // t: + Commit string // c: + ProviderPriority string // k: + Depends []string // D: + Provides []string // p: + InstallIf []string // i: +} + type RPMMetadata struct { RepoName string FilePath string diff --git a/internal/testsupport/apk.go b/internal/testsupport/apk.go new file mode 100644 index 0000000..b1a6d48 --- /dev/null +++ b/internal/testsupport/apk.go @@ -0,0 +1,38 @@ +package testsupport + +import ( + "bytes" + "fmt" +) + +// MinimalApk builds a valid-enough Alpine package in pure Go (no committed +// binary fixture, no abuild): two concatenated, independently gzipped tar +// streams -- a control stream carrying .PKGINFO and a data stream carrying a +// single payload file. It mirrors MinimalDeb/MinimalRPM and is parseable by the +// alpine provider (which derives arch/name/version and the Q1 pull checksum from +// the control stream). +func MinimalApk(name, version, arch string) []byte { + pkginfo := fmt.Sprintf( + "# generated by testsupport\n"+ + "pkgname = %s\n"+ + "pkgver = %s\n"+ + "arch = %s\n"+ + "pkgdesc = minimal test package\n"+ + "url = https://example.com/%s\n"+ + "license = MIT\n"+ + "origin = %s\n"+ + "maintainer = e2e \n"+ + "builddate = 1700000000\n"+ + "size = 4\n"+ + "depend = so:libc.musl-x86_64.so.1\n"+ + "provides = cmd:%s=%s\n", + name, version, arch, name, name, name, version) + + control := gzipBytes(tarSingle(".PKGINFO", []byte(pkginfo))) + data := gzipBytes(tarSingle("usr/bin/"+name, []byte("body"))) + + var buf bytes.Buffer + buf.Write(control) + buf.Write(data) + return buf.Bytes() +}