feat: deb (Debian/apt) local and remote repository support
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful

Adds a deb provider with feature parity to rpm for local and remote repos,
so .deb packages can be hosted (flat apt repo generated on upload) and a
Debian/Ubuntu mirror can be cached through the proxy.

- add PackageDeb to the package-type enum
- new internal/provider/deb: Classify (.deb immutable, Packages/Release/dists
  mutable), ValidateUpload to pool/<file>, pure-Go .deb parse (ar -> control.tar
  gz/xz/zst -> ./control), and a flat-repo LocalIndexer serving Packages,
  Packages.gz and an unsigned Release ([trusted=yes], mirroring rpm gpgcheck=0)
- remote proxy path: UpstreamURL/ContentType/AuthHeaders via BasicHeaders
- DebMetadata struct + store interfaces on provider; deb_metadata table in
  migrate() and Insert/Delete/List DB methods
- blank-import the deb provider in the server
- testsupport.MinimalDeb pure-Go fixture builder
- unit tests (classify, control parse across gz/xz/zst, Packages/Release
  generation, validate) and a dockere2e TestLocalDebRepo
This commit is contained in:
unkin-agent
2026-08-11 22:00:49 +10:00
parent 109ba2ce27
commit a35f260b74
13 changed files with 1008 additions and 3 deletions
+45
View File
@@ -8,6 +8,8 @@ import (
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
@@ -91,3 +93,46 @@ func TestLocalRPMRepodata(t *testing.T) {
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
}
}
// TestLocalDebRepo uploads a .deb and validates that the flat apt index
// (Packages / Release) is generated automatically from the parsed control
// stanza (the deb-local analog of rpm repodata generation).
func TestLocalDebRepo(t *testing.T) {
createRepo(t, `{"name":"local-deb","package_type":"deb","repo_type":"local"}`)
defer deleteRepo(t, "local-deb")
deb := testsupport.MinimalDeb("e2e-testpkg", "1.0.0", "amd64")
uploadFile(t, "local-deb", "e2e-testpkg_1.0.0_amd64.deb", deb, "application/vnd.debian.binary-package")
// The index is generated asynchronously after upload; poll for it.
resp, body := getEventually(t, api("/api/v1/local/local-deb/Packages"), 15*time.Second)
if resp.StatusCode != http.StatusOK {
t.Fatalf("Packages: status %d: %s", resp.StatusCode, body)
}
pkgs := string(body)
for _, want := range []string{"Package: e2e-testpkg", "Version: 1.0.0", "Architecture: amd64", "Filename: pool/e2e-testpkg_1.0.0_amd64.deb", "SHA256:"} {
if !strings.Contains(pkgs, want) {
t.Fatalf("Packages missing %q:\n%s", want, pkgs)
}
}
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/Release"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("Release: status %d: %s", resp.StatusCode, body)
}
rel := string(body)
for _, want := range []string{"Architectures: amd64", "SHA256:", "Packages"} {
if !strings.Contains(rel, want) {
t.Fatalf("Release missing %q:\n%s", want, rel)
}
}
// The .deb downloads back byte-identical from its pool path.
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/pool/e2e-testpkg_1.0.0_amd64.deb"), nil, "")
if resp.StatusCode != http.StatusOK {
t.Fatalf("download deb: status %d: %s", resp.StatusCode, body)
}
if !bytes.Equal(body, deb) {
t.Fatalf("deb content mismatch")
}
}
+2 -1
View File
@@ -9,11 +9,13 @@ require (
github.com/go-chi/chi/v5 v5.3.0
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.10.0
github.com/klauspost/compress v1.19.2
github.com/minio/minio-go/v7 v7.2.0
github.com/redis/go-redis/v9 v9.20.0
github.com/testcontainers/testcontainers-go v0.42.0
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
github.com/ulikunitz/xz v0.5.16
golang.org/x/crypto v0.51.0
golang.org/x/time v0.15.0
gopkg.in/yaml.v3 v3.0.1
@@ -51,7 +53,6 @@ require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.18.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
+4 -2
View File
@@ -85,8 +85,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -189,6 +189,8 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+57
View File
@@ -0,0 +1,57 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertDebMetadata(ctx context.Context, meta *provider.DebMetadata) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO deb_metadata (
repo_name, file_path, content_hash,
name, version, architecture, control,
size, md5, sha256
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
ON CONFLICT (repo_name, file_path) DO NOTHING
`,
meta.RepoName, meta.FilePath, meta.ContentHash,
meta.Name, meta.Version, meta.Architecture, meta.Control,
meta.Size, meta.MD5, meta.SHA256,
)
return err
}
func (db *DB) DeleteDebMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM deb_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]provider.DebMetadata, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash,
name, version, architecture, control,
size, md5, sha256
FROM deb_metadata
WHERE repo_name = $1
ORDER BY name, version, architecture
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []provider.DebMetadata
for rows.Next() {
var m provider.DebMetadata
if err := rows.Scan(
&m.RepoName, &m.FilePath, &m.ContentHash,
&m.Name, &m.Version, &m.Architecture, &m.Control,
&m.Size, &m.MD5, &m.SHA256,
); err != nil {
return nil, err
}
result = append(result, m)
}
return result, rows.Err()
}
+18
View File
@@ -164,6 +164,24 @@ func (db *DB) migrate() error {
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
CREATE TABLE IF NOT EXISTS deb_metadata (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
architecture TEXT NOT NULL,
control TEXT NOT NULL,
size BIGINT DEFAULT 0,
md5 TEXT DEFAULT '',
sha256 TEXT DEFAULT '',
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
remote_name TEXT PRIMARY KEY,
etag TEXT DEFAULT '',
+426
View File
@@ -0,0 +1,426 @@
package deb
import (
"archive/tar"
"bufio"
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
// mutableRe marks the apt index surface (both the flat local repo and a proxied
// Debian/Ubuntu mirror's dists/ tree) so the caching engine revalidates it
// instead of freezing it like an immutable .deb.
var mutableRe = []*regexp.Regexp{
regexp.MustCompile(`(^|/)Packages(\.gz|\.xz|\.bz2)?$`),
regexp.MustCompile(`(^|/)Sources(\.gz|\.xz|\.bz2)?$`),
regexp.MustCompile(`(^|/)Release$`),
regexp.MustCompile(`(^|/)InRelease$`),
regexp.MustCompile(`(^|/)Release\.gpg$`),
regexp.MustCompile(`(^|/)Contents-`),
regexp.MustCompile(`^dists/`),
regexp.MustCompile(`/by-hash/`),
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageDeb }
func (p *Provider) Classify(path string) provider.Mutability {
for _, re := range mutableRe {
if re.MatchString(path) {
return provider.Mutable
}
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
switch {
case strings.HasSuffix(path, ".deb"):
return "application/vnd.debian.binary-package"
case strings.HasSuffix(path, ".gz"):
return "application/gzip"
case strings.HasSuffix(path, ".xz"):
return "application/x-xz"
case strings.HasSuffix(path, "Packages"), strings.HasSuffix(path, "Release"),
strings.HasSuffix(path, "InRelease"), strings.HasSuffix(path, "Sources"):
return "text/plain"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := filePath
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
filename = filePath[idx+1:]
}
if !strings.HasSuffix(strings.ToLower(filename), ".deb") {
return "", "", fmt.Errorf("file must be a .deb package")
}
return "pool/" + filename, "application/vnd.debian.binary-package", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
filename := strings.TrimPrefix(storagePath, "pool/")
return map[string]any{
"filename": filename,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
reader, blobSize, err := blobs.Download(ctx, s3Key)
if err != nil {
slog.Error("deb metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer reader.Close()
raw, err := io.ReadAll(reader)
if err != nil {
slog.Error("deb metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
return
}
control, err := extractControl(raw)
if err != nil {
slog.Error("deb metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
fields := parseControlFields(control)
sum := md5.Sum(raw)
meta := &provider.DebMetadata{
RepoName: repoName,
FilePath: storagePath,
ContentHash: contentHash,
Name: fields["Package"],
Version: fields["Version"],
Architecture: fields["Architecture"],
Control: strings.TrimRight(control, "\n"),
Size: blobSize,
MD5: hex.EncodeToString(sum[:]),
SHA256: strings.TrimPrefix(contentHash, "sha256:"),
}
if meta.Name == "" {
slog.Error("deb metadata: control missing Package field", "repo", repoName, "path", storagePath)
return
}
if err := db.InsertDebMetadata(ctx, meta); err != nil {
slog.Error("deb metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("deb metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
if err := db.DeleteDebMetadata(ctx, repoName, storagePath); err != nil {
slog.Error("deb metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
return err
}
slog.Info("deb metadata: deleted", "repo", repoName, "path", storagePath)
return nil
}
// extractControl reads a .deb (an ar archive), locates the control.tar.* member,
// decompresses it, and returns the raw ./control paragraph. Pure Go: no dpkg.
func extractControl(deb []byte) (string, error) {
members, err := readAr(deb)
if err != nil {
return "", err
}
var name string
var data []byte
for _, m := range members {
if strings.HasPrefix(m.name, "control.tar") {
name = m.name
data = m.data
break
}
}
if data == nil {
return "", errors.New("no control.tar member in .deb")
}
tarBytes, err := decompress(name, data)
if err != nil {
return "", err
}
tr := tar.NewReader(bytes.NewReader(tarBytes))
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return "", fmt.Errorf("read control.tar: %w", err)
}
clean := strings.TrimPrefix(hdr.Name, "./")
if clean == "control" {
b, err := io.ReadAll(tr)
if err != nil {
return "", fmt.Errorf("read control file: %w", err)
}
return string(b), nil
}
}
return "", errors.New("no ./control in control.tar")
}
func decompress(name string, data []byte) ([]byte, error) {
switch {
case strings.HasSuffix(name, ".gz"):
zr, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
defer zr.Close()
return io.ReadAll(zr)
case strings.HasSuffix(name, ".xz"):
xr, err := xz.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
return io.ReadAll(xr)
case strings.HasSuffix(name, ".zst"):
zr, err := zstd.NewReader(bytes.NewReader(data))
if err != nil {
return nil, err
}
defer zr.Close()
return io.ReadAll(zr)
case strings.HasSuffix(name, ".tar"):
return data, nil
}
return nil, fmt.Errorf("unsupported control.tar compression: %s", name)
}
type arMember struct {
name string
data []byte
}
// readAr parses the (trivial) Unix ar archive that wraps a .deb. Each member has
// a 60-byte header; the size field is decimal ASCII and data is padded to an
// even offset.
func readAr(data []byte) ([]arMember, error) {
const magic = "!<arch>\n"
if len(data) < len(magic) || string(data[:len(magic)]) != magic {
return nil, errors.New("not an ar archive")
}
off := len(magic)
var members []arMember
for off+60 <= len(data) {
hdr := data[off : off+60]
off += 60
name := strings.TrimRight(string(hdr[0:16]), " ")
name = strings.TrimSuffix(name, "/")
size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
if err != nil {
return nil, fmt.Errorf("bad ar size for %q: %w", name, err)
}
if off+int(size) > len(data) {
return nil, fmt.Errorf("truncated ar member %q", name)
}
members = append(members, arMember{name: name, data: data[off : off+int(size)]})
off += int(size)
if size%2 == 1 {
off++
}
}
return members, nil
}
// parseControlFields reads the single-line fields of an RFC822-style control
// paragraph. Continuation lines (leading whitespace) belong to the previous
// field and are ignored here since only Package/Version/Architecture are read.
func parseControlFields(control string) map[string]string {
fields := map[string]string{}
sc := bufio.NewScanner(strings.NewReader(control))
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
for sc.Scan() {
line := sc.Text()
if line == "" || line[0] == ' ' || line[0] == '\t' {
continue
}
idx := strings.IndexByte(line, ':')
if idx < 0 {
continue
}
key := strings.TrimSpace(line[:idx])
if _, seen := fields[key]; seen {
continue
}
fields[key] = strings.TrimSpace(line[idx+1:])
}
return fields
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
switch path {
case "Packages", "Packages.gz", "Release":
default:
return false
}
reader, ok := files.(provider.DebMetadataReader)
if !ok {
http.Error(w, "deb metadata not available", http.StatusInternalServerError)
return true
}
metas, err := reader.ListDebMetadataEntries(r.Context(), repoName)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
slog.Warn("deb: metadata read canceled", "repo", repoName, "error", err)
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
return true
}
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
switch path {
case "Packages":
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
w.Write(generatePackages(metas))
case "Packages.gz":
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(gzipBytes(generatePackages(metas)))
case "Release":
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
w.Write(generateRelease(metas))
}
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("deb local index generation for virtual repos not supported")
}
// generatePackages emits the flat-repo Packages file: each package's raw control
// stanza followed by the apt-required Filename/Size/MD5sum/SHA256 fields, blank
// line separated.
func generatePackages(metas []provider.DebMetadata) []byte {
var b bytes.Buffer
for _, m := range metas {
b.WriteString(strings.TrimRight(m.Control, "\n"))
b.WriteString("\n")
fmt.Fprintf(&b, "Filename: %s\n", m.FilePath)
fmt.Fprintf(&b, "Size: %d\n", m.Size)
fmt.Fprintf(&b, "MD5sum: %s\n", m.MD5)
fmt.Fprintf(&b, "SHA256: %s\n", m.SHA256)
b.WriteString("\n")
}
return b.Bytes()
}
func generateRelease(metas []provider.DebMetadata) []byte {
packages := generatePackages(metas)
packagesGz := gzipBytes(packages)
arches := uniqueArches(metas)
var b bytes.Buffer
fmt.Fprintf(&b, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
fmt.Fprintf(&b, "Architectures: %s\n", strings.Join(arches, " "))
b.WriteString("Acquire-By-Hash: no\n")
b.WriteString("MD5Sum:\n")
writeReleaseEntry(&b, md5Hex(packages), len(packages), "Packages")
writeReleaseEntry(&b, md5Hex(packagesGz), len(packagesGz), "Packages.gz")
b.WriteString("SHA256:\n")
writeReleaseEntry(&b, sha256Hex(packages), len(packages), "Packages")
writeReleaseEntry(&b, sha256Hex(packagesGz), len(packagesGz), "Packages.gz")
return b.Bytes()
}
func writeReleaseEntry(b *bytes.Buffer, hash string, size int, name string) {
fmt.Fprintf(b, " %s %d %s\n", hash, size, name)
}
func uniqueArches(metas []provider.DebMetadata) []string {
seen := map[string]bool{}
var out []string
for _, m := range metas {
a := m.Architecture
if a == "" || seen[a] {
continue
}
seen[a] = true
out = append(out, a)
}
return out
}
func gzipBytes(data []byte) []byte {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
gz.Write(data)
gz.Close()
return buf.Bytes()
}
func md5Hex(data []byte) string {
h := md5.Sum(data)
return hex.EncodeToString(h[:])
}
func sha256Hex(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
+361
View File
@@ -0,0 +1,361 @@
package deb
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type fakeBlobReader struct{ data []byte }
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
}
type errBlobReader struct{}
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
return nil, 0, io.ErrUnexpectedEOF
}
// fakeDebStore satisfies provider.MetadataStore (both insert methods) and
// records the deb row that AfterUpload writes.
type fakeDebStore struct{ inserted *provider.DebMetadata }
func (f *fakeDebStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
func (f *fakeDebStore) InsertDebMetadata(_ context.Context, m *provider.DebMetadata) error {
f.inserted = m
return nil
}
type fakeDebReader struct{ metas []provider.DebMetadata }
func (f fakeDebReader) ListDebMetadataEntries(context.Context, string) ([]provider.DebMetadata, error) {
return f.metas, nil
}
func (f fakeDebReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (f fakeDebReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
type errDebReader struct{}
func (errDebReader) ListDebMetadataEntries(context.Context, string) ([]provider.DebMetadata, error) {
return nil, io.ErrUnexpectedEOF
}
func (errDebReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (errDebReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestDebPureFuncs(t *testing.T) {
p := &Provider{}
if p.Type() != models.PackageDeb {
t.Errorf("type = %q", p.Type())
}
if out, _ := p.RewriteResponse(nil, models.Remote{}, "http://p"); out != nil {
t.Error("deb never rewrites")
}
if got := p.UpstreamURL(models.Remote{BaseURL: "https://mirror/"}, "/dists/bookworm/Release"); got != "https://mirror/dists/bookworm/Release" {
t.Errorf("upstream url %q", got)
}
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
if h.Get("Authorization") == "" {
t.Error("auth header")
}
}
func TestDebClassify(t *testing.T) {
p := &Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"pool/foo_1.0_amd64.deb", provider.Immutable},
{"Packages", provider.Mutable},
{"Packages.gz", provider.Mutable},
{"Release", provider.Mutable},
{"InRelease", provider.Mutable},
{"Release.gpg", provider.Mutable},
{"dists/bookworm/main/binary-amd64/Packages", provider.Mutable},
{"dists/bookworm/Release", provider.Mutable},
{"dists/bookworm/main/by-hash/SHA256/abc", provider.Mutable},
{"dists/bookworm/main/Contents-amd64.gz", provider.Mutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestDebContentType(t *testing.T) {
p := &Provider{}
for path, want := range map[string]string{
"pool/foo_1.0_amd64.deb": "application/vnd.debian.binary-package",
"dists/bookworm/main/bin/Packages.gz": "application/gzip",
"dists/bookworm/main/bin/Packages.xz": "application/x-xz",
"Packages": "text/plain",
"Release": "text/plain",
"InRelease": "text/plain",
"pool/other": "application/octet-stream",
} {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
}
}
}
func TestDebValidateUpload(t *testing.T) {
p := &Provider{}
sp, ct, err := p.ValidateUpload("dir/foo_1.0_amd64.deb")
if err != nil || sp != "pool/foo_1.0_amd64.deb" || ct != "application/vnd.debian.binary-package" {
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
}
if _, _, err := p.ValidateUpload("foo.rpm"); err == nil {
t.Error("expected error for non-deb")
}
resp := p.UploadResponse("pool/foo_1.0_amd64.deb", "sha256:abc", 42)
if resp["filename"] != "foo_1.0_amd64.deb" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) {
t.Errorf("upload response %v", resp)
}
}
func TestDebAfterUpload(t *testing.T) {
data := testsupport.MinimalDeb("e2e-testpkg", "1.2.3", "amd64")
store := &fakeDebStore{}
(&Provider{}).AfterUpload(context.Background(), "myrepo", "pool/e2e-testpkg_1.2.3_amd64.deb",
"sha256:deadbeef", fakeBlobReader{data: data}, store)
m := store.inserted
if m == nil {
t.Fatal("no metadata inserted")
}
if m.Name != "e2e-testpkg" || m.Version != "1.2.3" || m.Architecture != "amd64" {
t.Errorf("unexpected metadata: %+v", m)
}
if m.Size != int64(len(data)) {
t.Errorf("Size = %d, want %d", m.Size, len(data))
}
if m.SHA256 != "deadbeef" {
t.Errorf("SHA256 = %q, want deadbeef", m.SHA256)
}
if m.MD5 == "" {
t.Error("MD5 not computed")
}
if !strings.Contains(m.Control, "Package: e2e-testpkg") {
t.Errorf("raw control not stored: %q", m.Control)
}
// The raw stanza is stored verbatim (no trailing newline) so Packages can
// reproduce it faithfully.
if strings.HasSuffix(m.Control, "\n") {
t.Error("control should be trimmed of trailing newline")
}
}
func TestDebAfterUploadErrors(t *testing.T) {
// Download failure: no insert, no panic.
store := &fakeDebStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", errBlobReader{}, store)
if store.inserted != nil {
t.Error("no metadata should be inserted on download error")
}
// Not a .deb (ar) archive.
store2 := &fakeDebStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", fakeBlobReader{data: []byte("not a deb")}, store2)
if store2.inserted != nil {
t.Error("no metadata should be inserted on parse error")
}
}
func TestDebControlDecompression(t *testing.T) {
// The control tarball may be gzip, xz, or zstd (goreleaser/nfpm emit gzip or
// xz); each must round-trip to the same control stanza.
for _, tc := range []struct {
name string
member string
comp func([]byte) []byte
}{
{"gzip", "control.tar.gz", gzipBytes},
{"xz", "control.tar.xz", xzBytes},
{"zstd", "control.tar.zst", zstdBytes},
} {
t.Run(tc.name, func(t *testing.T) {
deb := buildDeb("pkg", "9.9", "arm64", tc.member, tc.comp)
control, err := extractControl(deb)
if err != nil {
t.Fatalf("extractControl: %v", err)
}
fields := parseControlFields(control)
if fields["Package"] != "pkg" || fields["Version"] != "9.9" || fields["Architecture"] != "arm64" {
t.Errorf("fields = %v", fields)
}
})
}
}
func TestDebParseControlContinuationLines(t *testing.T) {
control := "Package: p\nVersion: 1\n" +
"Description: short\n very long\n .\n more\n" +
"Architecture: all\n"
f := parseControlFields(control)
if f["Package"] != "p" || f["Version"] != "1" || f["Architecture"] != "all" {
t.Errorf("continuation lines corrupted parse: %v", f)
}
if f["Description"] != "short" {
t.Errorf("Description folded continuation into value: %q", f["Description"])
}
}
func TestDebServeLocalIndex(t *testing.T) {
p := &Provider{}
reader := fakeDebReader{metas: []provider.DebMetadata{
{Name: "aaa", Version: "1.0", Architecture: "amd64", FilePath: "pool/aaa_1.0_amd64.deb",
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64", Size: 100, MD5: "md5aaa", SHA256: "sha256aaa"},
{Name: "bbb", Version: "2.0", Architecture: "arm64", FilePath: "pool/bbb_2.0_arm64.deb",
Control: "Package: bbb\nVersion: 2.0\nArchitecture: arm64", Size: 200, MD5: "md5bbb", SHA256: "sha256bbb"},
}}
serve := func(path string) *httptest.ResponseRecorder {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
t.Fatalf("ServeLocalIndex returned false for %q", path)
}
return w
}
// Packages lists both packages with their apt fields.
w := serve("Packages")
body := w.Body.String()
if w.Code != 200 {
t.Fatalf("Packages code %d", w.Code)
}
for _, want := range []string{
"Package: aaa", "Package: bbb",
"Filename: pool/aaa_1.0_amd64.deb", "Size: 100", "MD5sum: md5aaa", "SHA256: sha256aaa",
"Filename: pool/bbb_2.0_arm64.deb", "Size: 200",
} {
if !strings.Contains(body, want) {
t.Errorf("Packages missing %q:\n%s", want, body)
}
}
// Stanzas are blank-line separated.
if !strings.Contains(body, "SHA256: sha256aaa\n\n") {
t.Errorf("stanzas not blank-line separated:\n%s", body)
}
// Packages.gz decompresses to exactly the plain Packages bytes.
w = serve("Packages.gz")
if w.Code != 200 {
t.Fatalf("Packages.gz code %d", w.Code)
}
zr, err := gzip.NewReader(bytes.NewReader(w.Body.Bytes()))
if err != nil {
t.Fatalf("Packages.gz not gzip: %v", err)
}
plain, _ := io.ReadAll(zr)
if !bytes.Equal(plain, []byte(body)) {
t.Error("Packages.gz does not decompress to Packages")
}
// Release lists arches and both index files under MD5Sum/SHA256.
w = serve("Release")
rel := w.Body.String()
if w.Code != 200 {
t.Fatalf("Release code %d", w.Code)
}
for _, want := range []string{"Date:", "Architectures: amd64 arm64", "Acquire-By-Hash: no", "MD5Sum:", "SHA256:", " Packages\n", " Packages.gz\n"} {
if !strings.Contains(rel, want) {
t.Errorf("Release missing %q:\n%s", want, rel)
}
}
// Unsigned trust model: no InRelease / Release.gpg served here.
for _, path := range []string{"InRelease", "Release.gpg", "pool/aaa_1.0_amd64.deb"} {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
if p.ServeLocalIndex(w, r, reader, "myrepo", path) {
t.Errorf("ServeLocalIndex should return false for %q", path)
}
}
}
func TestDebServeMetadataError(t *testing.T) {
p := &Provider{}
for _, path := range []string{"Packages", "Packages.gz", "Release"} {
w := httptest.NewRecorder()
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
p.ServeLocalIndex(w, r, errDebReader{}, "repo", path)
if w.Code != 500 {
t.Errorf("%s with failing reader = %d, want 500", path, w.Code)
}
}
}
func TestDebGenerateLocalIndexUnsupported(t *testing.T) {
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeDebReader{}, "r", "Packages"); err == nil {
t.Error("expected unsupported error")
}
}
// buildDeb assembles an ar .deb whose control member uses the given name and
// compressor, so the decompression branches can be exercised directly.
func buildDeb(name, version, arch, member string, comp func([]byte) []byte) []byte {
control := "Package: " + name + "\nVersion: " + version + "\nArchitecture: " + arch + "\n"
controlTar := comp(tarSingle("./control", []byte(control)))
var buf bytes.Buffer
buf.WriteString("!<arch>\n")
arWrite(&buf, "debian-binary", []byte("2.0\n"))
arWrite(&buf, member, controlTar)
arWrite(&buf, "data.tar.gz", gzipBytes(tarSingle("./x", []byte("x"))))
return buf.Bytes()
}
func tarSingle(name string, data []byte) []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg})
tw.Write(data)
tw.Close()
return buf.Bytes()
}
func arWrite(buf *bytes.Buffer, name string, data []byte) {
fmt.Fprintf(buf, "%-16s%-12s%-6s%-6s%-8s%-10d`\n", name, "0", "0", "0", "100644", len(data))
buf.Write(data)
if len(data)%2 == 1 {
buf.WriteByte('\n')
}
}
func xzBytes(data []byte) []byte {
var buf bytes.Buffer
w, _ := xz.NewWriter(&buf)
w.Write(data)
w.Close()
return buf.Bytes()
}
func zstdBytes(data []byte) []byte {
var buf bytes.Buffer
w, _ := zstd.NewWriter(&buf)
w.Write(data)
w.Close()
return buf.Bytes()
}
+26
View File
@@ -61,6 +61,7 @@ type PostDeleteHook interface {
type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
}
// RemoteServer lets a remote provider fully answer a request itself instead of
@@ -83,12 +84,37 @@ type RemoteMetadataStore interface {
type MetadataDeleter interface {
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
}
type RPMMetadataReader interface {
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
}
// DebMetadataReader is the read surface the deb LocalIndexer needs to
// regenerate a flat apt repository (Packages/Release) from stored rows.
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
// mirroring how the rpm provider reaches its RPMMetadataReader.
type DebMetadataReader interface {
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
}
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
// the full raw control stanza so the Packages index can be regenerated
// faithfully alongside the computed size/md5/sha256 apt requires.
type DebMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
Architecture string
Control string
Size int64
MD5 string
SHA256 string
}
type RPMMetadata struct {
RepoName string
FilePath string
+3
View File
@@ -47,6 +47,9 @@ func (f *fakeStore) DeleteRPMMetadata(_ context.Context, _, filePath string) err
return nil
}
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
func (f *fakeStore) DeleteDebMetadata(context.Context, string, string) error { return nil }
func (f *fakeStore) ListRPMMetadataEntries(ctx context.Context, _ string) ([]provider.RPMMetadata, error) {
// Mirror pgx: a canceled/expired context fails the read. This is what
// poisons the repodata response if the read runs on the inbound request.
+2
View File
@@ -28,6 +28,8 @@ func (f *fakeMetaStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMeta
return nil
}
func (f *fakeMetaStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
type fakeRPMReader struct{ metas []provider.RPMMetadata }
func (f fakeRPMReader) ListRPMMetadataEntries(_ context.Context, _ string) ([]provider.RPMMetadata, error) {
+1
View File
@@ -21,6 +21,7 @@ import (
"git.unkin.net/unkin/artifactapi/internal/gc"
"git.unkin.net/unkin/artifactapi/internal/githubauth"
_ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
_ "git.unkin.net/unkin/artifactapi/internal/provider/deb"
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
+61
View File
@@ -0,0 +1,61 @@
package testsupport
import (
"archive/tar"
"bytes"
"compress/gzip"
"fmt"
)
// MinimalDeb builds a valid-enough Debian package in pure Go (no committed
// binary fixture, no dpkg-deb): an ar archive of debian-binary, a gzip
// control.tar.gz carrying ./control, and an (empty) gzip data.tar.gz. It is the
// deb analog of MinimalRPM and is parseable by the deb provider.
func MinimalDeb(name, version, arch string) []byte {
control := fmt.Sprintf(
"Package: %s\nVersion: %s\nArchitecture: %s\nMaintainer: e2e <e2e@example.com>\n"+
"Section: utils\nPriority: optional\nDescription: minimal test package\n",
name, version, arch)
controlTarGz := gzipBytes(tarSingle("./control", []byte(control)))
dataTarGz := gzipBytes(tarEmpty())
var buf bytes.Buffer
buf.WriteString("!<arch>\n")
arWrite(&buf, "debian-binary", []byte("2.0\n"))
arWrite(&buf, "control.tar.gz", controlTarGz)
arWrite(&buf, "data.tar.gz", dataTarGz)
return buf.Bytes()
}
func arWrite(buf *bytes.Buffer, name string, data []byte) {
fmt.Fprintf(buf, "%-16s%-12s%-6s%-6s%-8s%-10d`\n", name, "0", "0", "0", "100644", len(data))
buf.Write(data)
if len(data)%2 == 1 {
buf.WriteByte('\n')
}
}
func tarSingle(name string, data []byte) []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg})
tw.Write(data)
tw.Close()
return buf.Bytes()
}
func tarEmpty() []byte {
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
tw.Close()
return buf.Bytes()
}
func gzipBytes(data []byte) []byte {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
gz.Write(data)
gz.Close()
return buf.Bytes()
}
+2
View File
@@ -11,6 +11,7 @@ const (
PackagePyPI PackageType = "pypi"
PackageNPM PackageType = "npm"
PackageRPM PackageType = "rpm"
PackageDeb PackageType = "deb"
PackageAlpine PackageType = "alpine"
PackagePuppet PackageType = "puppet"
PackageTerraform PackageType = "terraform"
@@ -25,6 +26,7 @@ var validPackageTypes = map[PackageType]bool{
PackagePyPI: true,
PackageNPM: true,
PackageRPM: true,
PackageDeb: true,
PackageAlpine: true,
PackagePuppet: true,
PackageTerraform: true,