Add deb (Debian/apt) local and remote repository support (#111)
Brings Debian/apt to artifactapi with feature parity to the existing rpm support (local + remote), so `.deb` packages can be hosted as a flat apt repo and a Debian/Ubuntu mirror can be cached through the proxy.
- Adds `deb` to the package-type enum and registers a new `internal/provider/deb` provider.
- Classifies `.deb` blobs immutable and the apt index surface (`Packages`, `Release`, `InRelease`, `dists/`, by-hash) mutable so the caching engine revalidates it.
- Parses the `.deb` in pure Go (ar archive to `control.tar.{gz,xz,zst}` to `./control`), storing the raw control stanza plus computed size/md5/sha256 as `deb_metadata`.
- Serves a flat apt repo (`deb [trusted=yes] .../ ./`): generates `Packages`, `Packages.gz` and an unsigned `Release` (returns 404 for `InRelease`/`Release.gpg`), mirroring rpm unsigned repodata / gpgcheck=0 trust model.
- Proxies a remote mirror via `UpstreamURL`/`ContentType`/`AuthHeaders` (HTTP Basic).
- Adds the `deb_metadata` table to `migrate()`, DB access methods, a `MinimalDeb` pure-Go fixture, unit tests, and a `dockere2e` `TestLocalDebRepo`.
---------
Co-authored-by: unkin-agent <unkin-agent@git.unkin.net>
Reviewed-on: #111
Co-authored-by: Unkin Agent <unkin-agent@unkin.net>
Co-committed-by: Unkin Agent <unkin-agent@unkin.net>
This commit was merged in pull request #111.
This commit is contained in:
@@ -0,0 +1,436 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
// mutableRe marks the apt index surface (both the flat local repo and a proxied
|
||||
// Debian/Ubuntu mirror's dists/ tree) so the caching engine revalidates it
|
||||
// instead of freezing it like an immutable .deb.
|
||||
var mutableRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(^|/)Packages(\.gz|\.xz|\.bz2)?$`),
|
||||
regexp.MustCompile(`(^|/)Sources(\.gz|\.xz|\.bz2)?$`),
|
||||
regexp.MustCompile(`(^|/)Release$`),
|
||||
regexp.MustCompile(`(^|/)InRelease$`),
|
||||
regexp.MustCompile(`(^|/)Release\.gpg$`),
|
||||
regexp.MustCompile(`(^|/)Contents-`),
|
||||
regexp.MustCompile(`^dists/`),
|
||||
regexp.MustCompile(`/by-hash/`),
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageDeb }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
for _, re := range mutableRe {
|
||||
if re.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".deb"):
|
||||
return "application/vnd.debian.binary-package"
|
||||
case strings.HasSuffix(path, ".gz"):
|
||||
return "application/gzip"
|
||||
case strings.HasSuffix(path, ".xz"):
|
||||
return "application/x-xz"
|
||||
case strings.HasSuffix(path, "Packages"), strings.HasSuffix(path, "Release"),
|
||||
strings.HasSuffix(path, "InRelease"), strings.HasSuffix(path, "Sources"):
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".deb") {
|
||||
return "", "", fmt.Errorf("file must be a .deb package")
|
||||
}
|
||||
|
||||
return "pool/" + filename, "application/vnd.debian.binary-package", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := strings.TrimPrefix(storagePath, "pool/")
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
raw, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
control, err := extractControl(raw)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
fields := parseControlFields(control)
|
||||
|
||||
sum := md5.Sum(raw)
|
||||
meta := &provider.DebMetadata{
|
||||
RepoName: repoName,
|
||||
FilePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Name: fields["Package"],
|
||||
Version: fields["Version"],
|
||||
Architecture: fields["Architecture"],
|
||||
Control: strings.TrimRight(control, "\n"),
|
||||
Size: blobSize,
|
||||
MD5: hex.EncodeToString(sum[:]),
|
||||
SHA256: strings.TrimPrefix(contentHash, "sha256:"),
|
||||
}
|
||||
|
||||
if meta.Name == "" {
|
||||
slog.Error("deb metadata: control missing Package field", "repo", repoName, "path", storagePath)
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.InsertDebMetadata(ctx, meta); err != nil {
|
||||
slog.Error("deb metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("deb metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
if err := db.DeleteDebMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("deb metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("deb metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractControl reads a .deb (an ar archive), locates the control.tar.* member,
|
||||
// decompresses it, and returns the raw ./control paragraph. Pure Go: no dpkg.
|
||||
func extractControl(deb []byte) (string, error) {
|
||||
members, err := readAr(deb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var name string
|
||||
var data []byte
|
||||
for _, m := range members {
|
||||
if strings.HasPrefix(m.name, "control.tar") {
|
||||
name = m.name
|
||||
data = m.data
|
||||
break
|
||||
}
|
||||
}
|
||||
if data == nil {
|
||||
return "", errors.New("no control.tar member in .deb")
|
||||
}
|
||||
|
||||
tarBytes, err := decompress(name, data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read control.tar: %w", err)
|
||||
}
|
||||
clean := strings.TrimPrefix(hdr.Name, "./")
|
||||
if clean == "control" {
|
||||
b, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read control file: %w", err)
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("no ./control in control.tar")
|
||||
}
|
||||
|
||||
func decompress(name string, data []byte) ([]byte, error) {
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".gz"):
|
||||
zr, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zr.Close()
|
||||
return io.ReadAll(zr)
|
||||
case strings.HasSuffix(name, ".xz"):
|
||||
xr, err := xz.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(xr)
|
||||
case strings.HasSuffix(name, ".zst"):
|
||||
zr, err := zstd.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zr.Close()
|
||||
return io.ReadAll(zr)
|
||||
case strings.HasSuffix(name, ".tar"):
|
||||
return data, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported control.tar compression: %s", name)
|
||||
}
|
||||
|
||||
type arMember struct {
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
|
||||
// readAr parses the (trivial) Unix ar archive that wraps a .deb. Each member has
|
||||
// a 60-byte header; the size field is decimal ASCII and data is padded to an
|
||||
// even offset.
|
||||
func readAr(data []byte) ([]arMember, error) {
|
||||
const magic = "!<arch>\n"
|
||||
if len(data) < len(magic) || string(data[:len(magic)]) != magic {
|
||||
return nil, errors.New("not an ar archive")
|
||||
}
|
||||
off := len(magic)
|
||||
|
||||
var members []arMember
|
||||
for off+60 <= len(data) {
|
||||
hdr := data[off : off+60]
|
||||
off += 60
|
||||
|
||||
name := strings.TrimRight(string(hdr[0:16]), " ")
|
||||
name = strings.TrimSuffix(name, "/")
|
||||
size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad ar size for %q: %w", name, err)
|
||||
}
|
||||
if off+int(size) > len(data) {
|
||||
return nil, fmt.Errorf("truncated ar member %q", name)
|
||||
}
|
||||
members = append(members, arMember{name: name, data: data[off : off+int(size)]})
|
||||
off += int(size)
|
||||
if size%2 == 1 {
|
||||
off++
|
||||
}
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
// parseControlFields reads the single-line fields of an RFC822-style control
|
||||
// paragraph. Continuation lines (leading whitespace) belong to the previous
|
||||
// field and are ignored here since only Package/Version/Architecture are read.
|
||||
func parseControlFields(control string) map[string]string {
|
||||
fields := map[string]string{}
|
||||
sc := bufio.NewScanner(strings.NewReader(control))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if line == "" || line[0] == ' ' || line[0] == '\t' {
|
||||
continue
|
||||
}
|
||||
idx := strings.IndexByte(line, ':')
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
if _, seen := fields[key]; seen {
|
||||
continue
|
||||
}
|
||||
fields[key] = strings.TrimSpace(line[idx+1:])
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// normalizeIndexPath collapses apt's verbatim dist prefix from a flat-repo
|
||||
// request. For `deb ... <repo>/ ./`, apt appends the "./" dist literally and asks
|
||||
// for "./Packages" (and "./Release", "./InRelease"); dot-segments must be
|
||||
// collapsed so the index matcher sees "Packages". A no-op for pool/*.deb paths.
|
||||
func normalizeIndexPath(p string) string {
|
||||
return strings.TrimPrefix(path.Clean("/"+p), "/")
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
path := normalizeIndexPath(reqPath)
|
||||
switch path {
|
||||
case "Packages", "Packages.gz", "Release":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
reader, ok := files.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "deb metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
metas, err := reader.ListDebMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
slog.Warn("deb: metadata read canceled", "repo", repoName, "error", err)
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return true
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
switch path {
|
||||
case "Packages":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePackages(metas))
|
||||
case "Packages.gz":
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(gzipBytes(generatePackages(metas)))
|
||||
case "Release":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateRelease(metas))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("deb local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
// generatePackages emits the flat-repo Packages file: each package's raw control
|
||||
// stanza followed by the apt-required Filename/Size/MD5sum/SHA256 fields, blank
|
||||
// line separated.
|
||||
func generatePackages(metas []provider.DebMetadata) []byte {
|
||||
var b bytes.Buffer
|
||||
for _, m := range metas {
|
||||
b.WriteString(strings.TrimRight(m.Control, "\n"))
|
||||
b.WriteString("\n")
|
||||
fmt.Fprintf(&b, "Filename: %s\n", m.FilePath)
|
||||
fmt.Fprintf(&b, "Size: %d\n", m.Size)
|
||||
fmt.Fprintf(&b, "MD5sum: %s\n", m.MD5)
|
||||
fmt.Fprintf(&b, "SHA256: %s\n", m.SHA256)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func generateRelease(metas []provider.DebMetadata) []byte {
|
||||
packages := generatePackages(metas)
|
||||
packagesGz := gzipBytes(packages)
|
||||
|
||||
arches := uniqueArches(metas)
|
||||
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
|
||||
fmt.Fprintf(&b, "Architectures: %s\n", strings.Join(arches, " "))
|
||||
b.WriteString("Acquire-By-Hash: no\n")
|
||||
|
||||
b.WriteString("MD5Sum:\n")
|
||||
writeReleaseEntry(&b, md5Hex(packages), len(packages), "Packages")
|
||||
writeReleaseEntry(&b, md5Hex(packagesGz), len(packagesGz), "Packages.gz")
|
||||
|
||||
b.WriteString("SHA256:\n")
|
||||
writeReleaseEntry(&b, sha256Hex(packages), len(packages), "Packages")
|
||||
writeReleaseEntry(&b, sha256Hex(packagesGz), len(packagesGz), "Packages.gz")
|
||||
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func writeReleaseEntry(b *bytes.Buffer, hash string, size int, name string) {
|
||||
fmt.Fprintf(b, " %s %d %s\n", hash, size, name)
|
||||
}
|
||||
|
||||
func uniqueArches(metas []provider.DebMetadata) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, m := range metas {
|
||||
a := m.Architecture
|
||||
if a == "" || seen[a] {
|
||||
continue
|
||||
}
|
||||
seen[a] = true
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func md5Hex(data []byte) string {
|
||||
h := md5.Sum(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
Reference in New Issue
Block a user