artifactapi/internal/provider/deb/deb.go

package deb

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"context"
	"crypto/md5"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"path"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/klauspost/compress/zstd"
	"github.com/ulikunitz/xz"

	"git.unkin.net/unkin/artifactapi/internal/auth"
	"git.unkin.net/unkin/artifactapi/internal/provider"
	"git.unkin.net/unkin/artifactapi/internal/storage"
	"git.unkin.net/unkin/artifactapi/pkg/models"
)

func init() {
	provider.Register(&Provider{})
}

// mutableRe marks the apt index surface (both the flat local repo and a proxied
// Debian/Ubuntu mirror's dists/ tree) so the caching engine revalidates it
// instead of freezing it like an immutable .deb.
var mutableRe = []*regexp.Regexp{
	regexp.MustCompile(`(^|/)Packages(\.gz|\.xz|\.bz2)?$`),
	regexp.MustCompile(`(^|/)Sources(\.gz|\.xz|\.bz2)?$`),
	regexp.MustCompile(`(^|/)Release$`),
	regexp.MustCompile(`(^|/)InRelease$`),
	regexp.MustCompile(`(^|/)Release\.gpg$`),
	regexp.MustCompile(`(^|/)Contents-`),
	regexp.MustCompile(`^dists/`),
	regexp.MustCompile(`/by-hash/`),
}

type Provider struct{}

func (p *Provider) Type() models.PackageType { return models.PackageDeb }

func (p *Provider) Classify(path string) provider.Mutability {
	for _, re := range mutableRe {
		if re.MatchString(path) {
			return provider.Mutable
		}
	}
	return provider.Immutable
}

func (p *Provider) ContentType(path string) string {
	switch {
	case strings.HasSuffix(path, ".deb"):
		return "application/vnd.debian.binary-package"
	case strings.HasSuffix(path, ".gz"):
		return "application/gzip"
	case strings.HasSuffix(path, ".xz"):
		return "application/x-xz"
	case strings.HasSuffix(path, "Packages"), strings.HasSuffix(path, "Release"),
		strings.HasSuffix(path, "InRelease"), strings.HasSuffix(path, "Sources"):
		return "text/plain"
	}
	return "application/octet-stream"
}

func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
	return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}

func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
	return nil, nil
}

func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
	return auth.BasicHeaders(remote), nil
}

func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
	filename := filePath
	if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
		filename = filePath[idx+1:]
	}

	if !strings.HasSuffix(strings.ToLower(filename), ".deb") {
		return "", "", fmt.Errorf("file must be a .deb package")
	}

	return "pool/" + filename, "application/vnd.debian.binary-package", nil
}

func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
	filename := strings.TrimPrefix(storagePath, "pool/")
	return map[string]any{
		"filename":     filename,
		"content_hash": contentHash,
		"size_bytes":   sizeBytes,
	}
}

func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
	s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))

	reader, blobSize, err := blobs.Download(ctx, s3Key)
	if err != nil {
		slog.Error("deb metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
		return
	}
	defer reader.Close()

	raw, err := io.ReadAll(reader)
	if err != nil {
		slog.Error("deb metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
		return
	}

	control, err := extractControl(raw)
	if err != nil {
		slog.Error("deb metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
		return
	}
	fields := parseControlFields(control)

	sum := md5.Sum(raw)
	meta := &provider.DebMetadata{
		RepoName:     repoName,
		FilePath:     storagePath,
		ContentHash:  contentHash,
		Name:         fields["Package"],
		Version:      fields["Version"],
		Architecture: fields["Architecture"],
		Control:      strings.TrimRight(control, "\n"),
		Size:         blobSize,
		MD5:          hex.EncodeToString(sum[:]),
		SHA256:       strings.TrimPrefix(contentHash, "sha256:"),
	}

	if meta.Name == "" {
		slog.Error("deb metadata: control missing Package field", "repo", repoName, "path", storagePath)
		return
	}

	if err := db.InsertDebMetadata(ctx, meta); err != nil {
		slog.Error("deb metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
		return
	}

	slog.Info("deb metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
}

func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
	if err := db.DeleteDebMetadata(ctx, repoName, storagePath); err != nil {
		slog.Error("deb metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
		return err
	}
	slog.Info("deb metadata: deleted", "repo", repoName, "path", storagePath)
	return nil
}

// extractControl reads a .deb (an ar archive), locates the control.tar.* member,
// decompresses it, and returns the raw ./control paragraph. Pure Go: no dpkg.
func extractControl(deb []byte) (string, error) {
	members, err := readAr(deb)
	if err != nil {
		return "", err
	}

	var name string
	var data []byte
	for _, m := range members {
		if strings.HasPrefix(m.name, "control.tar") {
			name = m.name
			data = m.data
			break
		}
	}
	if data == nil {
		return "", errors.New("no control.tar member in .deb")
	}

	tarBytes, err := decompress(name, data)
	if err != nil {
		return "", err
	}

	return readControlParagraph(tarBytes)
}

// readControlParagraph scans a decompressed control.tar and returns the raw
// ./control paragraph. Shared by the local upload path (extractControl) and the
// github_deb ranged-prefix parser.
func readControlParagraph(controlTar []byte) (string, error) {
	tr := tar.NewReader(bytes.NewReader(controlTar))
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return "", fmt.Errorf("read control.tar: %w", err)
		}
		clean := strings.TrimPrefix(hdr.Name, "./")
		if clean == "control" {
			b, err := io.ReadAll(tr)
			if err != nil {
				return "", fmt.Errorf("read control file: %w", err)
			}
			return string(b), nil
		}
	}
	return "", errors.New("no ./control in control.tar")
}

func decompress(name string, data []byte) ([]byte, error) {
	switch {
	case strings.HasSuffix(name, ".gz"):
		zr, err := gzip.NewReader(bytes.NewReader(data))
		if err != nil {
			return nil, err
		}
		defer zr.Close()
		return io.ReadAll(zr)
	case strings.HasSuffix(name, ".xz"):
		xr, err := xz.NewReader(bytes.NewReader(data))
		if err != nil {
			return nil, err
		}
		return io.ReadAll(xr)
	case strings.HasSuffix(name, ".zst"):
		zr, err := zstd.NewReader(bytes.NewReader(data))
		if err != nil {
			return nil, err
		}
		defer zr.Close()
		return io.ReadAll(zr)
	case strings.HasSuffix(name, ".tar"):
		return data, nil
	}
	return nil, fmt.Errorf("unsupported control.tar compression: %s", name)
}

type arMember struct {
	name string
	data []byte
}

// readAr parses the (trivial) Unix ar archive that wraps a .deb. Each member has
// a 60-byte header; the size field is decimal ASCII and data is padded to an
// even offset.
func readAr(data []byte) ([]arMember, error) {
	const magic = "!<arch>\n"
	if len(data) < len(magic) || string(data[:len(magic)]) != magic {
		return nil, errors.New("not an ar archive")
	}
	off := len(magic)

	var members []arMember
	for off+60 <= len(data) {
		hdr := data[off : off+60]
		off += 60

		name := strings.TrimRight(string(hdr[0:16]), " ")
		name = strings.TrimSuffix(name, "/")
		size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
		if err != nil {
			return nil, fmt.Errorf("bad ar size for %q: %w", name, err)
		}
		if off+int(size) > len(data) {
			return nil, fmt.Errorf("truncated ar member %q", name)
		}
		members = append(members, arMember{name: name, data: data[off : off+int(size)]})
		off += int(size)
		if size%2 == 1 {
			off++
		}
	}
	return members, nil
}

// parseControlFields reads the single-line fields of an RFC822-style control
// paragraph. Continuation lines (leading whitespace) belong to the previous
// field and are ignored here since only Package/Version/Architecture are read.
func parseControlFields(control string) map[string]string {
	fields := map[string]string{}
	sc := bufio.NewScanner(strings.NewReader(control))
	sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
	for sc.Scan() {
		line := sc.Text()
		if line == "" || line[0] == ' ' || line[0] == '\t' {
			continue
		}
		idx := strings.IndexByte(line, ':')
		if idx < 0 {
			continue
		}
		key := strings.TrimSpace(line[:idx])
		if _, seen := fields[key]; seen {
			continue
		}
		fields[key] = strings.TrimSpace(line[idx+1:])
	}
	return fields
}

// normalizeIndexPath collapses apt's verbatim dist prefix from a flat-repo
// request. For `deb ... <repo>/ ./`, apt appends the "./" dist literally and asks
// for "./Packages" (and "./Release", "./InRelease"); dot-segments must be
// collapsed so the index matcher sees "Packages". A no-op for pool/*.deb paths.
func normalizeIndexPath(p string) string {
	return strings.TrimPrefix(path.Clean("/"+p), "/")
}

func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
	path := normalizeIndexPath(reqPath)
	switch path {
	case "Packages", "Packages.gz", "Release":
	default:
		return false
	}

	reader, ok := files.(provider.DebMetadataReader)
	if !ok {
		http.Error(w, "deb metadata not available", http.StatusInternalServerError)
		return true
	}

	metas, err := reader.ListDebMetadataEntries(r.Context(), repoName)
	if err != nil {
		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
			slog.Warn("deb: metadata read canceled", "repo", repoName, "error", err)
			http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
			return true
		}
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return true
	}

	switch path {
	case "Packages":
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusOK)
		w.Write(generatePackages(metas))
	case "Packages.gz":
		w.Header().Set("Content-Type", "application/gzip")
		w.WriteHeader(http.StatusOK)
		w.Write(gzipBytes(generatePackages(metas)))
	case "Release":
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusOK)
		w.Write(generateRelease(metas))
	}
	return true
}

func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
	return nil, fmt.Errorf("deb local index generation for virtual repos not supported")
}

// generatePackages emits the flat-repo Packages file: each package's raw control
// stanza followed by the apt-required Filename/Size/MD5sum/SHA256 fields, blank
// line separated.
func generatePackages(metas []provider.DebMetadata) []byte {
	var b bytes.Buffer
	for _, m := range metas {
		b.WriteString(strings.TrimRight(m.Control, "\n"))
		b.WriteString("\n")
		fmt.Fprintf(&b, "Filename: %s\n", m.FilePath)
		fmt.Fprintf(&b, "Size: %d\n", m.Size)
		if m.MD5 != "" {
			fmt.Fprintf(&b, "MD5sum: %s\n", m.MD5)
		}
		if m.SHA256 != "" {
			fmt.Fprintf(&b, "SHA256: %s\n", m.SHA256)
		}
		b.WriteString("\n")
	}
	return b.Bytes()
}

func generateRelease(metas []provider.DebMetadata) []byte {
	packages := generatePackages(metas)
	packagesGz := gzipBytes(packages)

	arches := uniqueArches(metas)

	var b bytes.Buffer
	fmt.Fprintf(&b, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
	fmt.Fprintf(&b, "Architectures: %s\n", strings.Join(arches, " "))
	b.WriteString("Acquire-By-Hash: no\n")

	b.WriteString("MD5Sum:\n")
	writeReleaseEntry(&b, md5Hex(packages), len(packages), "Packages")
	writeReleaseEntry(&b, md5Hex(packagesGz), len(packagesGz), "Packages.gz")

	b.WriteString("SHA256:\n")
	writeReleaseEntry(&b, sha256Hex(packages), len(packages), "Packages")
	writeReleaseEntry(&b, sha256Hex(packagesGz), len(packagesGz), "Packages.gz")

	return b.Bytes()
}

func writeReleaseEntry(b *bytes.Buffer, hash string, size int, name string) {
	fmt.Fprintf(b, " %s %d %s\n", hash, size, name)
}

func uniqueArches(metas []provider.DebMetadata) []string {
	seen := map[string]bool{}
	var out []string
	for _, m := range metas {
		a := m.Architecture
		if a == "" || seen[a] {
			continue
		}
		seen[a] = true
		out = append(out, a)
	}
	return out
}

func gzipBytes(data []byte) []byte {
	var buf bytes.Buffer
	gz := gzip.NewWriter(&buf)
	gz.Write(data)
	gz.Close()
	return buf.Bytes()
}

func md5Hex(data []byte) string {
	h := md5.Sum(data)
	return hex.EncodeToString(h[:])
}

func sha256Hex(data []byte) string {
	h := sha256.Sum256(data)
	return hex.EncodeToString(h[:])
}