deb/apk: make local repodata deterministic #119

Merged
benvin merged 1 commits from benvin/deb-apk-repodata-deterministic into master 2026-08-12 23:32:07 +10:00
7 changed files with 273 additions and 6 deletions
+1 -1
View File
@@ -41,7 +41,7 @@ func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([
depends, provides, install_if
FROM alpine_metadata
WHERE repo_name = $1
ORDER BY name, version, arch
ORDER BY name, version, arch, file_path
`, repoName)
if err != nil {
return nil, err
+3 -3
View File
@@ -31,10 +31,10 @@ func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]pr
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash,
name, version, architecture, control,
size, md5, sha256
size, md5, sha256, created_at
FROM deb_metadata
WHERE repo_name = $1
ORDER BY name, version, architecture
ORDER BY name, version, architecture, file_path
`, repoName)
if err != nil {
return nil, err
@@ -47,7 +47,7 @@ func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]pr
if err := rows.Scan(
&m.RepoName, &m.FilePath, &m.ContentHash,
&m.Name, &m.Version, &m.Architecture, &m.Control,
&m.Size, &m.MD5, &m.SHA256,
&m.Size, &m.MD5, &m.SHA256, &m.CreatedAt,
); err != nil {
return nil, err
}
+5 -1
View File
@@ -15,6 +15,7 @@ import (
"path"
"strconv"
"strings"
"time"
"archive/tar"
@@ -376,7 +377,10 @@ func generateAPKIndex(metas []provider.AlpineMetadata) []byte {
var tarBuf bytes.Buffer
tw := tar.NewWriter(&tarBuf)
body := idx.Bytes()
tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg})
// ModTime is pinned to the Unix epoch (never wall clock) so APKINDEX.tar.gz
// is byte-identical across replicas and regenerations (issue #117); apk
// clients ignore the tar mtime.
tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, ModTime: time.Unix(0, 0)})
tw.Write(body)
tw.Close()
@@ -0,0 +1,72 @@
package alpine
import (
"archive/tar"
"bytes"
"compress/gzip"
"io"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func apkFixture() []provider.AlpineMetadata {
return []provider.AlpineMetadata{
{
RepoName: "r", FilePath: "x86_64/aaa-1.0-r0.apk", Checksum: "Q1aaa",
Name: "aaa", Version: "1.0-r0", Arch: "x86_64", DownloadSize: 100, InstalledSize: 10,
Description: "pkg aaa", URL: "https://a", License: "MIT",
Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}, BuildTime: 1710000000,
},
{
RepoName: "r", FilePath: "x86_64/bbb-2.0-r0.apk", Checksum: "Q1bbb",
Name: "bbb", Version: "2.0-r0", Arch: "x86_64", DownloadSize: 200, InstalledSize: 20,
},
}
}
// TestAPKIndexDeterministic asserts APKINDEX.tar.gz is byte-identical across two
// generations separated by wall-clock time, so the two no-affinity replicas and
// every regeneration serve the same bytes (issue #117).
func TestAPKIndexDeterministic(t *testing.T) {
metas := apkFixture()
first := generateAPKIndex(metas)
time.Sleep(10 * time.Millisecond)
second := generateAPKIndex(metas)
if !bytes.Equal(first, second) {
t.Error("APKINDEX.tar.gz differs across generations")
}
}
// TestAPKIndexTarModTimePinned guards the tar header: its ModTime must be the
// pinned Unix epoch, never wall clock. Fails if a future edit stamps time.Now().
func TestAPKIndexTarModTimePinned(t *testing.T) {
metas := apkFixture()
zr, err := gzip.NewReader(bytes.NewReader(generateAPKIndex(metas)))
if err != nil {
t.Fatalf("gzip: %v", err)
}
if !zr.ModTime.IsZero() && zr.ModTime.Unix() != 0 {
t.Errorf("gzip header ModTime = %v, want zero/epoch", zr.ModTime)
}
tarBytes, err := io.ReadAll(zr)
if err != nil {
t.Fatalf("gunzip: %v", err)
}
tr := tar.NewReader(bytes.NewReader(tarBytes))
hdr, err := tr.Next()
if err != nil {
t.Fatalf("tar: %v", err)
}
if hdr.Name != "APKINDEX" {
t.Fatalf("tar entry = %q, want APKINDEX", hdr.Name)
}
if hdr.ModTime.Unix() != 0 {
t.Errorf("APKINDEX tar ModTime = %v (unix %d), want epoch (0)", hdr.ModTime, hdr.ModTime.Unix())
}
}
+16 -1
View File
@@ -395,7 +395,7 @@ func generateRelease(metas []provider.DebMetadata) []byte {
arches := uniqueArches(metas)
var b bytes.Buffer
fmt.Fprintf(&b, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z))
fmt.Fprintf(&b, "Date: %s\n", releaseDate(metas).Format(time.RFC1123Z))
fmt.Fprintf(&b, "Architectures: %s\n", strings.Join(arches, " "))
b.WriteString("Acquire-By-Hash: no\n")
@@ -410,6 +410,21 @@ func generateRelease(metas []provider.DebMetadata) []byte {
return b.Bytes()
}
// releaseDate derives the Release Date: from the newest package's persisted
// created_at (in UTC) so the file is byte-identical across the no-affinity
// replicas and across regenerations (issue #117); an empty repo falls back to
// the Unix epoch. This never uses wall clock, which also keeps Date: from
// running ahead of any Valid-Until logic.
func releaseDate(metas []provider.DebMetadata) time.Time {
newest := time.Unix(0, 0)
for _, m := range metas {
if m.CreatedAt.After(newest) {
newest = m.CreatedAt
}
}
return newest.UTC()
}
func writeReleaseEntry(b *bytes.Buffer, hash string, size int, name string) {
fmt.Fprintf(b, " %s %d %s\n", hash, size, name)
}
@@ -0,0 +1,172 @@
package deb
import (
"bytes"
"strconv"
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
// debFixture returns a fixed set of rows with persisted created_at values, in
// the total order ListDebMetadataEntries produces (name, version, arch,
// file_path), so the generators are exercised on a stable input.
func debFixture() []provider.DebMetadata {
t1 := time.Date(2026, 3, 1, 8, 30, 0, 0, time.UTC)
t2 := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) // newest
return []provider.DebMetadata{
{
RepoName: "r", FilePath: "pool/aaa_1.0_amd64.deb", ContentHash: "sha256:aa",
Name: "aaa", Version: "1.0", Architecture: "amd64",
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64",
Size: 100, MD5: "d41d8cd98f00b204e9800998ecf8427e", SHA256: "aa", CreatedAt: t1,
},
{
RepoName: "r", FilePath: "pool/bbb_2.0_arm64.deb", ContentHash: "sha256:bb",
Name: "bbb", Version: "2.0", Architecture: "arm64",
Control: "Package: bbb\nVersion: 2.0\nArchitecture: arm64",
Size: 200, MD5: "0cc175b9c0f1b6a831c399e269772661", SHA256: "bb", CreatedAt: t2,
},
}
}
// TestDebGeneratorsDeterministic asserts the served bytes are a pure function of
// DB state: Packages, Packages.gz and Release are byte-identical across two
// generations separated by wall-clock time. Fails against the old
// time.Now()-stamped Release Date:.
func TestDebGeneratorsDeterministic(t *testing.T) {
metas := debFixture()
pkgs1 := generatePackages(metas)
rel1 := generateRelease(metas)
gz1 := gzipBytes(pkgs1)
time.Sleep(10 * time.Millisecond)
pkgs2 := generatePackages(metas)
rel2 := generateRelease(metas)
gz2 := gzipBytes(pkgs2)
if !bytes.Equal(pkgs1, pkgs2) {
t.Error("Packages differs across generations")
}
if !bytes.Equal(gz1, gz2) {
t.Error("Packages.gz differs across generations")
}
if !bytes.Equal(rel1, rel2) {
t.Errorf("Release differs across generations:\n--- first ---\n%s\n--- second ---\n%s", rel1, rel2)
}
}
// TestDebReleaseDateUsesPersistedCreatedAt pins the Release Date: to the newest
// persisted created_at (RFC1123Z, UTC), not wall clock. Fails against the old
// time.Now() code.
func TestDebReleaseDateUsesPersistedCreatedAt(t *testing.T) {
metas := debFixture()
want := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC).Format(time.RFC1123Z)
rel := string(generateRelease(metas))
var got string
for _, line := range strings.Split(rel, "\n") {
if strings.HasPrefix(line, "Date:") {
got = strings.TrimSpace(strings.TrimPrefix(line, "Date:"))
break
}
}
if got != want {
t.Errorf("Release Date: = %q, want %q (newest created_at)", got, want)
}
}
// TestDebReleaseDateEmptyRepoIsEpoch guards the fallback: an empty repo yields a
// deterministic epoch Date: rather than wall clock.
func TestDebReleaseDateEmptyRepoIsEpoch(t *testing.T) {
want := time.Unix(0, 0).UTC().Format(time.RFC1123Z)
rel := string(generateRelease(nil))
if !strings.Contains(rel, "Date: "+want+"\n") {
t.Errorf("empty-repo Release missing epoch Date: %q\n%s", want, rel)
}
}
// TestDebReleaseChecksumsMatchServedBytes is the exact apt invariant: the
// sha256/size (and md5/size) advertised for Packages and Packages.gz in Release
// equal the sha256/size of the actual bytes ServeLocalIndex serves. apt rejects
// any mismatch.
func TestDebReleaseChecksumsMatchServedBytes(t *testing.T) {
metas := debFixture()
packages := generatePackages(metas)
packagesGz := gzipBytes(packages)
rel := string(generateRelease(metas))
wantSHA := map[string]struct {
hash string
size int
}{
"Packages": {sha256Hex(packages), len(packages)},
"Packages.gz": {sha256Hex(packagesGz), len(packagesGz)},
}
wantMD5 := map[string]struct {
hash string
size int
}{
"Packages": {md5Hex(packages), len(packages)},
"Packages.gz": {md5Hex(packagesGz), len(packagesGz)},
}
sha := parseReleaseSection(rel, "SHA256:")
md5s := parseReleaseSection(rel, "MD5Sum:")
for name, w := range wantSHA {
got, ok := sha[name]
if !ok {
t.Fatalf("Release SHA256 section missing %q", name)
}
if got.hash != w.hash || got.size != w.size {
t.Errorf("Release SHA256 %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size)
}
}
for name, w := range wantMD5 {
got, ok := md5s[name]
if !ok {
t.Fatalf("Release MD5Sum section missing %q", name)
}
if got.hash != w.hash || got.size != w.size {
t.Errorf("Release MD5Sum %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size)
}
}
}
type releaseEntry struct {
hash string
size int
}
// parseReleaseSection reads the indented " <hash> <size> <name>" lines that
// follow a "SHA256:" / "MD5Sum:" header until the next non-indented line.
func parseReleaseSection(release, header string) map[string]releaseEntry {
out := map[string]releaseEntry{}
lines := strings.Split(release, "\n")
in := false
for _, line := range lines {
if line == header {
in = true
continue
}
if !in {
continue
}
if !strings.HasPrefix(line, " ") {
break
}
fields := strings.Fields(line)
if len(fields) != 3 {
continue
}
size, _ := strconv.Atoi(fields[1])
out[fields[2]] = releaseEntry{hash: fields[0], size: size}
}
return out
}
+4
View File
@@ -114,6 +114,10 @@ type DebMetadata struct {
Size int64
MD5 string
SHA256 string
// CreatedAt is the persisted insert time; the Release Date: is derived from
// the newest value so the index is byte-identical across replicas and
// regenerations (issue #117) rather than stamped from wall clock.
CreatedAt time.Time
}
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the