From cea107d4b5dd06a95732a405aad4cd3ae963b64d Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Wed, 12 Aug 2026 23:21:20 +1000 Subject: [PATCH] deb/apk: make local repodata deterministic Part of #117. The two no-affinity replicas (and every regeneration) must serve byte-identical local repodata so apt/apk never hit a checksum mismatch between an index's advertised hash and the bytes actually served. - Derive the deb Release Date: from the newest persisted created_at (RFC1123Z, UTC) instead of time.Now(); carry created_at through the deb metadata SELECT and DebMetadata struct. - Pin the apk APKINDEX tar header ModTime to the Unix epoch instead of the zero-value time.Time, so it is never wall-clock derived. - Give both list queries a genuine total order by adding a file_path tiebreak (name/version/arch is not unique). - Add guard tests: deb generators byte-identical across generations, the Release checksum/size invariant matches the served Packages(.gz) bytes, the Date: is pinned to created_at; apk index byte-identical and tar ModTime pinned to epoch. --- internal/database/alpine_metadata.go | 2 +- internal/database/deb_metadata.go | 6 +- internal/provider/alpine/alpine.go | 6 +- .../alpine/alpine_deterministic_test.go | 72 ++++++++ internal/provider/deb/deb.go | 17 +- .../provider/deb/deb_deterministic_test.go | 172 ++++++++++++++++++ internal/provider/provider.go | 4 + 7 files changed, 273 insertions(+), 6 deletions(-) create mode 100644 internal/provider/alpine/alpine_deterministic_test.go create mode 100644 internal/provider/deb/deb_deterministic_test.go diff --git a/internal/database/alpine_metadata.go b/internal/database/alpine_metadata.go index ecf19d1..5438019 100644 --- a/internal/database/alpine_metadata.go +++ b/internal/database/alpine_metadata.go @@ -41,7 +41,7 @@ func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([ depends, provides, install_if FROM alpine_metadata WHERE repo_name = $1 - ORDER BY name, version, arch + ORDER BY name, version, arch, file_path `, repoName) if err != nil { return nil, err diff --git a/internal/database/deb_metadata.go b/internal/database/deb_metadata.go index bdd596d..bac1e13 100644 --- a/internal/database/deb_metadata.go +++ b/internal/database/deb_metadata.go @@ -31,10 +31,10 @@ func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]pr rows, err := db.Pool.Query(ctx, ` SELECT repo_name, file_path, content_hash, name, version, architecture, control, - size, md5, sha256 + size, md5, sha256, created_at FROM deb_metadata WHERE repo_name = $1 - ORDER BY name, version, architecture + ORDER BY name, version, architecture, file_path `, repoName) if err != nil { return nil, err @@ -47,7 +47,7 @@ func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]pr if err := rows.Scan( &m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.Architecture, &m.Control, - &m.Size, &m.MD5, &m.SHA256, + &m.Size, &m.MD5, &m.SHA256, &m.CreatedAt, ); err != nil { return nil, err } diff --git a/internal/provider/alpine/alpine.go b/internal/provider/alpine/alpine.go index b3198e2..e2803b3 100644 --- a/internal/provider/alpine/alpine.go +++ b/internal/provider/alpine/alpine.go @@ -15,6 +15,7 @@ import ( "path" "strconv" "strings" + "time" "archive/tar" @@ -376,7 +377,10 @@ func generateAPKIndex(metas []provider.AlpineMetadata) []byte { var tarBuf bytes.Buffer tw := tar.NewWriter(&tarBuf) body := idx.Bytes() - tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg}) + // ModTime is pinned to the Unix epoch (never wall clock) so APKINDEX.tar.gz + // is byte-identical across replicas and regenerations (issue #117); apk + // clients ignore the tar mtime. + tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, ModTime: time.Unix(0, 0)}) tw.Write(body) tw.Close() diff --git a/internal/provider/alpine/alpine_deterministic_test.go b/internal/provider/alpine/alpine_deterministic_test.go new file mode 100644 index 0000000..cc188b9 --- /dev/null +++ b/internal/provider/alpine/alpine_deterministic_test.go @@ -0,0 +1,72 @@ +package alpine + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "io" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func apkFixture() []provider.AlpineMetadata { + return []provider.AlpineMetadata{ + { + RepoName: "r", FilePath: "x86_64/aaa-1.0-r0.apk", Checksum: "Q1aaa", + Name: "aaa", Version: "1.0-r0", Arch: "x86_64", DownloadSize: 100, InstalledSize: 10, + Description: "pkg aaa", URL: "https://a", License: "MIT", + Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}, BuildTime: 1710000000, + }, + { + RepoName: "r", FilePath: "x86_64/bbb-2.0-r0.apk", Checksum: "Q1bbb", + Name: "bbb", Version: "2.0-r0", Arch: "x86_64", DownloadSize: 200, InstalledSize: 20, + }, + } +} + +// TestAPKIndexDeterministic asserts APKINDEX.tar.gz is byte-identical across two +// generations separated by wall-clock time, so the two no-affinity replicas and +// every regeneration serve the same bytes (issue #117). +func TestAPKIndexDeterministic(t *testing.T) { + metas := apkFixture() + + first := generateAPKIndex(metas) + time.Sleep(10 * time.Millisecond) + second := generateAPKIndex(metas) + + if !bytes.Equal(first, second) { + t.Error("APKINDEX.tar.gz differs across generations") + } +} + +// TestAPKIndexTarModTimePinned guards the tar header: its ModTime must be the +// pinned Unix epoch, never wall clock. Fails if a future edit stamps time.Now(). +func TestAPKIndexTarModTimePinned(t *testing.T) { + metas := apkFixture() + + zr, err := gzip.NewReader(bytes.NewReader(generateAPKIndex(metas))) + if err != nil { + t.Fatalf("gzip: %v", err) + } + if !zr.ModTime.IsZero() && zr.ModTime.Unix() != 0 { + t.Errorf("gzip header ModTime = %v, want zero/epoch", zr.ModTime) + } + + tarBytes, err := io.ReadAll(zr) + if err != nil { + t.Fatalf("gunzip: %v", err) + } + tr := tar.NewReader(bytes.NewReader(tarBytes)) + hdr, err := tr.Next() + if err != nil { + t.Fatalf("tar: %v", err) + } + if hdr.Name != "APKINDEX" { + t.Fatalf("tar entry = %q, want APKINDEX", hdr.Name) + } + if hdr.ModTime.Unix() != 0 { + t.Errorf("APKINDEX tar ModTime = %v (unix %d), want epoch (0)", hdr.ModTime, hdr.ModTime.Unix()) + } +} diff --git a/internal/provider/deb/deb.go b/internal/provider/deb/deb.go index 7db83fd..6946b16 100644 --- a/internal/provider/deb/deb.go +++ b/internal/provider/deb/deb.go @@ -395,7 +395,7 @@ func generateRelease(metas []provider.DebMetadata) []byte { arches := uniqueArches(metas) var b bytes.Buffer - fmt.Fprintf(&b, "Date: %s\n", time.Now().UTC().Format(time.RFC1123Z)) + fmt.Fprintf(&b, "Date: %s\n", releaseDate(metas).Format(time.RFC1123Z)) fmt.Fprintf(&b, "Architectures: %s\n", strings.Join(arches, " ")) b.WriteString("Acquire-By-Hash: no\n") @@ -410,6 +410,21 @@ func generateRelease(metas []provider.DebMetadata) []byte { return b.Bytes() } +// releaseDate derives the Release Date: from the newest package's persisted +// created_at (in UTC) so the file is byte-identical across the no-affinity +// replicas and across regenerations (issue #117); an empty repo falls back to +// the Unix epoch. This never uses wall clock, which also keeps Date: from +// running ahead of any Valid-Until logic. +func releaseDate(metas []provider.DebMetadata) time.Time { + newest := time.Unix(0, 0) + for _, m := range metas { + if m.CreatedAt.After(newest) { + newest = m.CreatedAt + } + } + return newest.UTC() +} + func writeReleaseEntry(b *bytes.Buffer, hash string, size int, name string) { fmt.Fprintf(b, " %s %d %s\n", hash, size, name) } diff --git a/internal/provider/deb/deb_deterministic_test.go b/internal/provider/deb/deb_deterministic_test.go new file mode 100644 index 0000000..75896e9 --- /dev/null +++ b/internal/provider/deb/deb_deterministic_test.go @@ -0,0 +1,172 @@ +package deb + +import ( + "bytes" + "strconv" + "strings" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +// debFixture returns a fixed set of rows with persisted created_at values, in +// the total order ListDebMetadataEntries produces (name, version, arch, +// file_path), so the generators are exercised on a stable input. +func debFixture() []provider.DebMetadata { + t1 := time.Date(2026, 3, 1, 8, 30, 0, 0, time.UTC) + t2 := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) // newest + return []provider.DebMetadata{ + { + RepoName: "r", FilePath: "pool/aaa_1.0_amd64.deb", ContentHash: "sha256:aa", + Name: "aaa", Version: "1.0", Architecture: "amd64", + Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64", + Size: 100, MD5: "d41d8cd98f00b204e9800998ecf8427e", SHA256: "aa", CreatedAt: t1, + }, + { + RepoName: "r", FilePath: "pool/bbb_2.0_arm64.deb", ContentHash: "sha256:bb", + Name: "bbb", Version: "2.0", Architecture: "arm64", + Control: "Package: bbb\nVersion: 2.0\nArchitecture: arm64", + Size: 200, MD5: "0cc175b9c0f1b6a831c399e269772661", SHA256: "bb", CreatedAt: t2, + }, + } +} + +// TestDebGeneratorsDeterministic asserts the served bytes are a pure function of +// DB state: Packages, Packages.gz and Release are byte-identical across two +// generations separated by wall-clock time. Fails against the old +// time.Now()-stamped Release Date:. +func TestDebGeneratorsDeterministic(t *testing.T) { + metas := debFixture() + + pkgs1 := generatePackages(metas) + rel1 := generateRelease(metas) + gz1 := gzipBytes(pkgs1) + + time.Sleep(10 * time.Millisecond) + + pkgs2 := generatePackages(metas) + rel2 := generateRelease(metas) + gz2 := gzipBytes(pkgs2) + + if !bytes.Equal(pkgs1, pkgs2) { + t.Error("Packages differs across generations") + } + if !bytes.Equal(gz1, gz2) { + t.Error("Packages.gz differs across generations") + } + if !bytes.Equal(rel1, rel2) { + t.Errorf("Release differs across generations:\n--- first ---\n%s\n--- second ---\n%s", rel1, rel2) + } +} + +// TestDebReleaseDateUsesPersistedCreatedAt pins the Release Date: to the newest +// persisted created_at (RFC1123Z, UTC), not wall clock. Fails against the old +// time.Now() code. +func TestDebReleaseDateUsesPersistedCreatedAt(t *testing.T) { + metas := debFixture() + want := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC).Format(time.RFC1123Z) + + rel := string(generateRelease(metas)) + var got string + for _, line := range strings.Split(rel, "\n") { + if strings.HasPrefix(line, "Date:") { + got = strings.TrimSpace(strings.TrimPrefix(line, "Date:")) + break + } + } + if got != want { + t.Errorf("Release Date: = %q, want %q (newest created_at)", got, want) + } +} + +// TestDebReleaseDateEmptyRepoIsEpoch guards the fallback: an empty repo yields a +// deterministic epoch Date: rather than wall clock. +func TestDebReleaseDateEmptyRepoIsEpoch(t *testing.T) { + want := time.Unix(0, 0).UTC().Format(time.RFC1123Z) + rel := string(generateRelease(nil)) + if !strings.Contains(rel, "Date: "+want+"\n") { + t.Errorf("empty-repo Release missing epoch Date: %q\n%s", want, rel) + } +} + +// TestDebReleaseChecksumsMatchServedBytes is the exact apt invariant: the +// sha256/size (and md5/size) advertised for Packages and Packages.gz in Release +// equal the sha256/size of the actual bytes ServeLocalIndex serves. apt rejects +// any mismatch. +func TestDebReleaseChecksumsMatchServedBytes(t *testing.T) { + metas := debFixture() + + packages := generatePackages(metas) + packagesGz := gzipBytes(packages) + rel := string(generateRelease(metas)) + + wantSHA := map[string]struct { + hash string + size int + }{ + "Packages": {sha256Hex(packages), len(packages)}, + "Packages.gz": {sha256Hex(packagesGz), len(packagesGz)}, + } + wantMD5 := map[string]struct { + hash string + size int + }{ + "Packages": {md5Hex(packages), len(packages)}, + "Packages.gz": {md5Hex(packagesGz), len(packagesGz)}, + } + + sha := parseReleaseSection(rel, "SHA256:") + md5s := parseReleaseSection(rel, "MD5Sum:") + + for name, w := range wantSHA { + got, ok := sha[name] + if !ok { + t.Fatalf("Release SHA256 section missing %q", name) + } + if got.hash != w.hash || got.size != w.size { + t.Errorf("Release SHA256 %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size) + } + } + for name, w := range wantMD5 { + got, ok := md5s[name] + if !ok { + t.Fatalf("Release MD5Sum section missing %q", name) + } + if got.hash != w.hash || got.size != w.size { + t.Errorf("Release MD5Sum %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size) + } + } +} + +type releaseEntry struct { + hash string + size int +} + +// parseReleaseSection reads the indented " " lines that +// follow a "SHA256:" / "MD5Sum:" header until the next non-indented line. +func parseReleaseSection(release, header string) map[string]releaseEntry { + out := map[string]releaseEntry{} + lines := strings.Split(release, "\n") + in := false + for _, line := range lines { + if line == header { + in = true + continue + } + if !in { + continue + } + if !strings.HasPrefix(line, " ") { + break + } + fields := strings.Fields(line) + if len(fields) != 3 { + continue + } + size, _ := strconv.Atoi(fields[1]) + out[fields[2]] = releaseEntry{hash: fields[0], size: size} + } + return out +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 6ccb1c9..8de9201 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -114,6 +114,10 @@ type DebMetadata struct { Size int64 MD5 string SHA256 string + // CreatedAt is the persisted insert time; the Release Date: is derived from + // the newest value so the index is byte-identical across replicas and + // regenerations (issue #117) rather than stamped from wall clock. + CreatedAt time.Time } // AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the -- 2.47.3