From 5d50eda58ab2897b065db8cb32a8c046fa7b4a10 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sun, 25 Oct 2026 13:16:28 +1100 Subject: [PATCH] Add cache_repodata toggle for github_rpm remotes Cache generated repodata per replica and rebuild only when the remote's rpm_metadata fingerprint (count, max id) changes. --- docs/rpm/remote-github.md | 8 + internal/database/database_test.go | 49 +++++- internal/database/remotes.go | 12 +- internal/database/rpm_metadata.go | 7 + internal/provider/rpm/github.go | 92 +++++++++++ .../rpm/github_repodata_cache_test.go | 147 ++++++++++++++++++ internal/provider/rpm/rpm.go | 30 ++-- migrations/0006_remote_cache_repodata.sql | 1 + pkg/models/remote.go | 4 + 9 files changed, 332 insertions(+), 18 deletions(-) create mode 100644 internal/provider/rpm/github_repodata_cache_test.go create mode 100644 migrations/0006_remote_cache_repodata.sql diff --git a/docs/rpm/remote-github.md b/docs/rpm/remote-github.md index f079b02..3d6eed1 100644 --- a/docs/rpm/remote-github.md +++ b/docs/rpm/remote-github.md @@ -53,6 +53,14 @@ derived metadata current off the client request path: ~once per `mutable_ttl` regardless of replica count, and the shared `etag` lets any replica issue the conditional request. +### Repodata cache + +Repodata is regenerated from the metadata rows on every request by default. Set +`cache_repodata: true` on the remote to keep the generated `repomd.xml` and its +`primary`/`filelists`/`other` files in memory per replica. Each request checks a +cheap row fingerprint (count + newest id) in the database and rebuilds only when +it moved, so every replica picks up a scan's changes on its next request. + ```hcl # Backend that serves the actual .rpm bytes from github.com. resource "artifactapi_remote_generic" "github" { diff --git a/internal/database/database_test.go b/internal/database/database_test.go index ef0e8a9..9a56b89 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -79,13 +79,17 @@ func TestRemotesCRUD(t *testing.T) { if err != nil || got.BaseURL != "https://example.com" { t.Fatalf("get: %v %v", got, err) } + if got.CacheRepodata { + t.Error("cache_repodata should default to false") + } got.BaseURL = "https://updated.example.com" + got.CacheRepodata = true if err := testDB.UpdateRemote(ctx(), got); err != nil { t.Fatalf("update: %v", err) } got, _ = testDB.GetRemote(ctx(), "r-crud") - if got.BaseURL != "https://updated.example.com" { - t.Errorf("update not applied: %v", got.BaseURL) + if got.BaseURL != "https://updated.example.com" || !got.CacheRepodata { + t.Errorf("update not applied: %v %v", got.BaseURL, got.CacheRepodata) } list, err := testDB.ListRemotes(ctx()) if err != nil || len(list) == 0 { @@ -453,3 +457,44 @@ func TestRPMMetadata(t *testing.T) { t.Fatalf("list rpm rows: %v %v", len(rows), err) } } + +func TestRPMMetadataFingerprint(t *testing.T) { + requireDB(t) + seedRemote(t, "r-rpm-fp") + fp := func() [2]int64 { + t.Helper() + c, m, err := testDB.RPMMetadataFingerprint(ctx(), "r-rpm-fp") + if err != nil { + t.Fatal(err) + } + return [2]int64{c, m} + } + insert := func(path string) { + t.Helper() + if err := testDB.InsertRPMMetadata(ctx(), &provider.RPMMetadata{ + RepoName: "r-rpm-fp", FilePath: path, ContentHash: "sha256:" + path, + Name: path, Version: "1", Release: "1", Arch: "noarch", + }); err != nil { + t.Fatal(err) + } + } + + if got := fp(); got != [2]int64{0, 0} { + t.Fatalf("empty fingerprint = %v", got) + } + insert("a.rpm") + one := fp() + insert("a.rpm") + if fp() != one { + t.Fatal("conflicting insert changed fingerprint") + } + insert("b.rpm") + two := fp() + if err := testDB.DeleteRPMMetadata(ctx(), "r-rpm-fp", "a.rpm"); err != nil { + t.Fatal(err) + } + afterDelete := fp() + if one[0] != 1 || two[0] != 2 || afterDelete[0] != 1 || afterDelete == one || afterDelete == two { + t.Fatalf("fingerprints did not track changes: %v %v %v", one, two, afterDelete) + } +} diff --git a/internal/database/remotes.go b/internal/database/remotes.go index 44b2eb5..dad69f9 100644 --- a/internal/database/remotes.go +++ b/internal/database/remotes.go @@ -15,6 +15,7 @@ const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_ quarantine_enabled, quarantine_days, stale_on_error, releases_remote, managed_by, upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout, + cache_repodata, created_at, updated_at` // normalizeMirrorStrategy maps an empty strategy to the round_robin default so @@ -35,6 +36,7 @@ func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error &r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError, &r.ReleasesRemote, &r.ManagedBy, &r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout, + &r.CacheRepodata, &r.CreatedAt, &r.UpdatedAt, ) } @@ -76,8 +78,8 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error { quarantine_enabled, quarantine_days, stale_on_error, releases_remote, managed_by, upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout, - mirror_strategy - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26) + mirror_strategy, cache_repodata + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27) `, r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password, r.ImmutableTTL, r.MutableTTL, r.CheckMutable, @@ -86,7 +88,7 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error { r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError, r.ReleasesRemote, r.ManagedBy, r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout, - normalizeMirrorStrategy(r.MirrorStrategy), + normalizeMirrorStrategy(r.MirrorStrategy), r.CacheRepodata, ) return err } @@ -101,7 +103,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error { quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19, releases_remote=$20, managed_by=$21, upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24, - mirror_strategy=$26, + mirror_strategy=$26, cache_repodata=$27, updated_at=NOW() WHERE name=$1 `, @@ -113,7 +115,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error { r.ReleasesRemote, r.ManagedBy, r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout, r.Mirrorlist, - normalizeMirrorStrategy(r.MirrorStrategy), + normalizeMirrorStrategy(r.MirrorStrategy), r.CacheRepodata, ) return err } diff --git a/internal/database/rpm_metadata.go b/internal/database/rpm_metadata.go index 2466b06..ccde123 100644 --- a/internal/database/rpm_metadata.go +++ b/internal/database/rpm_metadata.go @@ -40,6 +40,13 @@ func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string) return err } +// RPMMetadataFingerprint is a cheap change detector for a repo's rows: they are +// only ever inserted or deleted, so (count, max id) moves whenever they change. +func (db *DB) RPMMetadataFingerprint(ctx context.Context, repoName string) (count, maxID int64, err error) { + err = db.Pool.QueryRow(ctx, `SELECT count(*), coalesce(max(id), 0) FROM rpm_metadata WHERE repo_name = $1`, repoName).Scan(&count, &maxID) + return count, maxID, err +} + type RPMMetadataRow struct { RepoName string FilePath string diff --git a/internal/provider/rpm/github.go b/internal/provider/rpm/github.go index 707b54d..b6fcce4 100644 --- a/internal/provider/rpm/github.go +++ b/internal/provider/rpm/github.go @@ -18,6 +18,7 @@ import ( "time" rpmlib "github.com/cavaliergopher/rpm" + "golang.org/x/sync/singleflight" "golang.org/x/time/rate" "git.unkin.net/unkin/artifactapi/internal/githubauth" @@ -84,6 +85,22 @@ type GitHubProvider struct { mu sync.Mutex scanning map[string]bool lastScan map[string]time.Time + + // repodata caches generated indexes for cache_repodata remotes, keyed by remote name. + repodataMu sync.Mutex + repodataCache map[string]cachedRepodata + repodataGroup singleflight.Group +} + +// rpmFingerprinter is a cheap change detector over a remote's metadata rows. +// Rows are insert-or-delete only, so (count, max id) changes whenever they do. +type rpmFingerprinter interface { + RPMMetadataFingerprint(ctx context.Context, repoName string) (count, maxID int64, err error) +} + +type cachedRepodata struct { + count, maxID int64 + *repodata } func newGitHubProvider() *GitHubProvider { @@ -97,6 +114,7 @@ func newGitHubProvider() *GitHubProvider { coldWait: defaultColdWait, scanning: map[string]bool{}, lastScan: map[string]time.Time{}, + repodataCache: map[string]cachedRepodata{}, } } @@ -170,6 +188,11 @@ func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, rem } tail := strings.TrimPrefix(path, "repodata/") + if fp, ok := store.(rpmFingerprinter); ok && remote.CacheRepodata { + p.serveCachedRepodata(w, sr, store, fp, remote.Name, tail) + return true + } + p.dropRepodata(remote.Name) lp := &Provider{} switch { case tail == "repomd.xml": @@ -239,7 +262,76 @@ func (p *GitHubProvider) ensurePrimed(ctx context.Context, remote models.Remote, return false } +// serveCachedRepodata serves a file from the generated index, rebuilding it +// only when the rows' fingerprint moved. The fingerprint is read from the shared +// database on every request, so each replica rebuilds as soon as any replica's +// scan changes rows and repomd never outlives the files it references. +func (p *GitHubProvider) serveCachedRepodata(w http.ResponseWriter, r *http.Request, store provider.RPMMetadataReader, fp rpmFingerprinter, name, tail string) { + rd, err := p.cachedRepodata(r.Context(), store, fp, name) + if err != nil { + slog.Error("github_rpm: build repodata failed", "remote", name, "error", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + body, contentType := rd.other, "application/gzip" + switch { + case tail == "repomd.xml": + body, contentType = rd.repomd, "application/xml" + case strings.HasSuffix(tail, "-primary.xml.gz"): + body = rd.primary + case strings.HasSuffix(tail, "-filelists.xml.gz"): + body = rd.filelists + case !strings.HasSuffix(tail, "-other.xml.gz"): + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", contentType) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) +} + +func (p *GitHubProvider) cachedRepodata(ctx context.Context, store provider.RPMMetadataReader, fp rpmFingerprinter, name string) (*repodata, error) { + count, maxID, err := fp.RPMMetadataFingerprint(ctx, name) + if err != nil { + return nil, err + } + p.repodataMu.Lock() + c, ok := p.repodataCache[name] + p.repodataMu.Unlock() + if ok && c.count == count && c.maxID == maxID { + return c.repodata, nil + } + + v, err, _ := p.repodataGroup.Do(fmt.Sprintf("%s\x00%d\x00%d", name, count, maxID), func() (any, error) { + metas, err := store.ListRPMMetadataEntries(ctx, name) + if err != nil { + return nil, err + } + // Rows read after the fingerprint can only be newer than it, so the + // worst case is one extra rebuild on the next request. + rd := buildRepodata(metas) + p.repodataMu.Lock() + p.repodataCache[name] = cachedRepodata{count: count, maxID: maxID, repodata: rd} + p.repodataMu.Unlock() + return rd, nil + }) + if err != nil { + return nil, err + } + return v.(*repodata), nil +} + +func (p *GitHubProvider) dropRepodata(name string) { + p.repodataMu.Lock() + delete(p.repodataCache, name) + p.repodataMu.Unlock() +} + func (p *GitHubProvider) cacheEmpty(ctx context.Context, store provider.RemoteMetadataStore, name string) bool { + if fp, ok := store.(rpmFingerprinter); ok { + count, _, err := fp.RPMMetadataFingerprint(ctx, name) + return err == nil && count == 0 + } rows, err := store.ListRPMMetadataEntries(ctx, name) if err != nil { // Treat a failed read as "not empty" so a transient DB error becomes a diff --git a/internal/provider/rpm/github_repodata_cache_test.go b/internal/provider/rpm/github_repodata_cache_test.go new file mode 100644 index 0000000..611af6e --- /dev/null +++ b/internal/provider/rpm/github_repodata_cache_test.go @@ -0,0 +1,147 @@ +package rpm + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// fpStore adds a fingerprint and a full-read counter to fakeStore. +type fpStore struct { + *fakeStore + lists int + seq int64 +} + +func (s *fpStore) ListRPMMetadataEntries(ctx context.Context, name string) ([]provider.RPMMetadata, error) { + s.mu.Lock() + s.lists++ + s.mu.Unlock() + return s.fakeStore.ListRPMMetadataEntries(ctx, name) +} + +func (s *fpStore) InsertRPMMetadata(ctx context.Context, m *provider.RPMMetadata) error { + s.mu.Lock() + s.seq++ + s.mu.Unlock() + return s.fakeStore.InsertRPMMetadata(ctx, m) +} + +func (s *fpStore) RPMMetadataFingerprint(_ context.Context, _ string) (int64, int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + return int64(len(s.rows)), s.seq, nil +} + +func (s *fpStore) listCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.lists +} + +func newCacheFixture(t *testing.T, cache bool) (*GitHubProvider, *fpStore, models.Remote) { + t.Helper() + remote := models.Remote{Name: "cached-rpm", PackageType: models.PackageGitHubRPM, MutableTTL: 3600, CacheRepodata: cache} + p := newGitHubProvider() + p.lastScan[remote.Name] = time.Now() + store := &fpStore{fakeStore: newFakeStore()} + addPkg(t, store, "alpha") + return p, store, remote +} + +func addPkg(t *testing.T, store *fpStore, name string) { + t.Helper() + if err := store.InsertRPMMetadata(context.Background(), &provider.RPMMetadata{ + RepoName: "cached-rpm", FilePath: name + "-1-1.noarch.rpm", ContentHash: "sha256:" + name, + Name: name, Version: "1", Release: "1", Arch: "noarch", Group: "Unspecified", + Files: []provider.RPMFile{{Path: "/usr/bin/" + name}}, + }); err != nil { + t.Fatal(err) + } +} + +func serve(t *testing.T, p *GitHubProvider, remote models.Remote, store *fpStore, path string) []byte { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/x", nil) + if !p.ServeRemote(rec, req, remote, path, "https://proxy.example", store) { + t.Fatalf("%s not handled", path) + } + if rec.Code != http.StatusOK { + t.Fatalf("%s: code %d: %s", path, rec.Code, rec.Body.String()) + } + return rec.Body.Bytes() +} + +var repodataPaths = []string{ + "repodata/repomd.xml", + "repodata/x-primary.xml.gz", + "repodata/x-filelists.xml.gz", + "repodata/x-other.xml.gz", +} + +func TestCachedRepodataHitSkipsRegeneration(t *testing.T) { + p, store, remote := newCacheFixture(t, true) + + repomd := serve(t, p, remote, store, "repodata/repomd.xml") + primary := serve(t, p, remote, store, "repodata/x-primary.xml.gz") + for _, path := range repodataPaths { + serve(t, p, remote, store, path) + } + if n := store.listCount(); n != 1 { + t.Fatalf("full metadata reads = %d, want 1", n) + } + if !strings.Contains(string(repomd), sha256Hex(primary)+"-primary.xml.gz") { + t.Fatalf("cached repomd does not reference cached primary:\n%s", repomd) + } +} + +func TestCachedRepodataRebuildsOnDataChange(t *testing.T) { + p, store, remote := newCacheFixture(t, true) + before := serve(t, p, remote, store, "repodata/repomd.xml") + + addPkg(t, store, "beta") + after := serve(t, p, remote, store, "repodata/repomd.xml") + primary := serve(t, p, remote, store, "repodata/x-primary.xml.gz") + + if string(before) == string(after) { + t.Fatal("repomd unchanged after metadata change") + } + if !strings.Contains(string(after), sha256Hex(primary)+"-primary.xml.gz") { + t.Fatal("rebuilt repomd does not reference rebuilt primary") + } + if !strings.Contains(gunzip(t, primary), "beta") { + t.Fatal("rebuilt primary missing new package") + } + if n := store.listCount(); n != 2 { + t.Fatalf("full metadata reads = %d, want 2", n) + } +} + +func TestUncachedRepodataMatchesCachedAndReadsEveryTime(t *testing.T) { + p, store, remote := newCacheFixture(t, true) + cached := map[string]string{} + for _, path := range repodataPaths { + cached[path] = string(serve(t, p, remote, store, path)) + } + + remote.CacheRepodata = false + reads := store.listCount() + for _, path := range repodataPaths { + if got := string(serve(t, p, remote, store, path)); got != cached[path] { + t.Fatalf("%s differs between cached and uncached", path) + } + } + if n := store.listCount() - reads; n != len(repodataPaths) { + t.Fatalf("uncached full reads = %d, want %d", n, len(repodataPaths)) + } + if _, ok := p.repodataCache[remote.Name]; ok { + t.Fatal("disabling cache_repodata should drop the cached index") + } +} diff --git a/internal/provider/rpm/rpm.go b/internal/provider/rpm/rpm.go index a0f279b..723edfa 100644 --- a/internal/provider/rpm/rpm.go +++ b/internal/provider/rpm/rpm.go @@ -267,19 +267,27 @@ func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader pr return } - primary := generatePrimaryXMLGZ(metas) - filelists := generateFilelistsXMLGZ(metas) - other := generateOtherXMLGZ(metas) - - primaryHash := sha256Hex(primary) - filelistsHash := sha256Hex(filelists) - otherHash := sha256Hex(other) - - repomd := generateRepomd(repomdRevision(metas), primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other)) - w.Header().Set("Content-Type", "application/xml") w.WriteHeader(http.StatusOK) - w.Write(repomd) + _, _ = w.Write(buildRepodata(metas).repomd) +} + +// repodata is a complete generated index: repomd.xml plus the three gz files it references. +type repodata struct { + repomd, primary, filelists, other []byte +} + +func buildRepodata(metas []provider.RPMMetadata) *repodata { + rd := &repodata{ + primary: generatePrimaryXMLGZ(metas), + filelists: generateFilelistsXMLGZ(metas), + other: generateOtherXMLGZ(metas), + } + rd.repomd = generateRepomd(repomdRevision(metas), + sha256Hex(rd.primary), len(rd.primary), + sha256Hex(rd.filelists), len(rd.filelists), + sha256Hex(rd.other), len(rd.other)) + return rd } func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) { diff --git a/migrations/0006_remote_cache_repodata.sql b/migrations/0006_remote_cache_repodata.sql new file mode 100644 index 0000000..8fb7297 --- /dev/null +++ b/migrations/0006_remote_cache_repodata.sql @@ -0,0 +1 @@ +ALTER TABLE remotes ADD COLUMN IF NOT EXISTS cache_repodata BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/pkg/models/remote.go b/pkg/models/remote.go index fd27aa6..f377c68 100644 --- a/pkg/models/remote.go +++ b/pkg/models/remote.go @@ -77,6 +77,10 @@ type Remote struct { ReleasesRemote string `json:"releases_remote,omitempty"` ManagedBy string `json:"managed_by,omitempty"` + // CacheRepodata keeps a github_rpm remote's generated repodata in memory, + // rebuilding it only when its metadata rows change. + CacheRepodata bool `json:"cache_repodata"` + CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` }