Add cache_repodata toggle for github_rpm remotes

Cache generated repodata per replica and rebuild only when the remote's
rpm_metadata fingerprint (count, max id) changes.

docs/rpm/remote-github.md

@@ -53,6 +53,14 @@ derived metadata current off the client request path:
   ~once per `mutable_ttl` regardless of replica count, and the shared `etag`
   lets any replica issue the conditional request.
 
+### Repodata cache
+
+Repodata is regenerated from the metadata rows on every request by default. Set
+`cache_repodata: true` on the remote to keep the generated `repomd.xml` and its
+`primary`/`filelists`/`other` files in memory per replica. Each request checks a
+cheap row fingerprint (count + newest id) in the database and rebuilds only when
+it moved, so every replica picks up a scan's changes on its next request.
+
 ```hcl
 # Backend that serves the actual .rpm bytes from github.com.
 resource "artifactapi_remote_generic" "github" {

internal/database/database_test.go

@@ -79,13 +79,17 @@ func TestRemotesCRUD(t *testing.T) {
 	if err != nil || got.BaseURL != "https://example.com" {
 		t.Fatalf("get: %v %v", got, err)
 	}
+	if got.CacheRepodata {
+		t.Error("cache_repodata should default to false")
+	}
 	got.BaseURL = "https://updated.example.com"
+	got.CacheRepodata = true
 	if err := testDB.UpdateRemote(ctx(), got); err != nil {
 		t.Fatalf("update: %v", err)
 	}
 	got, _ = testDB.GetRemote(ctx(), "r-crud")
-	if got.BaseURL != "https://updated.example.com" {
-		t.Errorf("update not applied: %v", got.BaseURL)
+	if got.BaseURL != "https://updated.example.com" || !got.CacheRepodata {
+		t.Errorf("update not applied: %v %v", got.BaseURL, got.CacheRepodata)
 	}
 	list, err := testDB.ListRemotes(ctx())
 	if err != nil || len(list) == 0 {
@@ -453,3 +457,44 @@ func TestRPMMetadata(t *testing.T) {
 		t.Fatalf("list rpm rows: %v %v", len(rows), err)
 	}
 }
+
+func TestRPMMetadataFingerprint(t *testing.T) {
+	requireDB(t)
+	seedRemote(t, "r-rpm-fp")
+	fp := func() [2]int64 {
+		t.Helper()
+		c, m, err := testDB.RPMMetadataFingerprint(ctx(), "r-rpm-fp")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return [2]int64{c, m}
+	}
+	insert := func(path string) {
+		t.Helper()
+		if err := testDB.InsertRPMMetadata(ctx(), &provider.RPMMetadata{
+			RepoName: "r-rpm-fp", FilePath: path, ContentHash: "sha256:" + path,
+			Name: path, Version: "1", Release: "1", Arch: "noarch",
+		}); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	if got := fp(); got != [2]int64{0, 0} {
+		t.Fatalf("empty fingerprint = %v", got)
+	}
+	insert("a.rpm")
+	one := fp()
+	insert("a.rpm")
+	if fp() != one {
+		t.Fatal("conflicting insert changed fingerprint")
+	}
+	insert("b.rpm")
+	two := fp()
+	if err := testDB.DeleteRPMMetadata(ctx(), "r-rpm-fp", "a.rpm"); err != nil {
+		t.Fatal(err)
+	}
+	afterDelete := fp()
+	if one[0] != 1 || two[0] != 2 || afterDelete[0] != 1 || afterDelete == one || afterDelete == two {
+		t.Fatalf("fingerprints did not track changes: %v %v %v", one, two, afterDelete)
+	}
+}

internal/database/remotes.go

@@ -15,6 +15,7 @@ const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_
 	quarantine_enabled, quarantine_days, stale_on_error,
 	releases_remote, managed_by,
 	upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
+	cache_repodata,
 	created_at, updated_at`
 
 // normalizeMirrorStrategy maps an empty strategy to the round_robin default so
@@ -35,6 +36,7 @@ func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error
 		&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
 		&r.ReleasesRemote, &r.ManagedBy,
 		&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
+		&r.CacheRepodata,
 		&r.CreatedAt, &r.UpdatedAt,
 	)
 }
@@ -76,8 +78,8 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
 			quarantine_enabled, quarantine_days, stale_on_error,
 			releases_remote, managed_by,
 			upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
-			mirror_strategy
-		) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26)
+			mirror_strategy, cache_repodata
+		) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27)
 	`,
 		r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
 		r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
@@ -86,7 +88,7 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
 		r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
 		r.ReleasesRemote, r.ManagedBy,
 		r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
-		normalizeMirrorStrategy(r.MirrorStrategy),
+		normalizeMirrorStrategy(r.MirrorStrategy), r.CacheRepodata,
 	)
 	return err
 }
@@ -101,7 +103,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
 			quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
 			releases_remote=$20, managed_by=$21,
 			upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
-			mirror_strategy=$26,
+			mirror_strategy=$26, cache_repodata=$27,
 			updated_at=NOW()
 		WHERE name=$1
 	`,
@@ -113,7 +115,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
 		r.ReleasesRemote, r.ManagedBy,
 		r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
 		r.Mirrorlist,
-		normalizeMirrorStrategy(r.MirrorStrategy),
+		normalizeMirrorStrategy(r.MirrorStrategy), r.CacheRepodata,
 	)
 	return err
 }

internal/database/rpm_metadata.go

@@ -40,6 +40,13 @@ func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string)
 	return err
 }
 
+// RPMMetadataFingerprint is a cheap change detector for a repo's rows: they are
+// only ever inserted or deleted, so (count, max id) moves whenever they change.
+func (db *DB) RPMMetadataFingerprint(ctx context.Context, repoName string) (count, maxID int64, err error) {
+	err = db.Pool.QueryRow(ctx, `SELECT count(*), coalesce(max(id), 0) FROM rpm_metadata WHERE repo_name = $1`, repoName).Scan(&count, &maxID)
+	return count, maxID, err
+}
+
 type RPMMetadataRow struct {
 	RepoName      string
 	FilePath      string

internal/provider/rpm/github.go

@@ -18,6 +18,7 @@ import (
 	"time"
 
 	rpmlib "github.com/cavaliergopher/rpm"
+	"golang.org/x/sync/singleflight"
 	"golang.org/x/time/rate"
 
 	"git.unkin.net/unkin/artifactapi/internal/githubauth"
@@ -84,6 +85,22 @@ type GitHubProvider struct {
 	mu       sync.Mutex
 	scanning map[string]bool
 	lastScan map[string]time.Time
+
+	// repodata caches generated indexes for cache_repodata remotes, keyed by remote name.
+	repodataMu    sync.Mutex
+	repodataCache map[string]cachedRepodata
+	repodataGroup singleflight.Group
+}
+
+// rpmFingerprinter is a cheap change detector over a remote's metadata rows.
+// Rows are insert-or-delete only, so (count, max id) changes whenever they do.
+type rpmFingerprinter interface {
+	RPMMetadataFingerprint(ctx context.Context, repoName string) (count, maxID int64, err error)
+}
+
+type cachedRepodata struct {
+	count, maxID int64
+	*repodata
 }
 
 func newGitHubProvider() *GitHubProvider {
@@ -97,6 +114,7 @@ func newGitHubProvider() *GitHubProvider {
 		coldWait:      defaultColdWait,
 		scanning:      map[string]bool{},
 		lastScan:      map[string]time.Time{},
+		repodataCache: map[string]cachedRepodata{},
 	}
 }
 
@@ -170,6 +188,11 @@ func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, rem
 		}
 
 		tail := strings.TrimPrefix(path, "repodata/")
+		if fp, ok := store.(rpmFingerprinter); ok && remote.CacheRepodata {
+			p.serveCachedRepodata(w, sr, store, fp, remote.Name, tail)
+			return true
+		}
+		p.dropRepodata(remote.Name)
 		lp := &Provider{}
 		switch {
 		case tail == "repomd.xml":
@@ -239,7 +262,76 @@ func (p *GitHubProvider) ensurePrimed(ctx context.Context, remote models.Remote,
 	return false
 }
 
+// serveCachedRepodata serves a file from the generated index, rebuilding it
+// only when the rows' fingerprint moved. The fingerprint is read from the shared
+// database on every request, so each replica rebuilds as soon as any replica's
+// scan changes rows and repomd never outlives the files it references.
+func (p *GitHubProvider) serveCachedRepodata(w http.ResponseWriter, r *http.Request, store provider.RPMMetadataReader, fp rpmFingerprinter, name, tail string) {
+	rd, err := p.cachedRepodata(r.Context(), store, fp, name)
+	if err != nil {
+		slog.Error("github_rpm: build repodata failed", "remote", name, "error", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	body, contentType := rd.other, "application/gzip"
+	switch {
+	case tail == "repomd.xml":
+		body, contentType = rd.repomd, "application/xml"
+	case strings.HasSuffix(tail, "-primary.xml.gz"):
+		body = rd.primary
+	case strings.HasSuffix(tail, "-filelists.xml.gz"):
+		body = rd.filelists
+	case !strings.HasSuffix(tail, "-other.xml.gz"):
+		http.Error(w, "not found", http.StatusNotFound)
+		return
+	}
+	w.Header().Set("Content-Type", contentType)
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write(body)
+}
+
+func (p *GitHubProvider) cachedRepodata(ctx context.Context, store provider.RPMMetadataReader, fp rpmFingerprinter, name string) (*repodata, error) {
+	count, maxID, err := fp.RPMMetadataFingerprint(ctx, name)
+	if err != nil {
+		return nil, err
+	}
+	p.repodataMu.Lock()
+	c, ok := p.repodataCache[name]
+	p.repodataMu.Unlock()
+	if ok && c.count == count && c.maxID == maxID {
+		return c.repodata, nil
+	}
+
+	v, err, _ := p.repodataGroup.Do(fmt.Sprintf("%s\x00%d\x00%d", name, count, maxID), func() (any, error) {
+		metas, err := store.ListRPMMetadataEntries(ctx, name)
+		if err != nil {
+			return nil, err
+		}
+		// Rows read after the fingerprint can only be newer than it, so the
+		// worst case is one extra rebuild on the next request.
+		rd := buildRepodata(metas)
+		p.repodataMu.Lock()
+		p.repodataCache[name] = cachedRepodata{count: count, maxID: maxID, repodata: rd}
+		p.repodataMu.Unlock()
+		return rd, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return v.(*repodata), nil
+}
+
+func (p *GitHubProvider) dropRepodata(name string) {
+	p.repodataMu.Lock()
+	delete(p.repodataCache, name)
+	p.repodataMu.Unlock()
+}
+
 func (p *GitHubProvider) cacheEmpty(ctx context.Context, store provider.RemoteMetadataStore, name string) bool {
+	if fp, ok := store.(rpmFingerprinter); ok {
+		count, _, err := fp.RPMMetadataFingerprint(ctx, name)
+		return err == nil && count == 0
+	}
 	rows, err := store.ListRPMMetadataEntries(ctx, name)
 	if err != nil {
 		// Treat a failed read as "not empty" so a transient DB error becomes a

internal/provider/rpm/github_repodata_cache_test.go

@@ -0,0 +1,147 @@
+package rpm
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+// fpStore adds a fingerprint and a full-read counter to fakeStore.
+type fpStore struct {
+	*fakeStore
+	lists int
+	seq   int64
+}
+
+func (s *fpStore) ListRPMMetadataEntries(ctx context.Context, name string) ([]provider.RPMMetadata, error) {
+	s.mu.Lock()
+	s.lists++
+	s.mu.Unlock()
+	return s.fakeStore.ListRPMMetadataEntries(ctx, name)
+}
+
+func (s *fpStore) InsertRPMMetadata(ctx context.Context, m *provider.RPMMetadata) error {
+	s.mu.Lock()
+	s.seq++
+	s.mu.Unlock()
+	return s.fakeStore.InsertRPMMetadata(ctx, m)
+}
+
+func (s *fpStore) RPMMetadataFingerprint(_ context.Context, _ string) (int64, int64, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return int64(len(s.rows)), s.seq, nil
+}
+
+func (s *fpStore) listCount() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.lists
+}
+
+func newCacheFixture(t *testing.T, cache bool) (*GitHubProvider, *fpStore, models.Remote) {
+	t.Helper()
+	remote := models.Remote{Name: "cached-rpm", PackageType: models.PackageGitHubRPM, MutableTTL: 3600, CacheRepodata: cache}
+	p := newGitHubProvider()
+	p.lastScan[remote.Name] = time.Now()
+	store := &fpStore{fakeStore: newFakeStore()}
+	addPkg(t, store, "alpha")
+	return p, store, remote
+}
+
+func addPkg(t *testing.T, store *fpStore, name string) {
+	t.Helper()
+	if err := store.InsertRPMMetadata(context.Background(), &provider.RPMMetadata{
+		RepoName: "cached-rpm", FilePath: name + "-1-1.noarch.rpm", ContentHash: "sha256:" + name,
+		Name: name, Version: "1", Release: "1", Arch: "noarch", Group: "Unspecified",
+		Files: []provider.RPMFile{{Path: "/usr/bin/" + name}},
+	}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func serve(t *testing.T, p *GitHubProvider, remote models.Remote, store *fpStore, path string) []byte {
+	t.Helper()
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/x", nil)
+	if !p.ServeRemote(rec, req, remote, path, "https://proxy.example", store) {
+		t.Fatalf("%s not handled", path)
+	}
+	if rec.Code != http.StatusOK {
+		t.Fatalf("%s: code %d: %s", path, rec.Code, rec.Body.String())
+	}
+	return rec.Body.Bytes()
+}
+
+var repodataPaths = []string{
+	"repodata/repomd.xml",
+	"repodata/x-primary.xml.gz",
+	"repodata/x-filelists.xml.gz",
+	"repodata/x-other.xml.gz",
+}
+
+func TestCachedRepodataHitSkipsRegeneration(t *testing.T) {
+	p, store, remote := newCacheFixture(t, true)
+
+	repomd := serve(t, p, remote, store, "repodata/repomd.xml")
+	primary := serve(t, p, remote, store, "repodata/x-primary.xml.gz")
+	for _, path := range repodataPaths {
+		serve(t, p, remote, store, path)
+	}
+	if n := store.listCount(); n != 1 {
+		t.Fatalf("full metadata reads = %d, want 1", n)
+	}
+	if !strings.Contains(string(repomd), sha256Hex(primary)+"-primary.xml.gz") {
+		t.Fatalf("cached repomd does not reference cached primary:\n%s", repomd)
+	}
+}
+
+func TestCachedRepodataRebuildsOnDataChange(t *testing.T) {
+	p, store, remote := newCacheFixture(t, true)
+	before := serve(t, p, remote, store, "repodata/repomd.xml")
+
+	addPkg(t, store, "beta")
+	after := serve(t, p, remote, store, "repodata/repomd.xml")
+	primary := serve(t, p, remote, store, "repodata/x-primary.xml.gz")
+
+	if string(before) == string(after) {
+		t.Fatal("repomd unchanged after metadata change")
+	}
+	if !strings.Contains(string(after), sha256Hex(primary)+"-primary.xml.gz") {
+		t.Fatal("rebuilt repomd does not reference rebuilt primary")
+	}
+	if !strings.Contains(gunzip(t, primary), "<name>beta</name>") {
+		t.Fatal("rebuilt primary missing new package")
+	}
+	if n := store.listCount(); n != 2 {
+		t.Fatalf("full metadata reads = %d, want 2", n)
+	}
+}
+
+func TestUncachedRepodataMatchesCachedAndReadsEveryTime(t *testing.T) {
+	p, store, remote := newCacheFixture(t, true)
+	cached := map[string]string{}
+	for _, path := range repodataPaths {
+		cached[path] = string(serve(t, p, remote, store, path))
+	}
+
+	remote.CacheRepodata = false
+	reads := store.listCount()
+	for _, path := range repodataPaths {
+		if got := string(serve(t, p, remote, store, path)); got != cached[path] {
+			t.Fatalf("%s differs between cached and uncached", path)
+		}
+	}
+	if n := store.listCount() - reads; n != len(repodataPaths) {
+		t.Fatalf("uncached full reads = %d, want %d", n, len(repodataPaths))
+	}
+	if _, ok := p.repodataCache[remote.Name]; ok {
+		t.Fatal("disabling cache_repodata should drop the cached index")
+	}
+}

internal/provider/rpm/rpm.go

@@ -267,19 +267,27 @@ func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader pr
 		return
 	}
 
-	primary := generatePrimaryXMLGZ(metas)
-	filelists := generateFilelistsXMLGZ(metas)
-	other := generateOtherXMLGZ(metas)
-
-	primaryHash := sha256Hex(primary)
-	filelistsHash := sha256Hex(filelists)
-	otherHash := sha256Hex(other)
-
-	repomd := generateRepomd(repomdRevision(metas), primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
-
 	w.Header().Set("Content-Type", "application/xml")
 	w.WriteHeader(http.StatusOK)
-	w.Write(repomd)
+	_, _ = w.Write(buildRepodata(metas).repomd)
+}
+
+// repodata is a complete generated index: repomd.xml plus the three gz files it references.
+type repodata struct {
+	repomd, primary, filelists, other []byte
+}
+
+func buildRepodata(metas []provider.RPMMetadata) *repodata {
+	rd := &repodata{
+		primary:   generatePrimaryXMLGZ(metas),
+		filelists: generateFilelistsXMLGZ(metas),
+		other:     generateOtherXMLGZ(metas),
+	}
+	rd.repomd = generateRepomd(repomdRevision(metas),
+		sha256Hex(rd.primary), len(rd.primary),
+		sha256Hex(rd.filelists), len(rd.filelists),
+		sha256Hex(rd.other), len(rd.other))
+	return rd
 }
 
 func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {

migrations/0006_remote_cache_repodata.sql

@@ -0,0 +1 @@
+ALTER TABLE remotes ADD COLUMN IF NOT EXISTS cache_repodata BOOLEAN NOT NULL DEFAULT FALSE;

pkg/models/remote.go

@@ -77,6 +77,10 @@ type Remote struct {
 	ReleasesRemote string `json:"releases_remote,omitempty"`
 	ManagedBy      string `json:"managed_by,omitempty"`
 
+	// CacheRepodata keeps a github_rpm remote's generated repodata in memory,
+	// rebuilding it only when its metadata rows change.
+	CacheRepodata bool `json:"cache_repodata"`
+
 	CreatedAt time.Time `json:"created_at"`
 	UpdatedAt time.Time `json:"updated_at"`
 }