Add cache_repodata toggle for github_rpm remotes
Cache generated repodata per replica and rebuild only when the remote's rpm_metadata fingerprint (count, max id) changes.
This commit is contained in:
@@ -79,13 +79,17 @@ func TestRemotesCRUD(t *testing.T) {
|
||||
if err != nil || got.BaseURL != "https://example.com" {
|
||||
t.Fatalf("get: %v %v", got, err)
|
||||
}
|
||||
if got.CacheRepodata {
|
||||
t.Error("cache_repodata should default to false")
|
||||
}
|
||||
got.BaseURL = "https://updated.example.com"
|
||||
got.CacheRepodata = true
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-crud")
|
||||
if got.BaseURL != "https://updated.example.com" {
|
||||
t.Errorf("update not applied: %v", got.BaseURL)
|
||||
if got.BaseURL != "https://updated.example.com" || !got.CacheRepodata {
|
||||
t.Errorf("update not applied: %v %v", got.BaseURL, got.CacheRepodata)
|
||||
}
|
||||
list, err := testDB.ListRemotes(ctx())
|
||||
if err != nil || len(list) == 0 {
|
||||
@@ -453,3 +457,44 @@ func TestRPMMetadata(t *testing.T) {
|
||||
t.Fatalf("list rpm rows: %v %v", len(rows), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMMetadataFingerprint(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-rpm-fp")
|
||||
fp := func() [2]int64 {
|
||||
t.Helper()
|
||||
c, m, err := testDB.RPMMetadataFingerprint(ctx(), "r-rpm-fp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return [2]int64{c, m}
|
||||
}
|
||||
insert := func(path string) {
|
||||
t.Helper()
|
||||
if err := testDB.InsertRPMMetadata(ctx(), &provider.RPMMetadata{
|
||||
RepoName: "r-rpm-fp", FilePath: path, ContentHash: "sha256:" + path,
|
||||
Name: path, Version: "1", Release: "1", Arch: "noarch",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if got := fp(); got != [2]int64{0, 0} {
|
||||
t.Fatalf("empty fingerprint = %v", got)
|
||||
}
|
||||
insert("a.rpm")
|
||||
one := fp()
|
||||
insert("a.rpm")
|
||||
if fp() != one {
|
||||
t.Fatal("conflicting insert changed fingerprint")
|
||||
}
|
||||
insert("b.rpm")
|
||||
two := fp()
|
||||
if err := testDB.DeleteRPMMetadata(ctx(), "r-rpm-fp", "a.rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterDelete := fp()
|
||||
if one[0] != 1 || two[0] != 2 || afterDelete[0] != 1 || afterDelete == one || afterDelete == two {
|
||||
t.Fatalf("fingerprints did not track changes: %v %v %v", one, two, afterDelete)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
cache_repodata,
|
||||
created_at, updated_at`
|
||||
|
||||
// normalizeMirrorStrategy maps an empty strategy to the round_robin default so
|
||||
@@ -35,6 +36,7 @@ func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error
|
||||
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
||||
&r.ReleasesRemote, &r.ManagedBy,
|
||||
&r.UpstreamDialTimeout, &r.UpstreamTLSTimeout, &r.UpstreamResponseHeaderTimeout,
|
||||
&r.CacheRepodata,
|
||||
&r.CreatedAt, &r.UpdatedAt,
|
||||
)
|
||||
}
|
||||
@@ -76,8 +78,8 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
mirror_strategy
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26)
|
||||
mirror_strategy, cache_repodata
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
@@ -86,7 +88,7 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy),
|
||||
normalizeMirrorStrategy(r.MirrorStrategy), r.CacheRepodata,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -101,7 +103,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
mirror_strategy=$26,
|
||||
mirror_strategy=$26, cache_repodata=$27,
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
@@ -113,7 +115,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
r.Mirrorlist,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy),
|
||||
normalizeMirrorStrategy(r.MirrorStrategy), r.CacheRepodata,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -40,6 +40,13 @@ func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string)
|
||||
return err
|
||||
}
|
||||
|
||||
// RPMMetadataFingerprint is a cheap change detector for a repo's rows: they are
|
||||
// only ever inserted or deleted, so (count, max id) moves whenever they change.
|
||||
func (db *DB) RPMMetadataFingerprint(ctx context.Context, repoName string) (count, maxID int64, err error) {
|
||||
err = db.Pool.QueryRow(ctx, `SELECT count(*), coalesce(max(id), 0) FROM rpm_metadata WHERE repo_name = $1`, repoName).Scan(&count, &maxID)
|
||||
return count, maxID, err
|
||||
}
|
||||
|
||||
type RPMMetadataRow struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
|
||||
Reference in New Issue
Block a user