Add cache_repodata toggle for github_rpm remotes (#168)
github_rpm remotes regenerate repomd and all three gz files from every metadata row on each request, which is expensive for large repos under repeated dnf traffic. - add a per-remote `cache_repodata` toggle (column, model, v2 API, docs) - cache built repodata per replica, rebuilt via singleflight when the `count(*)`+`max(id)` row fingerprint moves - store a build only if no newer build has landed; evict idle entries hourly - map cached-path DB errors like the uncached path: 503 + Retry-After on cancel/deadline, generic 500 otherwise - check `cacheEmpty` via the fingerprint for all github_rpm remotes, not only cached ones Reviewed-on: #168 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #168.
This commit is contained in:
@@ -79,13 +79,19 @@ func TestRemotesCRUD(t *testing.T) {
|
||||
if err != nil || got.BaseURL != "https://example.com" {
|
||||
t.Fatalf("get: %v %v", got, err)
|
||||
}
|
||||
if got.CacheRepodata {
|
||||
t.Error("cache_repodata should default to false")
|
||||
}
|
||||
got.BaseURL = "https://updated.example.com"
|
||||
got.CacheRepodata = true
|
||||
got.SyncInterval = 600
|
||||
got.ServeStale = true
|
||||
if err := testDB.UpdateRemote(ctx(), got); err != nil {
|
||||
t.Fatalf("update: %v", err)
|
||||
}
|
||||
got, _ = testDB.GetRemote(ctx(), "r-crud")
|
||||
if got.BaseURL != "https://updated.example.com" {
|
||||
t.Errorf("update not applied: %v", got.BaseURL)
|
||||
if got.BaseURL != "https://updated.example.com" || !got.CacheRepodata || got.SyncInterval != 600 || !got.ServeStale {
|
||||
t.Errorf("update not applied: %v %v %v %v", got.BaseURL, got.CacheRepodata, got.SyncInterval, got.ServeStale)
|
||||
}
|
||||
list, err := testDB.ListRemotes(ctx())
|
||||
if err != nil || len(list) == 0 {
|
||||
@@ -487,3 +493,44 @@ func TestRPMMetadata(t *testing.T) {
|
||||
t.Fatalf("list rpm rows: %v %v", len(rows), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRPMMetadataFingerprint(t *testing.T) {
|
||||
requireDB(t)
|
||||
seedRemote(t, "r-rpm-fp")
|
||||
fp := func() [2]int64 {
|
||||
t.Helper()
|
||||
c, m, err := testDB.RPMMetadataFingerprint(ctx(), "r-rpm-fp")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return [2]int64{c, m}
|
||||
}
|
||||
insert := func(path string) {
|
||||
t.Helper()
|
||||
if err := testDB.InsertRPMMetadata(ctx(), &provider.RPMMetadata{
|
||||
RepoName: "r-rpm-fp", FilePath: path, ContentHash: "sha256:" + path,
|
||||
Name: path, Version: "1", Release: "1", Arch: "noarch",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
if got := fp(); got != [2]int64{0, 0} {
|
||||
t.Fatalf("empty fingerprint = %v", got)
|
||||
}
|
||||
insert("a.rpm")
|
||||
one := fp()
|
||||
insert("a.rpm")
|
||||
if fp() != one {
|
||||
t.Fatal("conflicting insert changed fingerprint")
|
||||
}
|
||||
insert("b.rpm")
|
||||
two := fp()
|
||||
if err := testDB.DeleteRPMMetadata(ctx(), "r-rpm-fp", "a.rpm"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterDelete := fp()
|
||||
if one[0] != 1 || two[0] != 2 || afterDelete[0] != 1 || afterDelete == one || afterDelete == two {
|
||||
t.Fatalf("fingerprints did not track changes: %v %v %v", one, two, afterDelete)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user