Add parallel_fetch toggle for virtual member index files (#165)

Virtual members already merge concurrently, but each rpm member fetches primary/filelists/other one after another, and each deb member its Packages files the same way. On large virtuals the slowest member's serial chain pushes the merge past dnf's 30s timeout.

- add `parallel_fetch` (default false) to virtuals: model, v2 API, migration 0007
- fetch a member's index files concurrently (up to 4) when set; first error still fails the member
- thread the flag through rpm and deb member fetches

Reviewed-on: #165
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #165.
This commit is contained in:
2026-10-25 15:13:57 +11:00
committed by BenVincent
parent 9294686ea6
commit cfb1f93d48
7 changed files with 395 additions and 51 deletions
+34
View File
@@ -335,6 +335,40 @@ func TestDeleteRemoteDropsLocalFiles(t *testing.T) {
}
}
func TestVirtualParallelFetchRoundTrip(t *testing.T) {
requireDB(t)
if err := testDB.CreateVirtual(ctx(), &models.Virtual{
Name: "v-parallel", PackageType: models.PackageRPM, Members: []string{"a"}, ParallelFetch: true,
}); err != nil {
t.Fatal(err)
}
defer func() { _ = testDB.DeleteVirtual(ctx(), "v-parallel") }()
v, err := testDB.GetVirtual(ctx(), "v-parallel")
if err != nil || !v.ParallelFetch {
t.Fatalf("created parallel_fetch = %v, %v; want true", v, err)
}
v.ParallelFetch = false
if err := testDB.UpdateVirtual(ctx(), v); err != nil {
t.Fatal(err)
}
vs, err := testDB.ListVirtuals(ctx())
if err != nil {
t.Fatal(err)
}
for _, l := range vs {
if l.Name == "v-parallel" && l.ParallelFetch {
t.Fatal("updated parallel_fetch not persisted")
}
}
if err := testDB.CreateVirtual(ctx(), &models.Virtual{Name: "v-serial", PackageType: models.PackageRPM, Members: []string{"a"}}); err != nil {
t.Fatal(err)
}
defer func() { _ = testDB.DeleteVirtual(ctx(), "v-serial") }()
if v, err := testDB.GetVirtual(ctx(), "v-serial"); err != nil || v.ParallelFetch {
t.Fatalf("default parallel_fetch = %v, %v; want false", v, err)
}
}
func TestVirtualsCRUD(t *testing.T) {
requireDB(t)
if err := testDB.CreateVirtual(ctx(), &models.Virtual{