Add parallel_fetch toggle for virtual member index files (#165)
Virtual members already merge concurrently, but each rpm member fetches primary/filelists/other one after another, and each deb member its Packages files the same way. On large virtuals the slowest member's serial chain pushes the merge past dnf's 30s timeout. - add `parallel_fetch` (default false) to virtuals: model, v2 API, migration 0007 - fetch a member's index files concurrently (up to 4) when set; first error still fails the member - thread the flag through rpm and deb member fetches Reviewed-on: #165 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #165.
This commit is contained in:
@@ -335,6 +335,40 @@ func TestDeleteRemoteDropsLocalFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualParallelFetchRoundTrip(t *testing.T) {
|
||||
requireDB(t)
|
||||
if err := testDB.CreateVirtual(ctx(), &models.Virtual{
|
||||
Name: "v-parallel", PackageType: models.PackageRPM, Members: []string{"a"}, ParallelFetch: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = testDB.DeleteVirtual(ctx(), "v-parallel") }()
|
||||
v, err := testDB.GetVirtual(ctx(), "v-parallel")
|
||||
if err != nil || !v.ParallelFetch {
|
||||
t.Fatalf("created parallel_fetch = %v, %v; want true", v, err)
|
||||
}
|
||||
v.ParallelFetch = false
|
||||
if err := testDB.UpdateVirtual(ctx(), v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
vs, err := testDB.ListVirtuals(ctx())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, l := range vs {
|
||||
if l.Name == "v-parallel" && l.ParallelFetch {
|
||||
t.Fatal("updated parallel_fetch not persisted")
|
||||
}
|
||||
}
|
||||
if err := testDB.CreateVirtual(ctx(), &models.Virtual{Name: "v-serial", PackageType: models.PackageRPM, Members: []string{"a"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = testDB.DeleteVirtual(ctx(), "v-serial") }()
|
||||
if v, err := testDB.GetVirtual(ctx(), "v-serial"); err != nil || v.ParallelFetch {
|
||||
t.Fatalf("default parallel_fetch = %v, %v; want false", v, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVirtualsCRUD(t *testing.T) {
|
||||
requireDB(t)
|
||||
if err := testDB.CreateVirtual(ctx(), &models.Virtual{
|
||||
|
||||
Reference in New Issue
Block a user