Add parallel_fetch toggle for virtual member index files

internal/virtual/engine_test.go

@@ -19,7 +19,7 @@ func fakeEngine(members map[string]*RPMMember) *Engine {
 		getRemote: func(_ context.Context, name string) (*models.Remote, error) {
 			return &models.Remote{Name: name, RepoType: models.RepoTypeRemote}, nil
 		},
-		rpmMember: func(_ context.Context, name string) (*RPMMember, error) {
+		rpmMember: func(_ context.Context, name string, _ bool) (*RPMMember, error) {
 			if m := members[name]; m != nil {
 				return m, nil
 			}
@@ -75,7 +75,7 @@ func TestFetchRPMLocalMemberWithoutRepodataIsUpstreamError(t *testing.T) {
 	e := fakeEngine(map[string]*RPMMember{
 		"a": {RemoteName: "a", Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}},
 	})
-	e.rpmMember = func(_ context.Context, name string) (*RPMMember, error) {
+	e.rpmMember = func(_ context.Context, name string, _ bool) (*RPMMember, error) {
 		if name == "local" {
 			return nil, fmt.Errorf("local/repodata/repomd.xml: %w", ErrNotFound)
 		}
@@ -91,7 +91,7 @@ func TestFetchRPMMergeIsShared(t *testing.T) {
 	var calls atomic.Int32
 	e := fakeEngine(nil)
 	e.mergeTTL = time.Minute
-	e.rpmMember = func(_ context.Context, name string) (*RPMMember, error) {
+	e.rpmMember = func(_ context.Context, name string, _ bool) (*RPMMember, error) {
 		calls.Add(1)
 		time.Sleep(20 * time.Millisecond)
 		return &RPMMember{RemoteName: name, Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", "aaa", "foo.rpm"))}}, nil
@@ -160,7 +160,7 @@ func TestMemberRedirect(t *testing.T) {
 
 func debEngine(members map[string]*DebMember) *Engine {
 	e := fakeEngine(nil)
-	e.debMember = func(_ context.Context, name, base string) (*DebMember, error) {
+	e.debMember = func(_ context.Context, name, base string, _ bool) (*DebMember, error) {
 		if m := members[name]; m != nil && base == "" {
 			return m, nil
 		}
@@ -213,7 +213,7 @@ func TestFetchDebMergeIsSharedPerSuite(t *testing.T) {
 	var calls atomic.Int32
 	e := fakeEngine(nil)
 	e.mergeTTL = time.Minute
-	e.debMember = func(_ context.Context, name, base string) (*DebMember, error) {
+	e.debMember = func(_ context.Context, name, base string, _ bool) (*DebMember, error) {
 		calls.Add(1)
 		m := flatDebMember(name, "", debStanza("foo", "1", "amd64", "pool/foo.deb"))
 		return &m, nil
@@ -307,7 +307,9 @@ func suiteDebMember(name, version string) *DebMember {
 func TestFetchDebByHashSurvivesRemerge(t *testing.T) {
 	version := "1"
 	e := fakeEngine(nil)
-	e.debMember = func(_ context.Context, name, _ string) (*DebMember, error) { return suiteDebMember(name, version), nil }
+	e.debMember = func(_ context.Context, name, _ string, _ bool) (*DebMember, error) {
+		return suiteDebMember(name, version), nil
+	}
 	virt := debVirt("a")
 	fetch := func(p string) []byte {
 		body, _, err := e.Fetch(context.Background(), virt, p, "")
@@ -340,7 +342,9 @@ func TestFetchDebByHashSurvivesRemerge(t *testing.T) {
 func TestFetchDebBoundsSuitesPerVirtual(t *testing.T) {
 	e := fakeEngine(nil)
 	e.mergeTTL = time.Minute
-	e.debMember = func(_ context.Context, name, _ string) (*DebMember, error) { return suiteDebMember(name, "1"), nil }
+	e.debMember = func(_ context.Context, name, _ string, _ bool) (*DebMember, error) {
+		return suiteDebMember(name, "1"), nil
+	}
 	other := models.Virtual{Name: "other", PackageType: models.PackageDeb, Members: []string{"a"}}
 	if _, _, err := e.Fetch(context.Background(), other, "dists/keep/Release", ""); err != nil {
 		t.Fatal(err)
@@ -367,3 +371,101 @@ func TestFetchDebBoundsSuitesPerVirtual(t *testing.T) {
 		t.Fatal("another virtual's suites must not be evicted")
 	}
 }
+
+func fetchFilesInput(n int) map[string]string {
+	files := map[string]string{}
+	for i := range n {
+		files[fmt.Sprintf("k%d", i)] = fmt.Sprintf("f%d", i)
+	}
+	return files
+}
+
+func TestFetchFilesParallelRunsConcurrently(t *testing.T) {
+	var inFlight, peak atomic.Int32
+	release := make(chan struct{})
+	go func() {
+		for peak.Load() < maxFileFetches {
+			time.Sleep(time.Millisecond)
+		}
+		close(release)
+	}()
+	got, err := fetchFiles(true, fetchFilesInput(6), func(f string) ([]byte, error) {
+		n := inFlight.Add(1)
+		defer inFlight.Add(-1)
+		for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
+		}
+		select {
+		case <-release:
+		case <-time.After(5 * time.Second):
+			return nil, errors.New("fetches never overlapped")
+		}
+		return []byte(f), nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if peak.Load() != maxFileFetches {
+		t.Fatalf("peak concurrency %d, want %d", peak.Load(), maxFileFetches)
+	}
+	for k, f := range fetchFilesInput(6) {
+		if string(got[k]) != f {
+			t.Fatalf("%s = %q, want %q", k, got[k], f)
+		}
+	}
+}
+
+func TestFetchFilesSerialByDefault(t *testing.T) {
+	var inFlight atomic.Int32
+	_, err := fetchFiles(false, fetchFilesInput(4), func(f string) ([]byte, error) {
+		defer inFlight.Add(-1)
+		if inFlight.Add(1) > 1 {
+			return nil, errors.New("concurrent fetch")
+		}
+		time.Sleep(time.Millisecond)
+		return []byte(f), nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFetchFilesErrorFailsMember(t *testing.T) {
+	boom := errors.New("boom")
+	for _, parallel := range []bool{false, true} {
+		got, err := fetchFiles(parallel, fetchFilesInput(5), func(f string) ([]byte, error) {
+			if f == "f2" {
+				return nil, boom
+			}
+			return []byte(f), nil
+		})
+		if !errors.Is(err, boom) || got != nil {
+			t.Fatalf("parallel=%v: got %v, %v; want boom and no files", parallel, got, err)
+		}
+	}
+}
+
+func TestMergeRPMPassesParallelFetchAndKeepsMemberOrder(t *testing.T) {
+	e := fakeEngine(nil)
+	var sawParallel atomic.Bool
+	e.rpmMember = func(_ context.Context, name string, parallel bool) (*RPMMember, error) {
+		sawParallel.Store(parallel)
+		if name == "a" {
+			time.Sleep(20 * time.Millisecond)
+		}
+		return &RPMMember{RemoteName: name, Data: map[string][]byte{"primary": primaryXML(primaryPkgXML("foo", "1", name+"sum", name+".rpm"))}}, nil
+	}
+	virt := rpmVirt("a", "b")
+	virt.ParallelFetch = true
+	repo, err := e.mergeRPM(context.Background(), virt)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !sawParallel.Load() {
+		t.Fatal("parallel_fetch not passed to member fetch")
+	}
+	primary := regexp.MustCompile(`repodata/[0-9a-f]+-primary\.xml\.gz`).FindString(string(repo.Repomd))
+	body := string(gunzip(t, repo.Files[primary]))
+	if a, b := strings.Index(body, "a.rpm"), strings.Index(body, "b.rpm"); a < 0 || b >= 0 && b < a {
+		t.Fatalf("member priority order lost:\n%s", body)
+	}
+}