Add parallel_fetch toggle for virtual member index files

internal/virtual/engine.go

@@ -19,6 +19,7 @@ import (
 	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/internal/proxy"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
+	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/singleflight"
 )
 
@@ -26,8 +27,8 @@ type Engine struct {
 	db          *database.DB
 	proxyEngine *proxy.Engine
 	getRemote   func(context.Context, string) (*models.Remote, error)
-	rpmMember   func(context.Context, string) (*RPMMember, error)
-	debMember   func(ctx context.Context, name, base string) (*DebMember, error)
+	rpmMember   func(ctx context.Context, name string, parallel bool) (*RPMMember, error)
+	debMember   func(ctx context.Context, name, base string, parallel bool) (*DebMember, error)
 	apkMember   func(ctx context.Context, name, dir string) (*AlpineMember, error)
 	npmMember   func(ctx context.Context, name, pkg string) (*NPMMember, error)
 
@@ -353,7 +354,7 @@ func (e *Engine) merged(ctx context.Context, key string, merge func(context.Cont
 }
 
 func (e *Engine) mergeRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, error) {
-	members, err := fetchMembers(virt, func(name string) (*RPMMember, error) { return e.rpmMember(ctx, name) })
+	members, err := fetchMembers(virt, func(name string) (*RPMMember, error) { return e.rpmMember(ctx, name, virt.ParallelFetch) })
 	if err != nil {
 		return nil, err
 	}
@@ -402,7 +403,7 @@ func (e *Engine) fetchDeb(ctx context.Context, virt models.Virtual, reqPath stri
 	}
 
 	repo, err := e.merged(ctx, key, func(ctx context.Context) (mergedRepo, error) {
-		members, err := fetchMembers(virt, func(name string) (*DebMember, error) { return e.debMember(ctx, name, base) })
+		members, err := fetchMembers(virt, func(name string) (*DebMember, error) { return e.debMember(ctx, name, base, virt.ParallelFetch) })
 		if err != nil {
 			return nil, err
 		}
@@ -430,7 +431,7 @@ func debContentType(name string) string {
 
 // fetchDebMember reads a member's Release under base and each binary Packages
 // index it lists.
-func (e *Engine) fetchDebMember(ctx context.Context, name, base string) (*DebMember, error) {
+func (e *Engine) fetchDebMember(ctx context.Context, name, base string, parallel bool) (*DebMember, error) {
 	remote, err := e.getRemote(ctx, name)
 	if err != nil {
 		return nil, fmt.Errorf("remote %q: %w", name, err)
@@ -439,17 +440,13 @@ func (e *Engine) fetchDebMember(ctx context.Context, name, base string) (*DebMem
 	if err != nil {
 		return nil, err
 	}
-	m := &DebMember{RemoteName: name, Release: rel, Packages: map[string][]byte{}}
-	for dir, file := range debIndexFiles(parseDebRelease(rel)) {
-		raw, err := e.fetchMemberPath(ctx, *remote, base+file)
-		if err != nil {
-			return nil, err
-		}
-		if m.Packages[dir], err = decompress(file, raw); err != nil {
-			return nil, fmt.Errorf("decompress %s: %w", file, err)
-		}
+	pkgs, err := fetchFiles(parallel, debIndexFiles(parseDebRelease(rel)), func(file string) ([]byte, error) {
+		return e.fetchDecompressed(ctx, *remote, base, file)
+	})
+	if err != nil {
+		return nil, err
 	}
-	return m, nil
+	return &DebMember{RemoteName: name, Release: rel, Packages: pkgs}, nil
 }
 
 func (e *Engine) mergedAlpine(ctx context.Context, virt models.Virtual, dir string) (mergedRepo, error) {
@@ -559,7 +556,7 @@ func (e *Engine) cachedFile(key, path string) ([]byte, bool) {
 	return nil, false
 }
 
-func (e *Engine) fetchRPMMember(ctx context.Context, name string) (*RPMMember, error) {
+func (e *Engine) fetchRPMMember(ctx context.Context, name string, parallel bool) (*RPMMember, error) {
 	remote, err := e.getRemote(ctx, name)
 	if err != nil {
 		return nil, fmt.Errorf("remote %q: %w", name, err)
@@ -572,17 +569,63 @@ func (e *Engine) fetchRPMMember(ctx context.Context, name string) (*RPMMember, e
 	if err != nil {
 		return nil, err
 	}
-	m := &RPMMember{RemoteName: name, Bases: remote.UpstreamPool(), Timestamp: ts, Data: map[string][]byte{}}
-	for t, href := range locs {
-		raw, err := e.fetchMemberPath(ctx, *remote, href)
-		if err != nil {
-			return nil, err
-		}
-		if m.Data[t], err = decompress(href, raw); err != nil {
-			return nil, fmt.Errorf("decompress %s: %w", href, err)
-		}
+	data, err := fetchFiles(parallel, locs, func(href string) ([]byte, error) {
+		return e.fetchDecompressed(ctx, *remote, "", href)
+	})
+	if err != nil {
+		return nil, err
 	}
-	return m, nil
+	return &RPMMember{RemoteName: name, Bases: remote.UpstreamPool(), Timestamp: ts, Data: data}, nil
+}
+
+func (e *Engine) fetchDecompressed(ctx context.Context, remote models.Remote, base, file string) ([]byte, error) {
+	raw, err := e.fetchMemberPath(ctx, remote, base+file)
+	if err != nil {
+		return nil, err
+	}
+	body, err := decompress(file, raw)
+	if err != nil {
+		return nil, fmt.Errorf("decompress %s: %w", file, err)
+	}
+	return body, nil
+}
+
+// maxFileFetches bounds the concurrent index file fetches per member.
+const maxFileFetches = 4
+
+// fetchFiles fetches each key's file, one at a time or up to maxFileFetches at
+// once, failing on the first error.
+func fetchFiles(parallel bool, files map[string]string, fetch func(file string) ([]byte, error)) (map[string][]byte, error) {
+	out := make(map[string][]byte, len(files))
+	if !parallel {
+		for k, f := range files {
+			body, err := fetch(f)
+			if err != nil {
+				return nil, err
+			}
+			out[k] = body
+		}
+		return out, nil
+	}
+	var mu sync.Mutex
+	var g errgroup.Group
+	g.SetLimit(maxFileFetches)
+	for k, f := range files {
+		g.Go(func() error {
+			body, err := fetch(f)
+			if err != nil {
+				return err
+			}
+			mu.Lock()
+			out[k] = body
+			mu.Unlock()
+			return nil
+		})
+	}
+	if err := g.Wait(); err != nil {
+		return nil, err
+	}
+	return out, nil
 }
 
 // fetchMemberPath reads one path from a member the same way its own route