unkin/artifactapi
3
0
Fork 0
Code Issues 5 Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

Add parallel_fetch toggle for virtual member index files (#165)

Browse Source 
Virtual members already merge concurrently, but each rpm member fetches primary/filelists/other one after another, and each deb member its Packages files the same way. On large virtuals the slowest member's serial chain pushes the merge past dnf's 30s timeout.

- add `parallel_fetch` (default false) to virtuals: model, v2 API, migration 0007
- fetch a member's index files concurrently (up to 4) when set; first error still fails the member
- thread the flag through rpm and deb member fetches

Reviewed-on: #165
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #165.
This commit is contained in:
Unkin Agent
2026-10-25 15:13:57 +11:00
committed by BenVincent
parent 9294686ea6
commit cfb1f93d48
7 changed files with 395 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/database/virtuals.go
+9 -9
View File
@@ -8,12 +8,12 @@ import (
func (db *DB) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
row := db.Pool.QueryRow(ctx, `
SELECT name, package_type, description, members, managed_by, created_at, updated_at
SELECT name, package_type, description, members, managed_by, parallel_fetch, created_at, updated_at
FROM virtuals WHERE name = $1
`, name)
var v models.Virtual
err := row.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt)
err := row.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.ParallelFetch, &v.CreatedAt, &v.UpdatedAt)
if err != nil {
return nil, err
}
@@ -22,7 +22,7 @@ func (db *DB) GetVirtual(ctx context.Context, name string) (*models.Virtual, err
func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
rows, err := db.Pool.Query(ctx, `
SELECT name, package_type, description, members, managed_by, created_at, updated_at
SELECT name, package_type, description, members, managed_by, parallel_fetch, created_at, updated_at
FROM virtuals ORDER BY name
`)
if err != nil {
@@ -33,7 +33,7 @@ func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
var virtuals []models.Virtual
for rows.Next() {
var v models.Virtual
if err := rows.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt); err != nil {
if err := rows.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.ParallelFetch, &v.CreatedAt, &v.UpdatedAt); err != nil {
return nil, err
}
virtuals = append(virtuals, v)
@@ -43,18 +43,18 @@ func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
func (db *DB) CreateVirtual(ctx context.Context, v *models.Virtual) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO virtuals (name, package_type, description, members, managed_by)
VALUES ($1, $2, $3, $4, $5)
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
INSERT INTO virtuals (name, package_type, description, members, managed_by, parallel_fetch)
VALUES ($1, $2, $3, $4, $5, $6)
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy, v.ParallelFetch)
return err
}
func (db *DB) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
_, err := db.Pool.Exec(ctx, `
UPDATE virtuals SET
package_type=$2, description=$3, members=$4, managed_by=$5, updated_at=NOW()
package_type=$2, description=$3, members=$4, managed_by=$5, parallel_fetch=$6, updated_at=NOW()
WHERE name=$1
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy, v.ParallelFetch)
return err
}