fix: decouple github_rpm scan and repodata read from the client request
A cold `dnf makecache` against a github_rpm remote with many release assets 500s on the first request: ServeRemote derives metadata for every asset synchronously on the inbound request context, so once dnf hits its makecache timeout and disconnects the canceled request context both aborts the in-flight derive and poisons the subsequent repodata DB read, which surfaces as HTTP 500. It only "works" on a lucky client retry that finds the partially-populated cache fresh. - detach the release scan to a background, timeout-bounded context so a client cancel can neither abort the shared derive nor cancel the read - single-flight the scan per remote so concurrent requests never launch duplicate derives - serve the current cache immediately when it is non-empty and derive in the background; only a completely empty cache blocks on a bounded first scan - serve repodata on a context detached from the request, and treat a canceled/deadline-exceeded metadata read as a retryable 503 instead of a hard 500
This commit is contained in:
@@ -33,6 +33,12 @@ const (
|
||||
defaultHeaderRangeInitial = 1 << 20 // 1 MiB — covers the header of almost every RPM
|
||||
defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset
|
||||
defaultReleasePageCap = 10 // 100 releases/page * 10 pages
|
||||
|
||||
// defaultScanTimeout bounds a detached background scan (which may do one
|
||||
// ranged fetch per asset across every release) so it can never run forever.
|
||||
defaultScanTimeout = 10 * time.Minute
|
||||
// defaultServeTimeout bounds a repodata DB read served on a detached context.
|
||||
defaultServeTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases
|
||||
@@ -46,9 +52,11 @@ type GitHubProvider struct {
|
||||
headerInitial int64
|
||||
headerMax int64
|
||||
pageCap int
|
||||
scanTimeout time.Duration
|
||||
serveTimeout time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
scanLock map[string]*sync.Mutex
|
||||
scanning map[string]bool
|
||||
lastScan map[string]time.Time
|
||||
}
|
||||
|
||||
@@ -58,7 +66,9 @@ func newGitHubProvider() *GitHubProvider {
|
||||
headerInitial: defaultHeaderRangeInitial,
|
||||
headerMax: defaultHeaderRangeMax,
|
||||
pageCap: defaultReleasePageCap,
|
||||
scanLock: map[string]*sync.Mutex{},
|
||||
scanTimeout: defaultScanTimeout,
|
||||
serveTimeout: defaultServeTimeout,
|
||||
scanning: map[string]bool{},
|
||||
lastScan: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
@@ -104,20 +114,27 @@ func (p *GitHubProvider) AuthHeaders(_ context.Context, remote models.Remote) (h
|
||||
// 302-redirects .rpm downloads to the backend releases_remote. Returns false
|
||||
// only for paths it does not own, letting the normal proxy path take over.
|
||||
func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store provider.RemoteMetadataStore) bool {
|
||||
p.ensureFresh(r.Context(), remote, store)
|
||||
p.refresh(remote, store)
|
||||
|
||||
if strings.HasPrefix(path, "repodata/") {
|
||||
// Serve repodata on a context detached from the inbound request: a
|
||||
// client disconnect (e.g. dnf makecache timing out) must never cancel
|
||||
// the metadata DB read and surface as a 500.
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
sr := r.WithContext(sctx)
|
||||
|
||||
tail := strings.TrimPrefix(path, "repodata/")
|
||||
lp := &Provider{}
|
||||
switch {
|
||||
case tail == "repomd.xml":
|
||||
lp.serveRepomd(w, r, store, remote.Name)
|
||||
lp.serveRepomd(w, sr, store, remote.Name)
|
||||
case strings.HasSuffix(tail, "-primary.xml.gz"):
|
||||
lp.servePrimary(w, r, store, remote.Name)
|
||||
lp.servePrimary(w, sr, store, remote.Name)
|
||||
case strings.HasSuffix(tail, "-filelists.xml.gz"):
|
||||
lp.serveFilelists(w, r, store, remote.Name)
|
||||
lp.serveFilelists(w, sr, store, remote.Name)
|
||||
case strings.HasSuffix(tail, "-other.xml.gz"):
|
||||
lp.serveOther(w, r, store, remote.Name)
|
||||
lp.serveOther(w, sr, store, remote.Name)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
@@ -137,22 +154,52 @@ func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, rem
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) ensureFresh(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
// refresh brings the derived metadata up to date without coupling the scan to
|
||||
// the inbound request. When the cache is stale it single-flights a scan: if the
|
||||
// cache already holds rows the scan runs in the background and the caller serves
|
||||
// the current cache immediately; only a completely empty cache blocks on a
|
||||
// bounded first scan (so the first client sees packages rather than an empty or
|
||||
// 500 repodata).
|
||||
func (p *GitHubProvider) refresh(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
ttl := time.Duration(remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 5 * time.Minute
|
||||
}
|
||||
|
||||
lock := p.lockFor(remote.Name)
|
||||
lock.Lock()
|
||||
defer lock.Unlock()
|
||||
|
||||
p.mu.Lock()
|
||||
last, ok := p.lastScan[remote.Name]
|
||||
p.mu.Unlock()
|
||||
if ok && time.Since(last) < ttl {
|
||||
fresh := ok && time.Since(last) < ttl
|
||||
if fresh || p.scanning[remote.Name] {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.scanning[remote.Name] = true
|
||||
p.mu.Unlock()
|
||||
|
||||
empty := true
|
||||
if rows, err := store.ListRPMMetadataEntries(context.Background(), remote.Name); err == nil {
|
||||
empty = len(rows) == 0
|
||||
}
|
||||
|
||||
if empty {
|
||||
p.runScan(remote, store)
|
||||
return
|
||||
}
|
||||
go p.runScan(remote, store)
|
||||
}
|
||||
|
||||
// runScan derives metadata on a detached, bounded context so a client cancel
|
||||
// can neither abort the shared derive nor poison the metadata read. The caller
|
||||
// must have already claimed the single-flight slot (scanning[name] = true).
|
||||
func (p *GitHubProvider) runScan(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.scanning, remote.Name)
|
||||
p.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), p.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := p.scan(ctx, remote, store); err != nil {
|
||||
// Keep serving whatever metadata is already cached rather than 500ing.
|
||||
@@ -165,17 +212,6 @@ func (p *GitHubProvider) ensureFresh(ctx context.Context, remote models.Remote,
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) lockFor(name string) *sync.Mutex {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
l, ok := p.scanLock[name]
|
||||
if !ok {
|
||||
l = &sync.Mutex{}
|
||||
p.scanLock[name] = l
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error {
|
||||
releases, err := p.fetchReleases(ctx, remote)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user