unkinben
6dc72920da
Lazy per-replica scans re-derived RPM metadata on the client request path and, run independently on every replica, multiplied GitHub queries by the replica count. A single background syncer with a shared rate limit, ETag conditional checks, and a DB lease keeps metadata fresh off the request path while bounding GitHub load to ~once per mutable_ttl across the fleet. - Add a single per-process syncer (started at boot, stopped on shutdown) that owns a deduped/coalescing work queue, a worker pool, and one global token-bucket rate limiter bound onto the github provider so every GitHub call (releases list + each ranged asset GET) acquires a token first. - Check each github_rpm remote for new/changed releases on its mutable_ttl cadence; derive only new/changed assets incrementally and prune assets that disappear upstream, so repodata is served from primed DB rows. - Prime metadata in the background on remote creation; the create call never blocks on a derive. - Send the stored releases-list ETag as If-None-Match; a 304 derives nothing (and does not count against GitHub's rate limit), making an unchanged repo nearly free. - Coordinate replicas through a github_rpm_sync_state row (last_synced_at, etag, sync_lease_owner, sync_lease_expires): a periodic scan runs only for the replica that atomically claims the lease, bounding total GitHub load to ~once per mutable_ttl regardless of replica count. - Keep the request path fast: serve current cache, enqueue a prime on an empty cache, and return a bounded wait then a retryable 503 rather than blocking on a cold derive. - Add GITHUB_SYNC_RATE/BURST/WORKERS/POLL_INTERVAL config (conservative defaults) and document the syncer in the README.
74 lines
2.8 KiB
Go
74 lines
2.8 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
// ListGitHubRPMRemotes returns every github_rpm remote so the syncer can sweep
|
|
// them on each poll tick.
|
|
func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error) {
|
|
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubRPM)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var remotes []models.Remote
|
|
for rows.Next() {
|
|
var r models.Remote
|
|
if err := scanRemote(rows, &r); err != nil {
|
|
return nil, err
|
|
}
|
|
remotes = append(remotes, r)
|
|
}
|
|
return remotes, rows.Err()
|
|
}
|
|
|
|
// ClaimGitHubSyncLease atomically claims the per-remote sync lease. It succeeds
|
|
// (claimed=true) only when the remote is due — never synced, or synced longer
|
|
// than freshness ago — and no live lease is held by another replica. This bounds
|
|
// total GitHub load to roughly one scan per freshness window regardless of how
|
|
// many replicas poll. The returned etag is the stored releases-list ETag, shared
|
|
// across replicas so a conditional request can short-circuit an unchanged repo.
|
|
// A zero freshness (used for prime scans) ignores the recency gate and claims
|
|
// whenever no live lease is held.
|
|
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
|
row := db.Pool.QueryRow(ctx, `
|
|
INSERT INTO github_rpm_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
|
VALUES ($1, $2, now() + make_interval(secs => $4))
|
|
ON CONFLICT (remote_name) DO UPDATE
|
|
SET sync_lease_owner = $2,
|
|
sync_lease_expires = now() + make_interval(secs => $4)
|
|
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
|
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
|
RETURNING s.etag
|
|
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
|
|
|
var etag string
|
|
if err := row.Scan(&etag); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return false, "", nil
|
|
}
|
|
return false, "", err
|
|
}
|
|
return true, etag, nil
|
|
}
|
|
|
|
// ReleaseGitHubSyncLease records the completed scan and frees the lease. Only the
|
|
// owning replica may release; last_synced_at advances so the next poll waits a
|
|
// full freshness window, and etag is persisted for the next conditional request.
|
|
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
|
_, err := db.Pool.Exec(ctx, `
|
|
UPDATE github_rpm_sync_state
|
|
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
|
WHERE remote_name = $1 AND sync_lease_owner = $2
|
|
`, remoteName, owner, syncedAt, etag)
|
|
return err
|
|
}
|