feat: background syncer for github_rpm remotes
Lazy per-replica scans re-derived RPM metadata on the client request path and, run independently on every replica, multiplied GitHub queries by the replica count. A single background syncer with a shared rate limit, ETag conditional checks, and a DB lease keeps metadata fresh off the request path while bounding GitHub load to ~once per mutable_ttl across the fleet. - Add a single per-process syncer (started at boot, stopped on shutdown) that owns a deduped/coalescing work queue, a worker pool, and one global token-bucket rate limiter bound onto the github provider so every GitHub call (releases list + each ranged asset GET) acquires a token first. - Check each github_rpm remote for new/changed releases on its mutable_ttl cadence; derive only new/changed assets incrementally and prune assets that disappear upstream, so repodata is served from primed DB rows. - Prime metadata in the background on remote creation; the create call never blocks on a derive. - Send the stored releases-list ETag as If-None-Match; a 304 derives nothing (and does not count against GitHub's rate limit), making an unchanged repo nearly free. - Coordinate replicas through a github_rpm_sync_state row (last_synced_at, etag, sync_lease_owner, sync_lease_expires): a periodic scan runs only for the replica that atomically claims the lease, bounding total GitHub load to ~once per mutable_ttl regardless of replica count. - Keep the request path fast: serve current cache, enqueue a prime on an empty cache, and return a bounded wait then a retryable 503 rather than blocking on a cold derive. - Add GITHUB_SYNC_RATE/BURST/WORKERS/POLL_INTERVAL config (conservative defaults) and document the syncer in the README.
This commit is contained in:
@@ -32,6 +32,18 @@ type Config struct {
|
||||
TFSigningKeyPath string
|
||||
TFSigningKeyPassphrase string
|
||||
TFProviderProtocols string
|
||||
|
||||
// github_rpm background syncer. The syncer keeps derived RPM metadata for
|
||||
// every github_rpm remote fresh off the client request path, sharing a
|
||||
// single global token-bucket limiter across all remotes so GitHub is never
|
||||
// hammered. Defaults are conservative: 1 req/s (3600/hr) sits well under an
|
||||
// authenticated token's 5000/hr. Unauthenticated remotes (60/hr) lean on
|
||||
// ETag/304 — an unchanged repo costs nothing — so keep those repos small or
|
||||
// configure a token.
|
||||
GitHubSyncRatePerSec float64
|
||||
GitHubSyncBurst int
|
||||
GitHubSyncWorkers int
|
||||
GitHubSyncPollInterval int
|
||||
}
|
||||
|
||||
func (c *Config) DatabaseDSN() string {
|
||||
@@ -49,6 +61,23 @@ func Load() (*Config, error) {
|
||||
|
||||
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
|
||||
|
||||
syncRate, err := strconv.ParseFloat(getenv("GITHUB_SYNC_RATE", "1"), 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_RATE: %w", err)
|
||||
}
|
||||
syncBurst, err := strconv.Atoi(getenv("GITHUB_SYNC_BURST", "5"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_BURST: %w", err)
|
||||
}
|
||||
syncWorkers, err := strconv.Atoi(getenv("GITHUB_SYNC_WORKERS", "3"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_WORKERS: %w", err)
|
||||
}
|
||||
syncPoll, err := strconv.Atoi(getenv("GITHUB_SYNC_POLL_INTERVAL", "60"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
|
||||
}
|
||||
|
||||
cfg := &Config{
|
||||
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
||||
|
||||
@@ -71,6 +100,11 @@ func Load() (*Config, error) {
|
||||
TFSigningKeyPath: getenv("TF_SIGNING_KEY_PATH", ""),
|
||||
TFSigningKeyPassphrase: getenv("TF_SIGNING_KEY_PASSPHRASE", ""),
|
||||
TFProviderProtocols: getenv("TF_PROVIDER_PROTOCOLS", "5.0,6.0"),
|
||||
|
||||
GitHubSyncRatePerSec: syncRate,
|
||||
GitHubSyncBurst: syncBurst,
|
||||
GitHubSyncWorkers: syncWorkers,
|
||||
GitHubSyncPollInterval: syncPoll,
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
|
||||
Reference in New Issue
Block a user