unkinben
6dc72920da
Lazy per-replica scans re-derived RPM metadata on the client request path and, run independently on every replica, multiplied GitHub queries by the replica count. A single background syncer with a shared rate limit, ETag conditional checks, and a DB lease keeps metadata fresh off the request path while bounding GitHub load to ~once per mutable_ttl across the fleet. - Add a single per-process syncer (started at boot, stopped on shutdown) that owns a deduped/coalescing work queue, a worker pool, and one global token-bucket rate limiter bound onto the github provider so every GitHub call (releases list + each ranged asset GET) acquires a token first. - Check each github_rpm remote for new/changed releases on its mutable_ttl cadence; derive only new/changed assets incrementally and prune assets that disappear upstream, so repodata is served from primed DB rows. - Prime metadata in the background on remote creation; the create call never blocks on a derive. - Send the stored releases-list ETag as If-None-Match; a 304 derives nothing (and does not count against GitHub's rate limit), making an unchanged repo nearly free. - Coordinate replicas through a github_rpm_sync_state row (last_synced_at, etag, sync_lease_owner, sync_lease_expires): a periodic scan runs only for the replica that atomically claims the lease, bounding total GitHub load to ~once per mutable_ttl regardless of replica count. - Keep the request path fast: serve current cache, enqueue a prime on an empty cache, and return a bounded wait then a retryable 503 rather than blocking on a cold derive. - Add GITHUB_SYNC_RATE/BURST/WORKERS/POLL_INTERVAL config (conservative defaults) and document the syncer in the README.
119 lines
3.3 KiB
Go
119 lines
3.3 KiB
Go
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
)
|
|
|
|
type Config struct {
|
|
ListenAddr string
|
|
|
|
DBHost string
|
|
DBPort int
|
|
DBUser string
|
|
DBPass string
|
|
DBName string
|
|
DBSSL string
|
|
|
|
RedisURL string
|
|
|
|
S3Endpoint string
|
|
S3AccessKey string
|
|
S3SecretKey string
|
|
S3Bucket string
|
|
S3Secure bool
|
|
S3Region string
|
|
|
|
// Terraform provider registry signing. When TFSigningKeyPath points at a
|
|
// readable armored GPG private key, artifactapi serves local terraform
|
|
// repos as a real provider registry (service discovery + signed
|
|
// SHA256SUMS). Left empty, the registry endpoints stay disabled.
|
|
TFSigningKeyPath string
|
|
TFSigningKeyPassphrase string
|
|
TFProviderProtocols string
|
|
|
|
// github_rpm background syncer. The syncer keeps derived RPM metadata for
|
|
// every github_rpm remote fresh off the client request path, sharing a
|
|
// single global token-bucket limiter across all remotes so GitHub is never
|
|
// hammered. Defaults are conservative: 1 req/s (3600/hr) sits well under an
|
|
// authenticated token's 5000/hr. Unauthenticated remotes (60/hr) lean on
|
|
// ETag/304 — an unchanged repo costs nothing — so keep those repos small or
|
|
// configure a token.
|
|
GitHubSyncRatePerSec float64
|
|
GitHubSyncBurst int
|
|
GitHubSyncWorkers int
|
|
GitHubSyncPollInterval int
|
|
}
|
|
|
|
func (c *Config) DatabaseDSN() string {
|
|
return fmt.Sprintf(
|
|
"postgres://%s:%s@%s:%d/%s?sslmode=%s",
|
|
c.DBUser, c.DBPass, c.DBHost, c.DBPort, c.DBName, c.DBSSL,
|
|
)
|
|
}
|
|
|
|
func Load() (*Config, error) {
|
|
dbPort, err := strconv.Atoi(getenv("DBPORT", "5432"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid DBPORT: %w", err)
|
|
}
|
|
|
|
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
|
|
|
|
syncRate, err := strconv.ParseFloat(getenv("GITHUB_SYNC_RATE", "1"), 64)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid GITHUB_SYNC_RATE: %w", err)
|
|
}
|
|
syncBurst, err := strconv.Atoi(getenv("GITHUB_SYNC_BURST", "5"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid GITHUB_SYNC_BURST: %w", err)
|
|
}
|
|
syncWorkers, err := strconv.Atoi(getenv("GITHUB_SYNC_WORKERS", "3"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid GITHUB_SYNC_WORKERS: %w", err)
|
|
}
|
|
syncPoll, err := strconv.Atoi(getenv("GITHUB_SYNC_POLL_INTERVAL", "60"))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
|
|
}
|
|
|
|
cfg := &Config{
|
|
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
|
|
|
|
DBHost: getenv("DBHOST", "localhost"),
|
|
DBPort: dbPort,
|
|
DBUser: getenv("DBUSER", "artifacts"),
|
|
DBPass: getenv("DBPASS", ""),
|
|
DBName: getenv("DBNAME", "artifacts"),
|
|
DBSSL: getenv("DBSSL", "disable"),
|
|
|
|
RedisURL: getenv("REDIS_URL", "redis://localhost:6379"),
|
|
|
|
S3Endpoint: getenv("MINIO_ENDPOINT", "localhost:9000"),
|
|
S3AccessKey: getenv("MINIO_ACCESS_KEY", ""),
|
|
S3SecretKey: getenv("MINIO_SECRET_KEY", ""),
|
|
S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
|
|
S3Secure: s3Secure,
|
|
S3Region: getenv("MINIO_REGION", ""),
|
|
|
|
TFSigningKeyPath: getenv("TF_SIGNING_KEY_PATH", ""),
|
|
TFSigningKeyPassphrase: getenv("TF_SIGNING_KEY_PASSPHRASE", ""),
|
|
TFProviderProtocols: getenv("TF_PROVIDER_PROTOCOLS", "5.0,6.0"),
|
|
|
|
GitHubSyncRatePerSec: syncRate,
|
|
GitHubSyncBurst: syncBurst,
|
|
GitHubSyncWorkers: syncWorkers,
|
|
GitHubSyncPollInterval: syncPoll,
|
|
}
|
|
|
|
return cfg, nil
|
|
}
|
|
|
|
func getenv(key, fallback string) string {
|
|
if v, ok := os.LookupEnv(key); ok {
|
|
return v
|
|
}
|
|
return fallback
|
|
}
|