feat: background syncer for github_rpm remotes
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful

Lazy per-replica scans re-derived RPM metadata on the client request path
and, run independently on every replica, multiplied GitHub queries by the
replica count. A single background syncer with a shared rate limit, ETag
conditional checks, and a DB lease keeps metadata fresh off the request path
while bounding GitHub load to ~once per mutable_ttl across the fleet.

- Add a single per-process syncer (started at boot, stopped on shutdown) that
  owns a deduped/coalescing work queue, a worker pool, and one global
  token-bucket rate limiter bound onto the github provider so every GitHub call
  (releases list + each ranged asset GET) acquires a token first.
- Check each github_rpm remote for new/changed releases on its mutable_ttl
  cadence; derive only new/changed assets incrementally and prune assets that
  disappear upstream, so repodata is served from primed DB rows.
- Prime metadata in the background on remote creation; the create call never
  blocks on a derive.
- Send the stored releases-list ETag as If-None-Match; a 304 derives nothing
  (and does not count against GitHub's rate limit), making an unchanged repo
  nearly free.
- Coordinate replicas through a github_rpm_sync_state row (last_synced_at,
  etag, sync_lease_owner, sync_lease_expires): a periodic scan runs only for
  the replica that atomically claims the lease, bounding total GitHub load to
  ~once per mutable_ttl regardless of replica count.
- Keep the request path fast: serve current cache, enqueue a prime on an empty
  cache, and return a bounded wait then a retryable 503 rather than blocking on
  a cold derive.
- Add GITHUB_SYNC_RATE/BURST/WORKERS/POLL_INTERVAL config (conservative
  defaults) and document the syncer in the README.
This commit is contained in:
2026-08-10 21:08:15 +10:00
parent d154fbf3f3
commit 6dc72920da
14 changed files with 1015 additions and 28 deletions
+12 -2
View File
@@ -27,7 +27,7 @@ import (
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
@@ -47,6 +47,7 @@ type Server struct {
localHandler *v2.LocalHandler
tfRegistry *tfregistry.Handler
gc *gc.Collector
syncer *rpm.Syncer
}
func New(cfg *config.Config, version string) (*Server, error) {
@@ -69,6 +70,12 @@ func New(cfg *config.Config, version string) (*Server, error) {
localHandler := v2.NewLocalHandler(db, s3)
virtEngine := virtual.NewEngine(db, engine)
collector := gc.New(db, s3, 1*time.Hour)
syncer := rpm.NewSyncer(db, rpm.SyncConfig{
RatePerSec: cfg.GitHubSyncRatePerSec,
Burst: cfg.GitHubSyncBurst,
Workers: cfg.GitHubSyncWorkers,
PollInterval: time.Duration(cfg.GitHubSyncPollInterval) * time.Second,
})
// The terraform registry signs with a GPG key. A configured file wins (BYO
// key); otherwise artifactapi generates one on first start and persists it in
@@ -100,6 +107,7 @@ func New(cfg *config.Config, version string) (*Server, error) {
localHandler: localHandler,
tfRegistry: tfRegistry,
gc: collector,
syncer: syncer,
}
s.router = s.routes()
@@ -129,7 +137,7 @@ func (s *Server) routes() chi.Router {
r.Mount("/api/v1", proxyHandler.Routes())
r.Mount("/v2", proxyHandler.DockerV2Routes())
remotesHandler := v2.NewRemotesHandler(s.db)
remotesHandler := v2.NewRemotesHandler(s.db, s.syncer)
virtualsHandler := v2.NewVirtualsHandler(s.db)
healthHandler := v2.NewHealthHandler(s.db, s.cache, s.store)
statsHandler := v2.NewStatsHandler(s.db)
@@ -196,6 +204,7 @@ func (s *Server) newHTTPServer() *http.Server {
func (s *Server) Run(ctx context.Context) error {
go s.gc.Run(ctx)
go s.syncer.Run(ctx)
httpServer := s.newHTTPServer()
@@ -216,6 +225,7 @@ func (s *Server) Run(ctx context.Context) error {
func (s *Server) RunOnListener(ctx context.Context, ln net.Listener) error {
go s.gc.Run(ctx)
go s.syncer.Run(ctx)
httpServer := s.newHTTPServer()