feat: background syncer for github_rpm remotes

Lazy per-replica scans re-derived RPM metadata on the client request path
and, run independently on every replica, multiplied GitHub queries by the
replica count. A single background syncer with a shared rate limit, ETag
conditional checks, and a DB lease keeps metadata fresh off the request path
while bounding GitHub load to ~once per mutable_ttl across the fleet.

- Add a single per-process syncer (started at boot, stopped on shutdown) that
  owns a deduped/coalescing work queue, a worker pool, and one global
  token-bucket rate limiter bound onto the github provider so every GitHub call
  (releases list + each ranged asset GET) acquires a token first.
- Check each github_rpm remote for new/changed releases on its mutable_ttl
  cadence; derive only new/changed assets incrementally and prune assets that
  disappear upstream, so repodata is served from primed DB rows.
- Prime metadata in the background on remote creation; the create call never
  blocks on a derive.
- Send the stored releases-list ETag as If-None-Match; a 304 derives nothing
  (and does not count against GitHub's rate limit), making an unchanged repo
  nearly free.
- Coordinate replicas through a github_rpm_sync_state row (last_synced_at,
  etag, sync_lease_owner, sync_lease_expires): a periodic scan runs only for
  the replica that atomically claims the lease, bounding total GitHub load to
  ~once per mutable_ttl regardless of replica count.
- Keep the request path fast: serve current cache, enqueue a prime on an empty
  cache, and return a bounded wait then a retryable 503 rather than blocking on
  a cold derive.
- Add GITHUB_SYNC_RATE/BURST/WORKERS/POLL_INTERVAL config (conservative
  defaults) and document the syncer in the README.

internal/database/github_sync.go

@@ -0,0 +1,73 @@
+package database
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/jackc/pgx/v5"
+
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+// ListGitHubRPMRemotes returns every github_rpm remote so the syncer can sweep
+// them on each poll tick.
+func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error) {
+	rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubRPM)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var remotes []models.Remote
+	for rows.Next() {
+		var r models.Remote
+		if err := scanRemote(rows, &r); err != nil {
+			return nil, err
+		}
+		remotes = append(remotes, r)
+	}
+	return remotes, rows.Err()
+}
+
+// ClaimGitHubSyncLease atomically claims the per-remote sync lease. It succeeds
+// (claimed=true) only when the remote is due — never synced, or synced longer
+// than freshness ago — and no live lease is held by another replica. This bounds
+// total GitHub load to roughly one scan per freshness window regardless of how
+// many replicas poll. The returned etag is the stored releases-list ETag, shared
+// across replicas so a conditional request can short-circuit an unchanged repo.
+// A zero freshness (used for prime scans) ignores the recency gate and claims
+// whenever no live lease is held.
+func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
+	row := db.Pool.QueryRow(ctx, `
+		INSERT INTO github_rpm_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
+		VALUES ($1, $2, now() + make_interval(secs => $4))
+		ON CONFLICT (remote_name) DO UPDATE
+			SET sync_lease_owner   = $2,
+			    sync_lease_expires = now() + make_interval(secs => $4)
+			WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
+			  AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
+		RETURNING s.etag
+	`, remoteName, owner, freshness.Seconds(), lease.Seconds())
+
+	var etag string
+	if err := row.Scan(&etag); err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return false, "", nil
+		}
+		return false, "", err
+	}
+	return true, etag, nil
+}
+
+// ReleaseGitHubSyncLease records the completed scan and frees the lease. Only the
+// owning replica may release; last_synced_at advances so the next poll waits a
+// full freshness window, and etag is persisted for the next conditional request.
+func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
+	_, err := db.Pool.Exec(ctx, `
+		UPDATE github_rpm_sync_state
+		   SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
+		 WHERE remote_name = $1 AND sync_lease_owner = $2
+	`, remoteName, owner, syncedAt, etag)
+	return err
+}

internal/database/github_sync_test.go

@@ -0,0 +1,95 @@
+package database
+
+import (
+	"testing"
+	"time"
+
+	"git.unkin.net/unkin/artifactapi/pkg/models"
+)
+
+func seedGitHubRPMRemote(t *testing.T, name string) {
+	t.Helper()
+	if err := testDB.CreateRemote(ctx(), &models.Remote{
+		Name: name, PackageType: models.PackageGitHubRPM, RepoType: models.RepoTypeRemote,
+		BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
+	}); err != nil {
+		t.Fatalf("seed github_rpm remote: %v", err)
+	}
+}
+
+// TestGitHubSyncLease exercises the real SQL: exactly one replica may hold the
+// lease, the recency window blocks a too-soon periodic re-claim, and a prime
+// (freshness 0) bypasses recency but still respects a live lease.
+func TestGitHubSyncLease(t *testing.T) {
+	requireDB(t)
+	name := "gh-lease-" + time.Now().Format("150405.000000")
+	seedGitHubRPMRemote(t, name)
+
+	const lease = 15 * time.Minute
+	freshness := time.Hour
+
+	// First claim on a never-synced remote wins; etag starts empty.
+	claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease)
+	if err != nil || !claimed {
+		t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
+	}
+	if etag != "" {
+		t.Fatalf("initial etag should be empty, got %q", etag)
+	}
+
+	// A second replica cannot claim while the lease is held.
+	claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
+	if err != nil {
+		t.Fatalf("replica-2 claim err: %v", err)
+	}
+	if claimed2 {
+		t.Fatal("replica-2 claimed while replica-1 holds the lease")
+	}
+
+	// Replica 1 finishes: record the sync and persist an etag.
+	if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
+		t.Fatalf("release: %v", err)
+	}
+
+	// A periodic re-claim inside the freshness window is blocked by recency.
+	claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
+	if err != nil {
+		t.Fatalf("replica-2 recency claim err: %v", err)
+	}
+	if claimed3 {
+		t.Fatal("periodic claim succeeded inside the freshness window")
+	}
+
+	// A prime (freshness 0) bypasses recency and reads the persisted etag.
+	claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease)
+	if err != nil || !claimed4 {
+		t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
+	}
+	if etag4 != `"etag-1"` {
+		t.Fatalf("prime claim etag = %q, want persisted \"etag-1\"", etag4)
+	}
+}
+
+func TestListGitHubRPMRemotes(t *testing.T) {
+	requireDB(t)
+	name := "gh-list-" + time.Now().Format("150405.000000")
+	seedGitHubRPMRemote(t, name)
+	seedRemote(t, "generic-"+time.Now().Format("150405.000000"))
+
+	remotes, err := testDB.ListGitHubRPMRemotes(ctx())
+	if err != nil {
+		t.Fatalf("list: %v", err)
+	}
+	found := false
+	for _, r := range remotes {
+		if r.PackageType != models.PackageGitHubRPM {
+			t.Fatalf("non-github_rpm remote returned: %s (%s)", r.Name, r.PackageType)
+		}
+		if r.Name == name {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("seeded remote %q not returned", name)
+	}
+}

internal/database/postgres.go

@@ -164,6 +164,14 @@ func (db *DB) migrate() error {
 		ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
 		ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
 
+		CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
+			remote_name        TEXT PRIMARY KEY,
+			etag               TEXT DEFAULT '',
+			last_synced_at     TIMESTAMPTZ,
+			sync_lease_owner   TEXT DEFAULT '',
+			sync_lease_expires TIMESTAMPTZ
+		);
+
 		CREATE TABLE IF NOT EXISTS signing_keys (
 			purpose             TEXT PRIMARY KEY,
 			private_key_armor   TEXT NOT NULL,