unkinben
e24c35f534
## Why Builds on #107 (merged), which derives `github_rpm` RPM metadata lazily on the client request path, single-flighted per replica. Two problems remain: the derive still happens per replica, so across a multi-replica deployment the same releases are scanned and re-derived N times, multiplying GitHub queries; and a cold cache blocks the first request on a full derive. GitHub's rate limits are low (~60/hr unauthenticated, ~5000/hr authenticated), so this needs a single coordinated syncer with a shared rate limit and conditional requests. ## How - Add a single per-process background syncer (started at boot, cleanly stopped on shutdown) that owns a deduped/coalescing work queue, a worker pool, and one global token-bucket rate limiter (`golang.org/x/time/rate`) bound onto the github provider so every GitHub call (releases list + each ranged asset GET) acquires a token first. - Re-check each `github_rpm` remote for new/changed releases on its existing `mutable_ttl` cadence; derive only new/changed assets incrementally and prune assets that disappear upstream. Repodata is served from primed DB rows. - Prime metadata in the background on remote creation; the create call returns immediately. - Send the stored releases-list `ETag` as `If-None-Match`; a `304` derives nothing and is not counted against GitHub's rate limit, so an unchanged repo is nearly free. - Coordinate replicas through a `github_rpm_sync_state` row (`last_synced_at`, `etag`, `sync_lease_owner`, `sync_lease_expires`): a periodic scan runs only for the replica that atomically claims the lease, bounding total GitHub load to ~once per `mutable_ttl` regardless of replica count; the ETag is shared through the same row. - Keep the request path fast: serve current cache, enqueue a prime on an empty cache, and return a bounded wait then a retryable `503` rather than blocking on a cold derive. - Add `GITHUB_SYNC_RATE` / `GITHUB_SYNC_BURST` / `GITHUB_SYNC_WORKERS` / `GITHUB_SYNC_POLL_INTERVAL` config with conservative defaults (1 req/s, burst 5, 3 workers, 60s tick) and document the syncer in the README. ## Tests - Unit (httptest, Range/ETag-aware fixture): `304` releases response derives nothing; incremental derive fetches only the newly added asset; the shared limiter caps request rate; work-queue enqueues coalesce to one job; prime enqueues a job; a held lease stops a second replica from scanning; cold-start serves `503` while warm cache serves `200`. - DB integration (testcontainers postgres): the real lease SQL — one holder at a time, recency gate blocks a too-soon periodic re-claim, prime (freshness 0) bypasses recency but respects a live lease. - Docker e2e re-run: `dnf install dotvault` works; prime-on-create derives in the background at ~1 req/s (global limiter); `dnf makecache` served fast from the priming cache (no cold block); clean shutdown mid-scan, no panics. ## Notes - Reuses `mutable_ttl` as the check interval (no new per-remote field), per brief. Reviewed-on: #108 Co-authored-by: Ben Vincent <ben@unkin.net> Co-committed-by: Ben Vincent <ben@unkin.net>
96 lines
2.9 KiB
Go
96 lines
2.9 KiB
Go
package database
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
func seedGitHubRPMRemote(t *testing.T, name string) {
|
|
t.Helper()
|
|
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
|
Name: name, PackageType: models.PackageGitHubRPM, RepoType: models.RepoTypeRemote,
|
|
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
|
|
}); err != nil {
|
|
t.Fatalf("seed github_rpm remote: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestGitHubSyncLease exercises the real SQL: exactly one replica may hold the
|
|
// lease, the recency window blocks a too-soon periodic re-claim, and a prime
|
|
// (freshness 0) bypasses recency but still respects a live lease.
|
|
func TestGitHubSyncLease(t *testing.T) {
|
|
requireDB(t)
|
|
name := "gh-lease-" + time.Now().Format("150405.000000")
|
|
seedGitHubRPMRemote(t, name)
|
|
|
|
const lease = 15 * time.Minute
|
|
freshness := time.Hour
|
|
|
|
// First claim on a never-synced remote wins; etag starts empty.
|
|
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease)
|
|
if err != nil || !claimed {
|
|
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
|
|
}
|
|
if etag != "" {
|
|
t.Fatalf("initial etag should be empty, got %q", etag)
|
|
}
|
|
|
|
// A second replica cannot claim while the lease is held.
|
|
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
|
|
if err != nil {
|
|
t.Fatalf("replica-2 claim err: %v", err)
|
|
}
|
|
if claimed2 {
|
|
t.Fatal("replica-2 claimed while replica-1 holds the lease")
|
|
}
|
|
|
|
// Replica 1 finishes: record the sync and persist an etag.
|
|
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
|
t.Fatalf("release: %v", err)
|
|
}
|
|
|
|
// A periodic re-claim inside the freshness window is blocked by recency.
|
|
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
|
|
if err != nil {
|
|
t.Fatalf("replica-2 recency claim err: %v", err)
|
|
}
|
|
if claimed3 {
|
|
t.Fatal("periodic claim succeeded inside the freshness window")
|
|
}
|
|
|
|
// A prime (freshness 0) bypasses recency and reads the persisted etag.
|
|
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease)
|
|
if err != nil || !claimed4 {
|
|
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
|
|
}
|
|
if etag4 != `"etag-1"` {
|
|
t.Fatalf("prime claim etag = %q, want persisted \"etag-1\"", etag4)
|
|
}
|
|
}
|
|
|
|
func TestListGitHubRPMRemotes(t *testing.T) {
|
|
requireDB(t)
|
|
name := "gh-list-" + time.Now().Format("150405.000000")
|
|
seedGitHubRPMRemote(t, name)
|
|
seedRemote(t, "generic-"+time.Now().Format("150405.000000"))
|
|
|
|
remotes, err := testDB.ListGitHubRPMRemotes(ctx())
|
|
if err != nil {
|
|
t.Fatalf("list: %v", err)
|
|
}
|
|
found := false
|
|
for _, r := range remotes {
|
|
if r.PackageType != models.PackageGitHubRPM {
|
|
t.Fatalf("non-github_rpm remote returned: %s (%s)", r.Name, r.PackageType)
|
|
}
|
|
if r.Name == name {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("seeded remote %q not returned", name)
|
|
}
|
|
}
|