Add per-remote sync_interval and spread GitHub syncs across it

internal/database/github_sync_test.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -276,3 +277,86 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
 		})
 	}
 }
+
+func TestSyncPhaseStableAndSpread(t *testing.T) {
+	const period = 5 * time.Minute
+	phases := map[time.Duration]bool{}
+	var lo, hi time.Duration = period, 0
+	for i := 0; i < 13; i++ {
+		name := fmt.Sprintf("github-rpm-%02d", i)
+		p := syncPhase(name, period)
+		if p != syncPhase(name, period) {
+			t.Fatalf("%s: phase not stable", name)
+		}
+		if p < 0 || p >= period {
+			t.Fatalf("%s: phase %v outside [0,%v)", name, p, period)
+		}
+		phases[p.Truncate(time.Second)] = true
+		lo, hi = min(lo, p), max(hi, p)
+	}
+	if len(phases) != 13 {
+		t.Fatalf("phases collide: %d distinct of 13", len(phases))
+	}
+	if hi-lo < period/2 {
+		t.Fatalf("phases bunched in %v of %v", hi-lo, period)
+	}
+	if syncPhase("x", 0) != 0 {
+		t.Fatal("zero period must give zero phase")
+	}
+}
+
+// TestGitHubSyncLeasePhasedSlots checks a remote is due once per slot: synced
+// before its latest boundary (or over a TTL ago) it is claimable by exactly one
+// replica; synced after the boundary it is not.
+func TestGitHubSyncLeasePhasedSlots(t *testing.T) {
+	requireDB(t)
+	const lease = 15 * time.Minute
+	ttl := time.Hour
+
+	setSynced := func(name string, at time.Time) {
+		t.Helper()
+		if _, err := testDB.Pool.Exec(ctx(), `UPDATE github_rpm_sync_state SET last_synced_at = $2, sync_lease_owner = '', sync_lease_expires = NULL WHERE remote_name = $1`, name, at); err != nil {
+			t.Fatalf("set last_synced_at: %v", err)
+		}
+	}
+	claim := func(name, owner string) bool {
+		t.Helper()
+		ok, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, owner, ttl, lease)
+		if err != nil {
+			t.Fatalf("claim: %v", err)
+		}
+		return ok
+	}
+
+	for i := 0; i < 5; i++ {
+		name := fmt.Sprintf("gh-slot-%d-%s", i, time.Now().Format("150405.000000"))
+		seedGitHubRPMRemote(t, name)
+		if !claim(name, "replica-1") {
+			t.Fatalf("%s: first claim failed", name)
+		}
+
+		now := time.Now()
+		phase := syncPhase(name, ttl)
+		boundary := now.Add(-phase).Truncate(ttl).Add(phase)
+
+		setSynced(name, now.Add(-ttl-time.Second))
+		if !claim(name, "replica-1") {
+			t.Fatalf("%s: not due after a full TTL", name)
+		}
+		if claim(name, "replica-2") {
+			t.Fatalf("%s: second replica claimed a held lease", name)
+		}
+
+		setSynced(name, boundary.Add(-5*time.Second))
+		if !claim(name, "replica-2") {
+			t.Fatalf("%s: not due after crossing its boundary %v", name, boundary)
+		}
+
+		if now.Sub(boundary) > 10*time.Second {
+			setSynced(name, boundary.Add(5*time.Second))
+			if claim(name, "replica-1") {
+				t.Fatalf("%s: re-synced within the same slot", name)
+			}
+		}
+	}
+}