Confine phased sync slots to sync_interval remotes
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful

Unset remotes keep the mutable_ttl freshness predicate. Scheduled remotes
compute their slot in Go, skip the create-time prime and wait for their
first slot; a cold-cache prime still runs at once. Document sync_interval.
This commit is contained in:
2026-10-25 13:27:43 +11:00
parent 707e380c98
commit 1582533e5a
16 changed files with 343 additions and 135 deletions
+2 -2
View File
@@ -30,8 +30,8 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
// ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
// lease. See claimSyncLease.
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration, slot time.Time) (bool, string, error) {
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease, slot)
}
// ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the
+2 -2
View File
@@ -30,8 +30,8 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
// ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
// lease. See claimSyncLease.
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration, slot time.Time) (bool, string, error) {
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease, slot)
}
// ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the
+4 -4
View File
@@ -29,7 +29,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
const lease = 15 * time.Minute
freshness := time.Hour
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease, time.Time{})
if err != nil || !claimed {
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
}
@@ -37,7 +37,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
t.Fatalf("initial etag should be empty, got %q", etag)
}
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, time.Time{})
if err != nil {
t.Fatalf("replica-2 claim err: %v", err)
}
@@ -49,7 +49,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
t.Fatalf("release: %v", err)
}
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, time.Time{})
if err != nil {
t.Fatalf("replica-2 recency claim err: %v", err)
}
@@ -57,7 +57,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
t.Fatal("periodic claim succeeded inside the freshness window")
}
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease, time.Time{})
if err != nil || !claimed4 {
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
}
+13 -23
View File
@@ -3,7 +3,6 @@ package database
import (
"context"
"errors"
"hash/fnv"
"time"
"github.com/jackc/pgx/v5"
@@ -35,8 +34,8 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
// ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
// See claimSyncLease.
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration, slot time.Time) (bool, string, error) {
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease, slot)
}
// ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
@@ -48,14 +47,17 @@ func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner stri
// claimSyncLease atomically claims a per-remote sync lease in table. It succeeds
// (claimed=true) only when the remote is due and no live lease is held by
// another replica. Due means: a pending retry after a failed scan has reached
// next_retry_at, or, with no retry pending, the remote was never synced or not
// since its latest slot boundary. Boundaries recur every freshness, offset by a
// stable per-remote phase so remotes sharing a TTL are spread across it rather
// than due together. A zero freshness (prime scans) ignores the recency gate
// but still honours a pending retry's backoff. The returned etag is
// next_retry_at, or, with no retry pending, the remote was never synced or was
// last synced before slot (when set) or longer than freshness ago (otherwise).
// A zero freshness with no slot (prime scans) ignores the recency gate but
// still honours a pending retry's backoff. The returned etag is
// the stored releases-list ETag, shared across replicas so a conditional request
// can short-circuit an unchanged repo.
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration, slot time.Time) (bool, string, error) {
var slotArg *time.Time
if !slot.IsZero() {
slotArg = &slot
}
row := db.Pool.QueryRow(ctx, `
INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
VALUES ($1, $2, now() + make_interval(secs => $4))
@@ -63,11 +65,10 @@ func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner strin
SET sync_lease_owner = $2,
sync_lease_expires = now() + make_interval(secs => $4)
WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
ELSE s.last_synced_at IS NULL OR s.last_synced_at < CASE WHEN $3 <= 0 THEN now()
ELSE to_timestamp(floor((extract(epoch from now())::float8 - $5) / NULLIF($3, 0)) * $3 + $5) END END)
ELSE s.last_synced_at IS NULL OR s.last_synced_at < COALESCE($5::timestamptz, now() - make_interval(secs => $3)) END)
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
RETURNING s.etag
`, remoteName, owner, freshness.Seconds(), lease.Seconds(), syncPhase(remoteName, freshness).Seconds())
`, remoteName, owner, freshness.Seconds(), lease.Seconds(), slotArg)
var etag string
if err := row.Scan(&etag); err != nil {
@@ -79,17 +80,6 @@ func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner strin
return true, etag, nil
}
// syncPhase is remoteName's stable offset within period, so every replica
// agrees on when each remote is due.
func syncPhase(remoteName string, period time.Duration) time.Duration {
if period <= 0 {
return 0
}
h := fnv.New64a()
_, _ = h.Write([]byte(remoteName))
return time.Duration(h.Sum64() % uint64(period))
}
// releaseSyncLease records a scan outcome and frees the lease; only the owning
// replica may release. Success advances last_synced_at, persists the etag and
// clears any retry. Failure leaves last_synced_at and etag untouched (the last
+92 -82
View File
@@ -1,7 +1,7 @@
package database
import (
"fmt"
"context"
"testing"
"time"
@@ -31,7 +31,7 @@ func TestGitHubSyncLease(t *testing.T) {
freshness := time.Hour
// First claim on a never-synced remote wins; etag starts empty.
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease)
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease, time.Time{})
if err != nil || !claimed {
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
}
@@ -40,7 +40,7 @@ func TestGitHubSyncLease(t *testing.T) {
}
// A second replica cannot claim while the lease is held.
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, time.Time{})
if err != nil {
t.Fatalf("replica-2 claim err: %v", err)
}
@@ -54,7 +54,7 @@ func TestGitHubSyncLease(t *testing.T) {
}
// A periodic re-claim inside the freshness window is blocked by recency.
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, time.Time{})
if err != nil {
t.Fatalf("replica-2 recency claim err: %v", err)
}
@@ -63,7 +63,7 @@ func TestGitHubSyncLease(t *testing.T) {
}
// A prime (freshness 0) bypasses recency and reads the persisted etag.
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease)
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease, time.Time{})
if err != nil || !claimed4 {
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
}
@@ -109,7 +109,7 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
freshness := time.Hour
claim := func(f time.Duration) (bool, string) {
t.Helper()
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease, time.Time{})
if err != nil {
t.Fatalf("claim: %v", err)
}
@@ -221,21 +221,21 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
}{
{"github_rpm_sync_state", models.PackageGitHubRPM,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute, time.Time{})
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
}},
{"github_deb_sync_state", models.PackageGitHubDeb,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute, time.Time{})
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
}},
{"github_alpine_sync_state", models.PackageGitHubAlpine,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute, time.Time{})
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
@@ -278,85 +278,95 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
}
}
func TestSyncPhaseStableAndSpread(t *testing.T) {
const period = 5 * time.Minute
phases := map[time.Duration]bool{}
var lo, hi time.Duration = period, 0
for i := 0; i < 13; i++ {
name := fmt.Sprintf("github-rpm-%02d", i)
p := syncPhase(name, period)
if p != syncPhase(name, period) {
t.Fatalf("%s: phase not stable", name)
type syncTable struct {
table string
pt models.PackageType
claim func(name, owner string, freshness time.Duration, slot time.Time) (bool, error)
}
func syncTables() []syncTable {
wrap := func(fn func(context.Context, string, string, time.Duration, time.Duration, time.Time) (bool, string, error)) func(string, string, time.Duration, time.Time) (bool, error) {
return func(name, owner string, freshness time.Duration, slot time.Time) (bool, error) {
ok, _, err := fn(ctx(), name, owner, freshness, 15*time.Minute, slot)
return ok, err
}
if p < 0 || p >= period {
t.Fatalf("%s: phase %v outside [0,%v)", name, p, period)
}
phases[p.Truncate(time.Second)] = true
lo, hi = min(lo, p), max(hi, p)
}
if len(phases) != 13 {
t.Fatalf("phases collide: %d distinct of 13", len(phases))
}
if hi-lo < period/2 {
t.Fatalf("phases bunched in %v of %v", hi-lo, period)
}
if syncPhase("x", 0) != 0 {
t.Fatal("zero period must give zero phase")
return []syncTable{
{"github_rpm_sync_state", models.PackageGitHubRPM, wrap(testDB.ClaimGitHubSyncLease)},
{"github_deb_sync_state", models.PackageGitHubDeb, wrap(testDB.ClaimGitHubDebSyncLease)},
{"github_alpine_sync_state", models.PackageGitHubAlpine, wrap(testDB.ClaimGitHubAlpineSyncLease)},
}
}
// TestGitHubSyncLeasePhasedSlots checks a remote is due once per slot: synced
// before its latest boundary (or over a TTL ago) it is claimable by exactly one
// replica; synced after the boundary it is not.
func TestGitHubSyncLeasePhasedSlots(t *testing.T) {
// seedSynced creates a remote of tc's type with a released lease last synced at.
func seedSynced(t *testing.T, tc syncTable, prefix string, at time.Time) string {
t.Helper()
name := prefix + "-" + tc.table + "-" + time.Now().Format("150405.000000")
if err := testDB.CreateRemote(ctx(), &models.Remote{
Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github",
}); err != nil {
t.Fatalf("create remote: %v", err)
}
if _, err := testDB.Pool.Exec(ctx(), `INSERT INTO `+tc.table+` (remote_name, last_synced_at) VALUES ($1, $2)`, name, at); err != nil {
t.Fatalf("seed sync state: %v", err)
}
return name
}
func mustClaim(t *testing.T, tc syncTable, name, owner string, freshness time.Duration, slot time.Time) bool {
t.Helper()
ok, err := tc.claim(name, owner, freshness, slot)
if err != nil {
t.Fatalf("claim: %v", err)
}
return ok
}
// With no slot the claim keeps the freshness-since-last-sync predicate.
func TestSyncLeaseUnsetIntervalFreshness(t *testing.T) {
requireDB(t)
const lease = 15 * time.Minute
ttl := time.Hour
setSynced := func(name string, at time.Time) {
t.Helper()
if _, err := testDB.Pool.Exec(ctx(), `UPDATE github_rpm_sync_state SET last_synced_at = $2, sync_lease_owner = '', sync_lease_expires = NULL WHERE remote_name = $1`, name, at); err != nil {
t.Fatalf("set last_synced_at: %v", err)
}
}
claim := func(name, owner string) bool {
t.Helper()
ok, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, owner, ttl, lease)
if err != nil {
t.Fatalf("claim: %v", err)
}
return ok
}
for i := 0; i < 5; i++ {
name := fmt.Sprintf("gh-slot-%d-%s", i, time.Now().Format("150405.000000"))
seedGitHubRPMRemote(t, name)
if !claim(name, "replica-1") {
t.Fatalf("%s: first claim failed", name)
}
now := time.Now()
phase := syncPhase(name, ttl)
boundary := now.Add(-phase).Truncate(ttl).Add(phase)
setSynced(name, now.Add(-ttl-time.Second))
if !claim(name, "replica-1") {
t.Fatalf("%s: not due after a full TTL", name)
}
if claim(name, "replica-2") {
t.Fatalf("%s: second replica claimed a held lease", name)
}
setSynced(name, boundary.Add(-5*time.Second))
if !claim(name, "replica-2") {
t.Fatalf("%s: not due after crossing its boundary %v", name, boundary)
}
if now.Sub(boundary) > 10*time.Second {
setSynced(name, boundary.Add(5*time.Second))
if claim(name, "replica-1") {
t.Fatalf("%s: re-synced within the same slot", name)
const ttl = time.Hour
for _, tc := range syncTables() {
t.Run(tc.table, func(t *testing.T) {
for _, c := range []struct {
ago time.Duration
freshness time.Duration
due bool
}{
{ttl + time.Minute, ttl, true},
{ttl - time.Minute, ttl, false},
{time.Second, ttl, false},
{time.Second, 0, true},
} {
name := seedSynced(t, tc, "fresh", time.Now().Add(-c.ago))
if got := mustClaim(t, tc, name, "r1", c.freshness, time.Time{}); got != c.due {
t.Errorf("synced %v ago, freshness %v: claimed=%v, want %v", c.ago, c.freshness, got, c.due)
}
}
}
})
}
}
// With a slot the remote is due only when last synced before it, regardless of
// freshness, and still one replica at a time.
func TestSyncLeaseSlot(t *testing.T) {
requireDB(t)
slot := time.Date(2026, 3, 1, 10, 17, 42, 0, time.UTC)
for _, tc := range syncTables() {
t.Run(tc.table, func(t *testing.T) {
name := seedSynced(t, tc, "slot", slot.Add(time.Second))
if mustClaim(t, tc, name, "r1", 0, slot) {
t.Fatal("claimed though synced inside the slot")
}
name = seedSynced(t, tc, "slot", slot.Add(-time.Second))
if !mustClaim(t, tc, name, "r1", time.Hour*24*365*10, slot) {
t.Fatal("not due though last synced before the slot")
}
if mustClaim(t, tc, name, "r2", 0, slot) {
t.Fatal("second replica claimed a held lease")
}
})
}
}