Add per-remote sync_interval and spread GitHub syncs across it
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful

This commit is contained in:
2026-10-25 13:12:59 +11:00
parent 9294686ea6
commit 707e380c98
14 changed files with 315 additions and 31 deletions
+20 -5
View File
@@ -3,6 +3,7 @@ package database
import (
"context"
"errors"
"hash/fnv"
"time"
"github.com/jackc/pgx/v5"
@@ -47,9 +48,11 @@ func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner stri
// claimSyncLease atomically claims a per-remote sync lease in table. It succeeds
// (claimed=true) only when the remote is due and no live lease is held by
// another replica. Due means: a pending retry after a failed scan has reached
// next_retry_at, or, with no retry pending, the remote was never synced or was
// synced longer than freshness ago. A zero freshness (prime scans) ignores the
// recency gate but still honours a pending retry's backoff. The returned etag is
// next_retry_at, or, with no retry pending, the remote was never synced or not
// since its latest slot boundary. Boundaries recur every freshness, offset by a
// stable per-remote phase so remotes sharing a TTL are spread across it rather
// than due together. A zero freshness (prime scans) ignores the recency gate
// but still honours a pending retry's backoff. The returned etag is
// the stored releases-list ETag, shared across replicas so a conditional request
// can short-circuit an unchanged repo.
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
@@ -60,10 +63,11 @@ func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner strin
SET sync_lease_owner = $2,
sync_lease_expires = now() + make_interval(secs => $4)
WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
ELSE s.last_synced_at IS NULL OR s.last_synced_at < CASE WHEN $3 <= 0 THEN now()
ELSE to_timestamp(floor((extract(epoch from now())::float8 - $5) / NULLIF($3, 0)) * $3 + $5) END END)
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
RETURNING s.etag
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
`, remoteName, owner, freshness.Seconds(), lease.Seconds(), syncPhase(remoteName, freshness).Seconds())
var etag string
if err := row.Scan(&etag); err != nil {
@@ -75,6 +79,17 @@ func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner strin
return true, etag, nil
}
// syncPhase is remoteName's stable offset within period, so every replica
// agrees on when each remote is due.
func syncPhase(remoteName string, period time.Duration) time.Duration {
if period <= 0 {
return 0
}
h := fnv.New64a()
_, _ = h.Write([]byte(remoteName))
return time.Duration(h.Sum64() % uint64(period))
}
// releaseSyncLease records a scan outcome and frees the lease; only the owning
// replica may release. Success advances last_synced_at, persists the etag and
// clears any retry. Failure leaves last_synced_at and etag untouched (the last