Merge master

internal/database/alpine_github_sync.go

@@ -30,8 +30,8 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
 
 // ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
 // lease. See claimSyncLease.
-func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
-	return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
+func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
+	return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease, interval, phase)
 }
 
 // ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the

internal/database/database_test.go

@@ -84,12 +84,14 @@ func TestRemotesCRUD(t *testing.T) {
 	}
 	got.BaseURL = "https://updated.example.com"
 	got.CacheRepodata = true
+	got.SyncInterval = 600
+	got.ServeStale = true
 	if err := testDB.UpdateRemote(ctx(), got); err != nil {
 		t.Fatalf("update: %v", err)
 	}
 	got, _ = testDB.GetRemote(ctx(), "r-crud")
-	if got.BaseURL != "https://updated.example.com" || !got.CacheRepodata {
-		t.Errorf("update not applied: %v %v", got.BaseURL, got.CacheRepodata)
+	if got.BaseURL != "https://updated.example.com" || !got.CacheRepodata || got.SyncInterval != 600 || !got.ServeStale {
+		t.Errorf("update not applied: %v %v %v %v", got.BaseURL, got.CacheRepodata, got.SyncInterval, got.ServeStale)
 	}
 	list, err := testDB.ListRemotes(ctx())
 	if err != nil || len(list) == 0 {

internal/database/deb_github_sync.go

@@ -30,8 +30,8 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
 
 // ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
 // lease. See claimSyncLease.
-func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
-	return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
+func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
+	return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease, interval, phase)
 }
 
 // ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the

internal/database/deb_github_sync_test.go

@@ -29,7 +29,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
 	const lease = 15 * time.Minute
 	freshness := time.Hour
 
-	claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
+	claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease, 0, 0)
 	if err != nil || !claimed {
 		t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
 	}
@@ -37,7 +37,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
 		t.Fatalf("initial etag should be empty, got %q", etag)
 	}
 
-	claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
+	claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
 	if err != nil {
 		t.Fatalf("replica-2 claim err: %v", err)
 	}
@@ -49,7 +49,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
 		t.Fatalf("release: %v", err)
 	}
 
-	claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
+	claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
 	if err != nil {
 		t.Fatalf("replica-2 recency claim err: %v", err)
 	}
@@ -57,7 +57,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
 		t.Fatal("periodic claim succeeded inside the freshness window")
 	}
 
-	claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
+	claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease, 0, 0)
 	if err != nil || !claimed4 {
 		t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
 	}

internal/database/github_sync.go

@@ -34,8 +34,8 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
 
 // ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
 // See claimSyncLease.
-func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
-	return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
+func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
+	return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease, interval, phase)
 }
 
 // ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
@@ -48,22 +48,30 @@ func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner stri
 // (claimed=true) only when the remote is due and no live lease is held by
 // another replica. Due means: a pending retry after a failed scan has reached
 // next_retry_at, or, with no retry pending, the remote was never synced or was
-// synced longer than freshness ago. A zero freshness (prime scans) ignores the
-// recency gate but still honours a pending retry's backoff. The returned etag is
-// the stored releases-list ETag, shared across replicas so a conditional request
+// last synced before the current slot (interval set) or longer than freshness
+// ago (otherwise). With interval set, slots start at phase past each multiple
+// of interval and a remote created inside the current slot waits for the next.
+// Slots are computed from the DB clock so replicas agree whatever their own
+// clocks say. A zero freshness and interval (prime scans) ignores the recency
+// gate but still honours a pending retry's backoff. The returned etag is the
+// stored releases-list ETag, shared across replicas so a conditional request
 // can short-circuit an unchanged repo.
-func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
+func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
+	const slot = `to_timestamp(floor((extract(epoch FROM now())::float8 - $6::float8) / NULLIF($5::float8, 0)) * $5::float8 + $6::float8)`
 	row := db.Pool.QueryRow(ctx, `
 		INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
-		VALUES ($1, $2, now() + make_interval(secs => $4))
+		SELECT r.name, $2, now() + make_interval(secs => $4)
+		  FROM remotes r
+		 WHERE r.name = $1
+		   AND ($5::float8 <= 0 OR COALESCE(r.created_at, '-infinity') < `+slot+`)
 		ON CONFLICT (remote_name) DO UPDATE
 			SET sync_lease_owner   = $2,
 			    sync_lease_expires = now() + make_interval(secs => $4)
 			WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
-			            ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
+			            ELSE s.last_synced_at IS NULL OR s.last_synced_at < COALESCE(`+slot+`, now() - make_interval(secs => $3)) END)
 			  AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
 		RETURNING s.etag
-	`, remoteName, owner, freshness.Seconds(), lease.Seconds())
+	`, remoteName, owner, freshness.Seconds(), lease.Seconds(), interval.Seconds(), phase.Seconds())
 
 	var etag string
 	if err := row.Scan(&etag); err != nil {

internal/database/github_sync_test.go

@@ -1,6 +1,10 @@
 package database
 
 import (
+	"context"
+	"fmt"
+	"math"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -30,7 +34,7 @@ func TestGitHubSyncLease(t *testing.T) {
 	freshness := time.Hour
 
 	// First claim on a never-synced remote wins; etag starts empty.
-	claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease)
+	claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease, 0, 0)
 	if err != nil || !claimed {
 		t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
 	}
@@ -39,7 +43,7 @@ func TestGitHubSyncLease(t *testing.T) {
 	}
 
 	// A second replica cannot claim while the lease is held.
-	claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
+	claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
 	if err != nil {
 		t.Fatalf("replica-2 claim err: %v", err)
 	}
@@ -53,7 +57,7 @@ func TestGitHubSyncLease(t *testing.T) {
 	}
 
 	// A periodic re-claim inside the freshness window is blocked by recency.
-	claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
+	claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
 	if err != nil {
 		t.Fatalf("replica-2 recency claim err: %v", err)
 	}
@@ -62,7 +66,7 @@ func TestGitHubSyncLease(t *testing.T) {
 	}
 
 	// A prime (freshness 0) bypasses recency and reads the persisted etag.
-	claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease)
+	claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease, 0, 0)
 	if err != nil || !claimed4 {
 		t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
 	}
@@ -108,7 +112,7 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
 	freshness := time.Hour
 	claim := func(f time.Duration) (bool, string) {
 		t.Helper()
-		ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
+		ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease, 0, 0)
 		if err != nil {
 			t.Fatalf("claim: %v", err)
 		}
@@ -220,21 +224,21 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
 	}{
 		{"github_rpm_sync_state", models.PackageGitHubRPM,
 			func(n string) (bool, string, error) {
-				return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+				return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
 			},
 			func(n string, res provider.SyncResult) error {
 				return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
 			}},
 		{"github_deb_sync_state", models.PackageGitHubDeb,
 			func(n string) (bool, string, error) {
-				return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+				return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
 			},
 			func(n string, res provider.SyncResult) error {
 				return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
 			}},
 		{"github_alpine_sync_state", models.PackageGitHubAlpine,
 			func(n string) (bool, string, error) {
-				return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+				return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
 			},
 			func(n string, res provider.SyncResult) error {
 				return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
@@ -276,3 +280,123 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
 		})
 	}
 }
+
+type syncTable struct {
+	table string
+	pt    models.PackageType
+	claim func(name, owner string, freshness, interval, phase time.Duration) (bool, error)
+}
+
+func syncTables() []syncTable {
+	wrap := func(fn func(context.Context, string, string, time.Duration, time.Duration, time.Duration, time.Duration) (bool, string, error)) func(string, string, time.Duration, time.Duration, time.Duration) (bool, error) {
+		return func(name, owner string, freshness, interval, phase time.Duration) (bool, error) {
+			ok, _, err := fn(ctx(), name, owner, freshness, 15*time.Minute, interval, phase)
+			return ok, err
+		}
+	}
+	return []syncTable{
+		{"github_rpm_sync_state", models.PackageGitHubRPM, wrap(testDB.ClaimGitHubSyncLease)},
+		{"github_deb_sync_state", models.PackageGitHubDeb, wrap(testDB.ClaimGitHubDebSyncLease)},
+		{"github_alpine_sync_state", models.PackageGitHubAlpine, wrap(testDB.ClaimGitHubAlpineSyncLease)},
+	}
+}
+
+var syncSeq atomic.Int64
+
+// seedSynced creates a remote of tc's type created createdAgo before the DB
+// clock and, when syncedAgo >= 0, a released lease last synced syncedAgo before it.
+func seedSynced(t *testing.T, tc syncTable, createdAgo, syncedAgo time.Duration) string {
+	t.Helper()
+	name := fmt.Sprintf("sync-%s-%d", tc.table, syncSeq.Add(1))
+	if err := testDB.CreateRemote(ctx(), &models.Remote{
+		Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
+		BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github",
+	}); err != nil {
+		t.Fatalf("create remote: %v", err)
+	}
+	if _, err := testDB.Pool.Exec(ctx(), `UPDATE remotes SET created_at = now() - make_interval(secs => $2) WHERE name = $1`, name, createdAgo.Seconds()); err != nil {
+		t.Fatalf("seed created_at: %v", err)
+	}
+	if syncedAgo >= 0 {
+		if _, err := testDB.Pool.Exec(ctx(), `INSERT INTO `+tc.table+` (remote_name, last_synced_at) VALUES ($1, now() - make_interval(secs => $2))`, name, syncedAgo.Seconds()); err != nil {
+			t.Fatalf("seed sync state: %v", err)
+		}
+	}
+	return name
+}
+
+func mustClaim(t *testing.T, tc syncTable, name, owner string, freshness, interval, phase time.Duration) bool {
+	t.Helper()
+	ok, err := tc.claim(name, owner, freshness, interval, phase)
+	if err != nil {
+		t.Fatalf("claim: %v", err)
+	}
+	return ok
+}
+
+const neverSynced = -1
+
+// With no interval the claim keeps the freshness-since-last-sync predicate and
+// a new remote is due at once.
+func TestSyncLeaseUnsetIntervalFreshness(t *testing.T) {
+	requireDB(t)
+	const ttl = time.Hour
+	for _, tc := range syncTables() {
+		t.Run(tc.table, func(t *testing.T) {
+			for _, c := range []struct {
+				ago       time.Duration
+				freshness time.Duration
+				due       bool
+			}{
+				{neverSynced, ttl, true},
+				{ttl + time.Minute, ttl, true},
+				{ttl - time.Minute, ttl, false},
+				{time.Second, ttl, false},
+				{time.Second, 0, true},
+			} {
+				name := seedSynced(t, tc, time.Second, c.ago)
+				if got := mustClaim(t, tc, name, "r1", c.freshness, 0, 0); got != c.due {
+					t.Errorf("synced %v ago, freshness %v: claimed=%v, want %v", c.ago, c.freshness, got, c.due)
+				}
+			}
+		})
+	}
+}
+
+// With an interval the slot comes from the DB clock: a remote is due only when
+// created and last synced before the current slot, regardless of freshness,
+// and still one replica at a time.
+func TestSyncLeaseSlot(t *testing.T) {
+	requireDB(t)
+	var epoch float64
+	if err := testDB.Pool.QueryRow(ctx(), `SELECT extract(epoch FROM now())::float8`).Scan(&epoch); err != nil {
+		t.Fatalf("db now: %v", err)
+	}
+	const interval = time.Hour
+	// Pick the phase that puts the current slot's start 30m before the DB now.
+	phase := time.Duration(math.Mod(epoch-1800, interval.Seconds()) * float64(time.Second))
+	const day = 24 * time.Hour
+	for _, tc := range syncTables() {
+		t.Run(tc.table, func(t *testing.T) {
+			for _, c := range []struct {
+				desc            string
+				createdAgo, ago time.Duration
+				freshness       time.Duration
+				due             bool
+			}{
+				{"created inside the slot", 10 * time.Second, neverSynced, 0, false},
+				{"created before the slot", 35 * time.Minute, neverSynced, 0, true},
+				{"synced inside the slot", day, 25 * time.Minute, 0, false},
+				{"synced before the slot", day, 35 * time.Minute, 10 * 365 * day, true},
+			} {
+				name := seedSynced(t, tc, c.createdAgo, c.ago)
+				if got := mustClaim(t, tc, name, "r1", c.freshness, interval, phase); got != c.due {
+					t.Errorf("%s: claimed=%v, want %v", c.desc, got, c.due)
+				}
+				if c.due && mustClaim(t, tc, name, "r2", 0, interval, phase) {
+					t.Errorf("%s: second replica claimed a held lease", c.desc)
+				}
+			}
+		})
+	}
+}

internal/database/remotes.go

@@ -9,7 +9,7 @@ import (
 )
 
 const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_strategy, description, username, password,
-	immutable_ttl, mutable_ttl, check_mutable,
+	immutable_ttl, mutable_ttl, check_mutable, sync_interval,
 	patterns, blocklist, mutable_patterns, immutable_patterns,
 	ban_tags_enabled, ban_tags,
 	quarantine_enabled, quarantine_days, stale_on_error,
@@ -30,7 +30,7 @@ func normalizeMirrorStrategy(s string) string {
 func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
 	return scanner.Scan(
 		&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.MirrorStrategy, &r.Description, &r.Username, &r.Password,
-		&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
+		&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable, &r.SyncInterval,
 		&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
 		&r.BanTagsEnabled, &r.BanTags,
 		&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
@@ -78,8 +78,8 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
 			quarantine_enabled, quarantine_days, stale_on_error,
 			releases_remote, managed_by,
 			upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
-			mirror_strategy, serve_stale, cache_repodata
-		) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28)
+			mirror_strategy, sync_interval, serve_stale, cache_repodata
+		) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28,$29)
 	`,
 		r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
 		r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
@@ -88,7 +88,7 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
 		r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
 		r.ReleasesRemote, r.ManagedBy,
 		r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
-		normalizeMirrorStrategy(r.MirrorStrategy), r.ServeStale, r.CacheRepodata,
+		normalizeMirrorStrategy(r.MirrorStrategy), r.SyncInterval, r.ServeStale, r.CacheRepodata,
 	)
 	return err
 }
@@ -103,7 +103,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
 			quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
 			releases_remote=$20, managed_by=$21,
 			upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
-			mirror_strategy=$26, serve_stale=$27, cache_repodata=$28,
+			mirror_strategy=$26, sync_interval=$27, serve_stale=$28, cache_repodata=$29,
 			updated_at=NOW()
 		WHERE name=$1
 	`,
@@ -115,7 +115,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
 		r.ReleasesRemote, r.ManagedBy,
 		r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
 		r.Mirrorlist,
-		normalizeMirrorStrategy(r.MirrorStrategy), r.ServeStale, r.CacheRepodata,
+		normalizeMirrorStrategy(r.MirrorStrategy), r.SyncInterval, r.ServeStale, r.CacheRepodata,
 	)
 	return err
 }