Add per-remote sync_interval and spread GitHub syncs (#166)

github_* remotes sharing a mutable_ttl all become due at the same moment, so many remotes created or synced together burst GitHub at once.

- add per-remote `sync_interval` (seconds) for github_rpm/github_deb/github_alpine remotes
- offset each remote's slots by a stable name-hashed phase so remotes on one interval spread out
- compute slots and the first-sync gate in the lease claim from the DB clock so replicas agree
- wait for the first slot on a scheduled first sync and skip the create-time prime; cold-cache primes stay immediate
- keep the mutable_ttl freshness predicate unchanged when `sync_interval` is unset
- document `sync_interval` in README, rpm docs and UI types

Reviewed-on: #166
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

pkg/models/remote.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"fmt"
+	"hash/fnv"
 	"net/url"
 	"regexp"
 	"time"
@@ -55,6 +56,9 @@ type Remote struct {
 	ImmutableTTL int  `json:"immutable_ttl"`
 	MutableTTL   int  `json:"mutable_ttl"`
 	CheckMutable bool `json:"check_mutable"`
+	// SyncInterval (seconds) sets how often a github_* remote rescans releases
+	// and rebuilds its index; 0 falls back to mutable_ttl.
+	SyncInterval int `json:"sync_interval,omitempty"`
 
 	// Upstream HTTP timeouts in seconds. 0 means use the server default.
 	UpstreamDialTimeout           int `json:"upstream_dial_timeout,omitempty"`
@@ -152,6 +156,51 @@ func (r *Remote) ValidateMirrorStrategy() error {
 	return nil
 }
 
+// SyncPeriod is the background sync cadence: sync_interval, else mutable_ttl,
+// else def.
+func (r Remote) SyncPeriod(def time.Duration) time.Duration {
+	if r.SyncInterval > 0 {
+		return time.Duration(r.SyncInterval) * time.Second
+	}
+	if r.MutableTTL > 0 {
+		return time.Duration(r.MutableTTL) * time.Second
+	}
+	return def
+}
+
+// SyncSchedule is the sync_interval and the stable per-remote phase offsetting
+// its slots, so remotes sharing an interval spread across it; both zero when
+// sync_interval is unset.
+func (r Remote) SyncSchedule() (interval, phase time.Duration) {
+	if r.SyncInterval <= 0 {
+		return 0, 0
+	}
+	interval = time.Duration(r.SyncInterval) * time.Second
+	return interval, syncPhase(r.Name, interval)
+}
+
+func syncPhase(name string, period time.Duration) time.Duration {
+	h := fnv.New64a()
+	_, _ = h.Write([]byte(name))
+	return time.Duration(h.Sum64() % uint64(period))
+}
+
+// ValidateSyncInterval rejects a negative sync_interval, or one set on a type
+// without a background syncer.
+func (r *Remote) ValidateSyncInterval() error {
+	if r.SyncInterval < 0 {
+		return fmt.Errorf("sync_interval must be >= 0, got %d", r.SyncInterval)
+	}
+	switch r.PackageType {
+	case PackageGitHubRPM, PackageGitHubDeb, PackageGitHubAlpine:
+	default:
+		if r.SyncInterval > 0 {
+			return fmt.Errorf("sync_interval is only valid on github_rpm, github_deb and github_alpine remotes")
+		}
+	}
+	return nil
+}
+
 // DefaultQuarantineDays matches the remotes.quarantine_days column default.
 const DefaultQuarantineDays = 3
 

pkg/models/remote_test.go

@@ -2,8 +2,10 @@ package models
 
 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestRemote_ValidatePatterns(t *testing.T) {
@@ -123,3 +125,66 @@ func TestValidateMirrorlist(t *testing.T) {
 		})
 	}
 }
+
+func TestSyncPeriod(t *testing.T) {
+	const def = 5 * time.Minute
+	for _, tc := range []struct {
+		interval, ttl int
+		want          time.Duration
+	}{
+		{0, 0, def},
+		{0, 600, 10 * time.Minute},
+		{3600, 600, time.Hour},
+	} {
+		r := Remote{SyncInterval: tc.interval, MutableTTL: tc.ttl}
+		if got := r.SyncPeriod(def); got != tc.want {
+			t.Errorf("SyncPeriod(interval=%d, ttl=%d) = %v, want %v", tc.interval, tc.ttl, got, tc.want)
+		}
+	}
+}
+
+func TestValidateSyncInterval(t *testing.T) {
+	for _, tc := range []struct {
+		pt       PackageType
+		interval int
+		ok       bool
+	}{
+		{PackageGitHubRPM, 3600, true},
+		{PackageGitHubDeb, 3600, true},
+		{PackageGitHubAlpine, 3600, true},
+		{PackageGitHubRPM, -1, false},
+		{PackageRPM, 0, true},
+		{PackageRPM, 60, false},
+	} {
+		r := Remote{PackageType: tc.pt, SyncInterval: tc.interval}
+		if err := r.ValidateSyncInterval(); (err == nil) != tc.ok {
+			t.Errorf("%s interval=%d: err=%v, want ok=%v", tc.pt, tc.interval, err, tc.ok)
+		}
+	}
+}
+
+func TestSyncSchedule(t *testing.T) {
+	if i, p := (Remote{Name: "x", MutableTTL: 60}).SyncSchedule(); i != 0 || p != 0 {
+		t.Fatalf("unset sync_interval schedule = %v/%v, want 0/0", i, p)
+	}
+
+	const period = 5 * time.Minute
+	phases := map[time.Duration]bool{}
+	var lo, hi time.Duration = period, 0
+	for i := 0; i < 13; i++ {
+		r := Remote{Name: fmt.Sprintf("github-rpm-%02d", i), SyncInterval: int(period.Seconds())}
+		interval, p := r.SyncSchedule()
+		if interval != period || p < 0 || p >= period {
+			t.Fatalf("%s: schedule %v/%v outside [0, %v)", r.Name, interval, p, period)
+		}
+		if _, again := r.SyncSchedule(); again != p {
+			t.Fatalf("%s: phase not stable", r.Name)
+		}
+		p = p.Truncate(time.Second)
+		phases[p] = true
+		lo, hi = min(lo, p), max(hi, p)
+	}
+	if len(phases) != 13 || hi-lo < period/2 {
+		t.Fatalf("phases bunched: %d distinct, spread %v of %v", len(phases), hi-lo, period)
+	}
+}