Add per-remote sync_interval and spread GitHub syncs (#166)
ci/woodpecker/tag/docker Pipeline was successful

github_* remotes sharing a mutable_ttl all become due at the same moment, so many remotes created or synced together burst GitHub at once.

- add per-remote `sync_interval` (seconds) for github_rpm/github_deb/github_alpine remotes
- offset each remote's slots by a stable name-hashed phase so remotes on one interval spread out
- compute slots and the first-sync gate in the lease claim from the DB clock so replicas agree
- wait for the first slot on a scheduled first sync and skip the create-time prime; cold-cache primes stay immediate
- keep the mutable_ttl freshness predicate unchanged when `sync_interval` is unset
- document `sync_interval` in README, rpm docs and UI types

Reviewed-on: #166
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #166.
This commit is contained in:
2026-10-25 18:58:22 +11:00
committed by BenVincent
parent e5313342ba
commit ee55ef3fcd
20 changed files with 503 additions and 70 deletions
+49
View File
@@ -2,6 +2,7 @@ package models
import (
"fmt"
"hash/fnv"
"net/url"
"regexp"
"time"
@@ -55,6 +56,9 @@ type Remote struct {
ImmutableTTL int `json:"immutable_ttl"`
MutableTTL int `json:"mutable_ttl"`
CheckMutable bool `json:"check_mutable"`
// SyncInterval (seconds) sets how often a github_* remote rescans releases
// and rebuilds its index; 0 falls back to mutable_ttl.
SyncInterval int `json:"sync_interval,omitempty"`
// Upstream HTTP timeouts in seconds. 0 means use the server default.
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
@@ -152,6 +156,51 @@ func (r *Remote) ValidateMirrorStrategy() error {
return nil
}
// SyncPeriod is the background sync cadence: sync_interval, else mutable_ttl,
// else def.
func (r Remote) SyncPeriod(def time.Duration) time.Duration {
if r.SyncInterval > 0 {
return time.Duration(r.SyncInterval) * time.Second
}
if r.MutableTTL > 0 {
return time.Duration(r.MutableTTL) * time.Second
}
return def
}
// SyncSchedule is the sync_interval and the stable per-remote phase offsetting
// its slots, so remotes sharing an interval spread across it; both zero when
// sync_interval is unset.
func (r Remote) SyncSchedule() (interval, phase time.Duration) {
if r.SyncInterval <= 0 {
return 0, 0
}
interval = time.Duration(r.SyncInterval) * time.Second
return interval, syncPhase(r.Name, interval)
}
func syncPhase(name string, period time.Duration) time.Duration {
h := fnv.New64a()
_, _ = h.Write([]byte(name))
return time.Duration(h.Sum64() % uint64(period))
}
// ValidateSyncInterval rejects a negative sync_interval, or one set on a type
// without a background syncer.
func (r *Remote) ValidateSyncInterval() error {
if r.SyncInterval < 0 {
return fmt.Errorf("sync_interval must be >= 0, got %d", r.SyncInterval)
}
switch r.PackageType {
case PackageGitHubRPM, PackageGitHubDeb, PackageGitHubAlpine:
default:
if r.SyncInterval > 0 {
return fmt.Errorf("sync_interval is only valid on github_rpm, github_deb and github_alpine remotes")
}
}
return nil
}
// DefaultQuarantineDays matches the remotes.quarantine_days column default.
const DefaultQuarantineDays = 3