Add per-remote sync_interval and spread GitHub syncs (#166)
ci/woodpecker/tag/docker Pipeline was successful
ci/woodpecker/tag/docker Pipeline was successful
github_* remotes sharing a mutable_ttl all become due at the same moment, so many remotes created or synced together burst GitHub at once. - add per-remote `sync_interval` (seconds) for github_rpm/github_deb/github_alpine remotes - offset each remote's slots by a stable name-hashed phase so remotes on one interval spread out - compute slots and the first-sync gate in the lease claim from the DB clock so replicas agree - wait for the first slot on a scheduled first sync and skip the create-time prime; cold-cache primes stay immediate - keep the mutable_ttl freshness predicate unchanged when `sync_interval` is unset - document `sync_interval` in README, rpm docs and UI types Reviewed-on: #166 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #166.
This commit is contained in:
@@ -2,6 +2,7 @@ package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"time"
|
||||
@@ -55,6 +56,9 @@ type Remote struct {
|
||||
ImmutableTTL int `json:"immutable_ttl"`
|
||||
MutableTTL int `json:"mutable_ttl"`
|
||||
CheckMutable bool `json:"check_mutable"`
|
||||
// SyncInterval (seconds) sets how often a github_* remote rescans releases
|
||||
// and rebuilds its index; 0 falls back to mutable_ttl.
|
||||
SyncInterval int `json:"sync_interval,omitempty"`
|
||||
|
||||
// Upstream HTTP timeouts in seconds. 0 means use the server default.
|
||||
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
|
||||
@@ -152,6 +156,51 @@ func (r *Remote) ValidateMirrorStrategy() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SyncPeriod is the background sync cadence: sync_interval, else mutable_ttl,
|
||||
// else def.
|
||||
func (r Remote) SyncPeriod(def time.Duration) time.Duration {
|
||||
if r.SyncInterval > 0 {
|
||||
return time.Duration(r.SyncInterval) * time.Second
|
||||
}
|
||||
if r.MutableTTL > 0 {
|
||||
return time.Duration(r.MutableTTL) * time.Second
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// SyncSchedule is the sync_interval and the stable per-remote phase offsetting
|
||||
// its slots, so remotes sharing an interval spread across it; both zero when
|
||||
// sync_interval is unset.
|
||||
func (r Remote) SyncSchedule() (interval, phase time.Duration) {
|
||||
if r.SyncInterval <= 0 {
|
||||
return 0, 0
|
||||
}
|
||||
interval = time.Duration(r.SyncInterval) * time.Second
|
||||
return interval, syncPhase(r.Name, interval)
|
||||
}
|
||||
|
||||
func syncPhase(name string, period time.Duration) time.Duration {
|
||||
h := fnv.New64a()
|
||||
_, _ = h.Write([]byte(name))
|
||||
return time.Duration(h.Sum64() % uint64(period))
|
||||
}
|
||||
|
||||
// ValidateSyncInterval rejects a negative sync_interval, or one set on a type
|
||||
// without a background syncer.
|
||||
func (r *Remote) ValidateSyncInterval() error {
|
||||
if r.SyncInterval < 0 {
|
||||
return fmt.Errorf("sync_interval must be >= 0, got %d", r.SyncInterval)
|
||||
}
|
||||
switch r.PackageType {
|
||||
case PackageGitHubRPM, PackageGitHubDeb, PackageGitHubAlpine:
|
||||
default:
|
||||
if r.SyncInterval > 0 {
|
||||
return fmt.Errorf("sync_interval is only valid on github_rpm, github_deb and github_alpine remotes")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultQuarantineDays matches the remotes.quarantine_days column default.
|
||||
const DefaultQuarantineDays = 3
|
||||
|
||||
|
||||
Reference in New Issue
Block a user