Add per-remote sync_interval and spread GitHub syncs #166

Merged
benvin merged 5 commits from benvin/github-sync-spread into master 2026-10-25 18:58:23 +11:00
20 changed files with 503 additions and 70 deletions
+1 -1
View File
@@ -203,7 +203,7 @@ additive and idempotent for the same reason; a test enforces it.
| `GITHUB_SYNC_RATE` | `1` | `github_rpm` syncer global GitHub request rate (req/s), shared across all remotes. `1`/s = 3600/hr, under an authenticated token's ~5000/hr; unauthenticated (~60/hr) relies on ETag/304 |
| `GITHUB_SYNC_BURST` | `5` | Token-bucket burst for the shared limiter |
| `GITHUB_SYNC_WORKERS` | `3` | Concurrent `github_rpm` scan workers |
| `GITHUB_SYNC_POLL_INTERVAL` | `60` | Base scheduler tick in seconds; per-remote cadence is its `mutable_ttl`, enforced by the DB lease |
| `GITHUB_SYNC_POLL_INTERVAL` | `60` | Base scheduler tick in seconds; per-remote cadence is its `sync_interval` (phase-spread), else `mutable_ttl`, enforced by the DB lease |
| `GITHUB_TOKEN` | | Server-level GitHub PAT (fine-grained or classic), sent as `Authorization: Bearer`. Applies to every GitHub request; per-remote creds override it. See [GitHub authentication](#github-authentication) |
| `GITHUB_APP_ID` | | GitHub App id (App auth mode; mutually exclusive with `GITHUB_TOKEN`) |
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
+7 -3
View File
@@ -32,8 +32,12 @@ derived metadata current off the client request path:
The first `dnf` request is served from cache. If a request arrives before the
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
serving an empty repo or blocking on a multi-minute derive.
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
new or changed releases no more often than its `mutable_ttl`. New/changed
- **Periodic re-check, driven by `sync_interval` or `mutable_ttl`.** With
`sync_interval` (seconds) set, a remote re-checks once per interval at a
stable per-remote offset, so remotes on the same interval spread out instead
of syncing together; its first scheduled sync waits for that slot (a
cold-cache request still primes at once). Unset, it re-checks no more often
than its `mutable_ttl`. New/changed
assets are derived incrementally; assets already cached are never re-fetched,
and assets that disappear upstream are pruned.
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
@@ -50,7 +54,7 @@ derived metadata current off the client request path:
periodic scan a replica must atomically claim a per-remote lease
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
~once per `sync_interval` (else `mutable_ttl`) regardless of replica count, and the shared `etag`
lets any replica issue the conditional request.
```hcl
+8
View File
@@ -104,6 +104,10 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := remote.ValidateSyncInterval(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := remote.ValidateQuarantine(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@@ -149,6 +153,10 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := remote.ValidateSyncInterval(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err := remote.ValidateQuarantine(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
+45
View File
@@ -56,6 +56,51 @@ func TestMutableTTLOmittedVsZero(t *testing.T) {
}
}
// sync_interval round-trips through the API and DB on github_* remotes and is
// rejected when negative or on a type without a background syncer.
func TestSyncIntervalAPI(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
h := NewRemotesHandler(db, nil, nil).Routes()
interval := func(name string) int {
t.Helper()
r, err := db.GetRemote(context.Background(), name)
if err != nil {
t.Fatalf("get %s: %v", name, err)
}
return r.SyncInterval
}
const gh = `"package_type":"github_deb","base_url":"https://api.github.com/repos/acme/tools","releases_remote":"github"`
if c := do(t, h, "POST", "/", `{"name":"si-gh",`+gh+`}`); c != 201 {
t.Fatalf("create unset = %d", c)
}
if got := interval("si-gh"); got != 0 {
t.Fatalf("unset sync_interval stored as %d", got)
}
if c := do(t, h, "PUT", "/si-gh", `{`+gh+`,"sync_interval":3600}`); c != 200 {
t.Fatalf("update = %d", c)
}
if got := interval("si-gh"); got != 3600 {
t.Fatalf("sync_interval = %d, want 3600", got)
}
for _, body := range []string{
`{"name":"si-neg",` + gh + `,"sync_interval":-1}`,
`{"name":"si-rpm","package_type":"rpm","base_url":"https://example.com/repo","sync_interval":60}`,
} {
if c := do(t, h, "POST", "/", body); c != 400 {
t.Errorf("%s = %d, want 400", body, c)
}
}
}
func TestQuarantineDaysDefault(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
+2 -2
View File
@@ -30,8 +30,8 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
// ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
// lease. See claimSyncLease.
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease, interval, phase)
}
// ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the
+2 -2
View File
@@ -30,8 +30,8 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
// ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
// lease. See claimSyncLease.
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease, interval, phase)
}
// ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the
+4 -4
View File
@@ -29,7 +29,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
const lease = 15 * time.Minute
freshness := time.Hour
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease, 0, 0)
if err != nil || !claimed {
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
}
@@ -37,7 +37,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
t.Fatalf("initial etag should be empty, got %q", etag)
}
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
if err != nil {
t.Fatalf("replica-2 claim err: %v", err)
}
@@ -49,7 +49,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
t.Fatalf("release: %v", err)
}
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
if err != nil {
t.Fatalf("replica-2 recency claim err: %v", err)
}
@@ -57,7 +57,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
t.Fatal("periodic claim succeeded inside the freshness window")
}
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease, 0, 0)
if err != nil || !claimed4 {
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
}
+17 -9
View File
@@ -34,8 +34,8 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
// ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
// See claimSyncLease.
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease, interval, phase)
}
// ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
@@ -48,22 +48,30 @@ func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner stri
// (claimed=true) only when the remote is due and no live lease is held by
// another replica. Due means: a pending retry after a failed scan has reached
// next_retry_at, or, with no retry pending, the remote was never synced or was
// synced longer than freshness ago. A zero freshness (prime scans) ignores the
// recency gate but still honours a pending retry's backoff. The returned etag is
// the stored releases-list ETag, shared across replicas so a conditional request
// last synced before the current slot (interval set) or longer than freshness
// ago (otherwise). With interval set, slots start at phase past each multiple
// of interval and a remote created inside the current slot waits for the next.
// Slots are computed from the DB clock so replicas agree whatever their own
// clocks say. A zero freshness and interval (prime scans) ignores the recency
// gate but still honours a pending retry's backoff. The returned etag is the
// stored releases-list ETag, shared across replicas so a conditional request
// can short-circuit an unchanged repo.
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
const slot = `to_timestamp(floor((extract(epoch FROM now())::float8 - $6::float8) / NULLIF($5::float8, 0)) * $5::float8 + $6::float8)`
row := db.Pool.QueryRow(ctx, `
INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
VALUES ($1, $2, now() + make_interval(secs => $4))
SELECT r.name, $2, now() + make_interval(secs => $4)
FROM remotes r
WHERE r.name = $1
AND ($5::float8 <= 0 OR COALESCE(r.created_at, '-infinity') < `+slot+`)
ON CONFLICT (remote_name) DO UPDATE
SET sync_lease_owner = $2,
sync_lease_expires = now() + make_interval(secs => $4)
WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
ELSE s.last_synced_at IS NULL OR s.last_synced_at < COALESCE(`+slot+`, now() - make_interval(secs => $3)) END)
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
RETURNING s.etag
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
`, remoteName, owner, freshness.Seconds(), lease.Seconds(), interval.Seconds(), phase.Seconds())
var etag string
if err := row.Scan(&etag); err != nil {
+132 -8
View File
@@ -1,6 +1,10 @@
package database
import (
"context"
"fmt"
"math"
"sync/atomic"
"testing"
"time"
@@ -30,7 +34,7 @@ func TestGitHubSyncLease(t *testing.T) {
freshness := time.Hour
// First claim on a never-synced remote wins; etag starts empty.
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease)
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease, 0, 0)
if err != nil || !claimed {
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
}
@@ -39,7 +43,7 @@ func TestGitHubSyncLease(t *testing.T) {
}
// A second replica cannot claim while the lease is held.
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
if err != nil {
t.Fatalf("replica-2 claim err: %v", err)
}
@@ -53,7 +57,7 @@ func TestGitHubSyncLease(t *testing.T) {
}
// A periodic re-claim inside the freshness window is blocked by recency.
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
if err != nil {
t.Fatalf("replica-2 recency claim err: %v", err)
}
@@ -62,7 +66,7 @@ func TestGitHubSyncLease(t *testing.T) {
}
// A prime (freshness 0) bypasses recency and reads the persisted etag.
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease)
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease, 0, 0)
if err != nil || !claimed4 {
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
}
@@ -108,7 +112,7 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
freshness := time.Hour
claim := func(f time.Duration) (bool, string) {
t.Helper()
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease, 0, 0)
if err != nil {
t.Fatalf("claim: %v", err)
}
@@ -220,21 +224,21 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
}{
{"github_rpm_sync_state", models.PackageGitHubRPM,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
}},
{"github_deb_sync_state", models.PackageGitHubDeb,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
}},
{"github_alpine_sync_state", models.PackageGitHubAlpine,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
@@ -276,3 +280,123 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
})
}
}
type syncTable struct {
table string
pt models.PackageType
claim func(name, owner string, freshness, interval, phase time.Duration) (bool, error)
}
func syncTables() []syncTable {
wrap := func(fn func(context.Context, string, string, time.Duration, time.Duration, time.Duration, time.Duration) (bool, string, error)) func(string, string, time.Duration, time.Duration, time.Duration) (bool, error) {
return func(name, owner string, freshness, interval, phase time.Duration) (bool, error) {
ok, _, err := fn(ctx(), name, owner, freshness, 15*time.Minute, interval, phase)
return ok, err
}
}
return []syncTable{
{"github_rpm_sync_state", models.PackageGitHubRPM, wrap(testDB.ClaimGitHubSyncLease)},
{"github_deb_sync_state", models.PackageGitHubDeb, wrap(testDB.ClaimGitHubDebSyncLease)},
{"github_alpine_sync_state", models.PackageGitHubAlpine, wrap(testDB.ClaimGitHubAlpineSyncLease)},
}
}
var syncSeq atomic.Int64
// seedSynced creates a remote of tc's type created createdAgo before the DB
// clock and, when syncedAgo >= 0, a released lease last synced syncedAgo before it.
func seedSynced(t *testing.T, tc syncTable, createdAgo, syncedAgo time.Duration) string {
t.Helper()
name := fmt.Sprintf("sync-%s-%d", tc.table, syncSeq.Add(1))
if err := testDB.CreateRemote(ctx(), &models.Remote{
Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github",
}); err != nil {
t.Fatalf("create remote: %v", err)
}
if _, err := testDB.Pool.Exec(ctx(), `UPDATE remotes SET created_at = now() - make_interval(secs => $2) WHERE name = $1`, name, createdAgo.Seconds()); err != nil {
t.Fatalf("seed created_at: %v", err)
}
if syncedAgo >= 0 {
if _, err := testDB.Pool.Exec(ctx(), `INSERT INTO `+tc.table+` (remote_name, last_synced_at) VALUES ($1, now() - make_interval(secs => $2))`, name, syncedAgo.Seconds()); err != nil {
t.Fatalf("seed sync state: %v", err)
}
}
return name
}
func mustClaim(t *testing.T, tc syncTable, name, owner string, freshness, interval, phase time.Duration) bool {
t.Helper()
ok, err := tc.claim(name, owner, freshness, interval, phase)
if err != nil {
t.Fatalf("claim: %v", err)
}
return ok
}
const neverSynced = -1
// With no interval the claim keeps the freshness-since-last-sync predicate and
// a new remote is due at once.
func TestSyncLeaseUnsetIntervalFreshness(t *testing.T) {
requireDB(t)
const ttl = time.Hour
for _, tc := range syncTables() {
t.Run(tc.table, func(t *testing.T) {
for _, c := range []struct {
ago time.Duration
freshness time.Duration
due bool
}{
{neverSynced, ttl, true},
{ttl + time.Minute, ttl, true},
{ttl - time.Minute, ttl, false},
{time.Second, ttl, false},
{time.Second, 0, true},
} {
name := seedSynced(t, tc, time.Second, c.ago)
if got := mustClaim(t, tc, name, "r1", c.freshness, 0, 0); got != c.due {
t.Errorf("synced %v ago, freshness %v: claimed=%v, want %v", c.ago, c.freshness, got, c.due)
}
}
})
}
}
// With an interval the slot comes from the DB clock: a remote is due only when
// created and last synced before the current slot, regardless of freshness,
// and still one replica at a time.
func TestSyncLeaseSlot(t *testing.T) {
requireDB(t)
var epoch float64
if err := testDB.Pool.QueryRow(ctx(), `SELECT extract(epoch FROM now())::float8`).Scan(&epoch); err != nil {
t.Fatalf("db now: %v", err)
}
const interval = time.Hour
// Pick the phase that puts the current slot's start 30m before the DB now.
phase := time.Duration(math.Mod(epoch-1800, interval.Seconds()) * float64(time.Second))
const day = 24 * time.Hour
for _, tc := range syncTables() {
t.Run(tc.table, func(t *testing.T) {
for _, c := range []struct {
desc string
createdAgo, ago time.Duration
freshness time.Duration
due bool
}{
{"created inside the slot", 10 * time.Second, neverSynced, 0, false},
{"created before the slot", 35 * time.Minute, neverSynced, 0, true},
{"synced inside the slot", day, 25 * time.Minute, 0, false},
{"synced before the slot", day, 35 * time.Minute, 10 * 365 * day, true},
} {
name := seedSynced(t, tc, c.createdAgo, c.ago)
if got := mustClaim(t, tc, name, "r1", c.freshness, interval, phase); got != c.due {
t.Errorf("%s: claimed=%v, want %v", c.desc, got, c.due)
}
if c.due && mustClaim(t, tc, name, "r2", 0, interval, phase) {
t.Errorf("%s: second replica claimed a held lease", c.desc)
}
}
})
}
}
+7 -7
View File
@@ -9,7 +9,7 @@ import (
)
const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_strategy, description, username, password,
immutable_ttl, mutable_ttl, check_mutable,
immutable_ttl, mutable_ttl, check_mutable, sync_interval,
patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error,
@@ -29,7 +29,7 @@ func normalizeMirrorStrategy(s string) string {
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
return scanner.Scan(
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.MirrorStrategy, &r.Description, &r.Username, &r.Password,
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable, &r.SyncInterval,
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
&r.BanTagsEnabled, &r.BanTags,
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
@@ -76,8 +76,8 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by,
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
mirror_strategy, serve_stale
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27)
mirror_strategy, sync_interval, serve_stale
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28)
`,
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
@@ -86,7 +86,7 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
normalizeMirrorStrategy(r.MirrorStrategy), r.ServeStale,
normalizeMirrorStrategy(r.MirrorStrategy), r.SyncInterval, r.ServeStale,
)
return err
}
@@ -101,7 +101,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
releases_remote=$20, managed_by=$21,
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
mirror_strategy=$26, serve_stale=$27,
mirror_strategy=$26, sync_interval=$27, serve_stale=$28,
updated_at=NOW()
WHERE name=$1
`,
@@ -113,7 +113,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
r.ReleasesRemote, r.ManagedBy,
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
r.Mirrorlist,
normalizeMirrorStrategy(r.MirrorStrategy), r.ServeStale,
normalizeMirrorStrategy(r.MirrorStrategy), r.SyncInterval, r.ServeStale,
)
return err
}
+11 -8
View File
@@ -27,7 +27,7 @@ const (
type SyncStore interface {
provider.RemoteMetadataStore
ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error)
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (claimed bool, etag string, err error)
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
}
@@ -129,7 +129,8 @@ func (s *Syncer) Run(ctx context.Context) {
}
// schedule enqueues a periodic check for every github_alpine remote. The DB lease
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
// enforces the per-remote sync_interval (else mutable_ttl) cadence and
// cross-replica coordination.
func (s *Syncer) schedule(ctx context.Context) {
remotes, err := s.store.ListGitHubAlpineRemotes(ctx)
if err != nil {
@@ -142,8 +143,10 @@ func (s *Syncer) schedule(ctx context.Context) {
}
// EnqueuePrime queues an immediate background prime for a freshly created remote.
// A sync_interval remote skips it and syncs in its first slot, so remotes
// created together do not burst.
func (s *Syncer) EnqueuePrime(remote models.Remote) {
if s == nil {
if s == nil || remote.SyncInterval > 0 {
return
}
s.enqueue(remote, true)
@@ -189,16 +192,16 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
s.mu.Unlock()
}()
ttl := time.Duration(job.remote.MutableTTL) * time.Second
if ttl <= 0 {
ttl = defaultSyncFreshness
}
ttl := job.remote.SyncPeriod(defaultSyncFreshness)
freshness := ttl
var interval, phase time.Duration
if job.prime {
freshness = 0
} else {
interval, phase = job.remote.SyncSchedule()
}
claimed, etag, err := s.store.ClaimGitHubAlpineSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
claimed, etag, err := s.store.ClaimGitHubAlpineSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration, interval, phase)
if err != nil {
slog.Error("github_alpine syncer: claim lease", "remote", job.remote.Name, "error", err)
return
+42 -2
View File
@@ -28,6 +28,10 @@ type fakeSyncStore struct {
lastSynced map[string]time.Time
etags map[string]string
retryAt map[string]time.Time
freshness time.Duration
interval time.Duration
phase time.Duration
attempts int
}
func newFakeSyncStore() *fakeSyncStore {
@@ -47,9 +51,11 @@ func (f *fakeSyncStore) ListGitHubAlpineRemotes(_ context.Context) ([]models.Rem
return append([]models.Remote(nil), f.remotes...), nil
}
func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.freshness, f.interval, f.phase = freshness, interval, phase
f.attempts++
now := time.Now()
ls, hasLS := f.lastSynced[name]
exp, hasExp := f.leaseExp[name]
@@ -223,7 +229,7 @@ func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
s := newSyncer(store, p, testSyncConfig())
remote := fx.remote()
claimed, _, err := store.ClaimGitHubAlpineSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
claimed, _, err := store.ClaimGitHubAlpineSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration, 0, 0)
if err != nil || !claimed {
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
}
@@ -363,3 +369,37 @@ func TestSyncerFailedAssetRetry(t *testing.T) {
})
}
}
// A periodic check claims with sync_interval as its freshness and passes the
// remote's slot schedule; unset, it falls back to mutable_ttl with no schedule.
func TestSyncerHonoursSyncInterval(t *testing.T) {
for _, interval := range []int{0, 7200} {
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
store.lastSynced["sched"] = time.Now()
remote := models.Remote{Name: "sched", MutableTTL: 90, SyncInterval: interval}
wantInterval, wantPhase := remote.SyncSchedule()
s.process(context.Background(), syncJob{remote: remote})
if want := remote.SyncPeriod(0); store.freshness != want || store.interval != wantInterval || store.phase != wantPhase {
t.Errorf("sync_interval=%d: claim freshness/interval/phase = %v/%v/%v, want %v/%v/%v",
interval, store.freshness, store.interval, store.phase, want, wantInterval, wantPhase)
}
}
}
// Create-time primes are skipped for a scheduled remote so remotes created
// together spread out; a cold-cache prime still claims with no schedule.
func TestSyncerScheduledPrime(t *testing.T) {
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
remote := models.Remote{Name: "fresh", SyncInterval: 3600}
s.EnqueuePrime(remote)
if len(s.jobs) != 0 {
t.Fatal("create-time prime enqueued for a sync_interval remote")
}
s.process(context.Background(), syncJob{remote: remote, prime: true})
if store.attempts != 1 || store.freshness != 0 || store.interval != 0 {
t.Fatalf("cold-cache prime: attempts=%d freshness=%v interval=%v, want 1/0/0", store.attempts, store.freshness, store.interval)
}
}
+11 -8
View File
@@ -27,7 +27,7 @@ const (
type SyncStore interface {
provider.RemoteMetadataStore
ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (claimed bool, etag string, err error)
ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
}
@@ -129,7 +129,8 @@ func (s *Syncer) Run(ctx context.Context) {
}
// schedule enqueues a periodic check for every github_deb remote. The DB lease
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
// enforces the per-remote sync_interval (else mutable_ttl) cadence and
// cross-replica coordination.
func (s *Syncer) schedule(ctx context.Context) {
remotes, err := s.store.ListGitHubDebRemotes(ctx)
if err != nil {
@@ -142,8 +143,10 @@ func (s *Syncer) schedule(ctx context.Context) {
}
// EnqueuePrime queues an immediate background prime for a freshly created remote.
// A sync_interval remote skips it and syncs in its first slot, so remotes
// created together do not burst.
func (s *Syncer) EnqueuePrime(remote models.Remote) {
if s == nil {
if s == nil || remote.SyncInterval > 0 {
return
}
s.enqueue(remote, true)
@@ -189,16 +192,16 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
s.mu.Unlock()
}()
ttl := time.Duration(job.remote.MutableTTL) * time.Second
if ttl <= 0 {
ttl = defaultSyncFreshness
}
ttl := job.remote.SyncPeriod(defaultSyncFreshness)
freshness := ttl
var interval, phase time.Duration
if job.prime {
freshness = 0
} else {
interval, phase = job.remote.SyncSchedule()
}
claimed, etag, err := s.store.ClaimGitHubDebSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
claimed, etag, err := s.store.ClaimGitHubDebSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration, interval, phase)
if err != nil {
slog.Error("github_deb syncer: claim lease", "remote", job.remote.Name, "error", err)
return
+42 -2
View File
@@ -28,6 +28,10 @@ type fakeSyncStore struct {
lastSynced map[string]time.Time
etags map[string]string
retryAt map[string]time.Time
freshness time.Duration
interval time.Duration
phase time.Duration
attempts int
}
func newFakeSyncStore() *fakeSyncStore {
@@ -47,9 +51,11 @@ func (f *fakeSyncStore) ListGitHubDebRemotes(_ context.Context) ([]models.Remote
return append([]models.Remote(nil), f.remotes...), nil
}
func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.freshness, f.interval, f.phase = freshness, interval, phase
f.attempts++
now := time.Now()
ls, hasLS := f.lastSynced[name]
exp, hasExp := f.leaseExp[name]
@@ -223,7 +229,7 @@ func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
s := newSyncer(store, p, testSyncConfig())
remote := fx.remote()
claimed, _, err := store.ClaimGitHubDebSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
claimed, _, err := store.ClaimGitHubDebSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration, 0, 0)
if err != nil || !claimed {
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
}
@@ -363,3 +369,37 @@ func TestSyncerFailedAssetRetry(t *testing.T) {
})
}
}
// A periodic check claims with sync_interval as its freshness and passes the
// remote's slot schedule; unset, it falls back to mutable_ttl with no schedule.
func TestSyncerHonoursSyncInterval(t *testing.T) {
for _, interval := range []int{0, 7200} {
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
store.lastSynced["sched"] = time.Now()
remote := models.Remote{Name: "sched", MutableTTL: 90, SyncInterval: interval}
wantInterval, wantPhase := remote.SyncSchedule()
s.process(context.Background(), syncJob{remote: remote})
if want := remote.SyncPeriod(0); store.freshness != want || store.interval != wantInterval || store.phase != wantPhase {
t.Errorf("sync_interval=%d: claim freshness/interval/phase = %v/%v/%v, want %v/%v/%v",
interval, store.freshness, store.interval, store.phase, want, wantInterval, wantPhase)
}
}
}
// Create-time primes are skipped for a scheduled remote so remotes created
// together spread out; a cold-cache prime still claims with no schedule.
func TestSyncerScheduledPrime(t *testing.T) {
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
remote := models.Remote{Name: "fresh", SyncInterval: 3600}
s.EnqueuePrime(remote)
if len(s.jobs) != 0 {
t.Fatal("create-time prime enqueued for a sync_interval remote")
}
s.process(context.Background(), syncJob{remote: remote, prime: true})
if store.attempts != 1 || store.freshness != 0 || store.interval != 0 {
t.Fatalf("cold-cache prime: attempts=%d freshness=%v interval=%v, want 1/0/0", store.attempts, store.freshness, store.interval)
}
}
+14 -12
View File
@@ -21,7 +21,7 @@ const (
// scan never loses its lease, while a crashed replica's lease still expires.
syncLeaseDuration = 15 * time.Minute
// defaultSyncFreshness is the periodic re-check interval used when a remote's
// mutable_ttl is unset.
// sync_interval and mutable_ttl are unset.
defaultSyncFreshness = 5 * time.Minute
// jobQueueDepth bounds the pending work queue; enqueues past it are dropped
// (a later poll re-enqueues), never blocking the caller.
@@ -34,7 +34,7 @@ const (
type SyncStore interface {
provider.RemoteMetadataStore
ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (claimed bool, etag string, err error)
ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
}
@@ -43,7 +43,7 @@ type SyncConfig struct {
RatePerSec float64 // global GitHub request rate (req/s)
Burst int // token-bucket burst
Workers int // concurrent scan workers
PollInterval time.Duration // base scheduler tick; per-remote cadence is mutable_ttl
PollInterval time.Duration // base scheduler tick; per-remote cadence is sync_interval or mutable_ttl
}
type syncJob struct {
@@ -139,9 +139,9 @@ func (s *Syncer) Run(ctx context.Context) {
}
// schedule enqueues a periodic check for every github_rpm remote. The DB lease
// (claimed in the worker) enforces the per-remote mutable_ttl cadence and cross
// replica coordination, so enqueuing every tick is cheap: a not-yet-due remote
// simply fails to claim and is skipped.
// (claimed in the worker) enforces the per-remote sync_interval (else
// mutable_ttl) cadence and cross replica coordination, so enqueuing every tick
// is cheap: a not-yet-due remote simply fails to claim and is skipped.
func (s *Syncer) schedule(ctx context.Context) {
remotes, err := s.store.ListGitHubRPMRemotes(ctx)
if err != nil {
@@ -155,8 +155,10 @@ func (s *Syncer) schedule(ctx context.Context) {
// EnqueuePrime queues an immediate background prime for a freshly created
// remote so its metadata is derived without blocking the create call.
// A sync_interval remote skips it and syncs in its first slot, so remotes
// created together do not burst.
func (s *Syncer) EnqueuePrime(remote models.Remote) {
if s == nil {
if s == nil || remote.SyncInterval > 0 {
return
}
s.enqueue(remote, true)
@@ -205,16 +207,16 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
s.mu.Unlock()
}()
ttl := time.Duration(job.remote.MutableTTL) * time.Second
if ttl <= 0 {
ttl = defaultSyncFreshness
}
ttl := job.remote.SyncPeriod(defaultSyncFreshness)
freshness := ttl
var interval, phase time.Duration
if job.prime {
freshness = 0 // prime ignores the recency gate but still respects a live lease
} else {
interval, phase = job.remote.SyncSchedule()
}
claimed, etag, err := s.store.ClaimGitHubSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
claimed, etag, err := s.store.ClaimGitHubSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration, interval, phase)
if err != nil {
slog.Error("github_rpm syncer: claim lease", "remote", job.remote.Name, "error", err)
return
+42 -2
View File
@@ -29,6 +29,10 @@ type fakeSyncStore struct {
etags map[string]string
retryAt map[string]time.Time
lastResult provider.SyncResult
freshness time.Duration
interval time.Duration
phase time.Duration
attempts int
}
func newFakeSyncStore() *fakeSyncStore {
@@ -48,9 +52,11 @@ func (f *fakeSyncStore) ListGitHubRPMRemotes(_ context.Context) ([]models.Remote
return append([]models.Remote(nil), f.remotes...), nil
}
func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.freshness, f.interval, f.phase = freshness, interval, phase
f.attempts++
now := time.Now()
ls, hasLS := f.lastSynced[name]
exp, hasExp := f.leaseExp[name]
@@ -230,7 +236,7 @@ func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
remote := fx.remote()
// Replica 1 holds the lease.
claimed, _, err := store.ClaimGitHubSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
claimed, _, err := store.ClaimGitHubSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration, 0, 0)
if err != nil || !claimed {
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
}
@@ -470,3 +476,37 @@ func TestSyncerFailedAssetRetry(t *testing.T) {
})
}
}
// A periodic check claims with sync_interval as its freshness and passes the
// remote's slot schedule; unset, it falls back to mutable_ttl with no schedule.
func TestSyncerHonoursSyncInterval(t *testing.T) {
for _, interval := range []int{0, 7200} {
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
store.lastSynced["sched"] = time.Now()
remote := models.Remote{Name: "sched", MutableTTL: 90, SyncInterval: interval}
wantInterval, wantPhase := remote.SyncSchedule()
s.process(context.Background(), syncJob{remote: remote})
if want := remote.SyncPeriod(0); store.freshness != want || store.interval != wantInterval || store.phase != wantPhase {
t.Errorf("sync_interval=%d: claim freshness/interval/phase = %v/%v/%v, want %v/%v/%v",
interval, store.freshness, store.interval, store.phase, want, wantInterval, wantPhase)
}
}
}
// Create-time primes are skipped for a scheduled remote so remotes created
// together spread out; a cold-cache prime still claims with no schedule.
func TestSyncerScheduledPrime(t *testing.T) {
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
remote := models.Remote{Name: "fresh", SyncInterval: 3600}
s.EnqueuePrime(remote)
if len(s.jobs) != 0 {
t.Fatal("create-time prime enqueued for a sync_interval remote")
}
s.process(context.Background(), syncJob{remote: remote, prime: true})
if store.attempts != 1 || store.freshness != 0 || store.interval != 0 {
t.Fatalf("cold-cache prime: attempts=%d freshness=%v interval=%v, want 1/0/0", store.attempts, store.freshness, store.interval)
}
}
+1
View File
@@ -0,0 +1 @@
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS sync_interval INTEGER NOT NULL DEFAULT 0;
+49
View File
@@ -2,6 +2,7 @@ package models
import (
"fmt"
"hash/fnv"
"net/url"
"regexp"
"time"
@@ -55,6 +56,9 @@ type Remote struct {
ImmutableTTL int `json:"immutable_ttl"`
MutableTTL int `json:"mutable_ttl"`
CheckMutable bool `json:"check_mutable"`
// SyncInterval (seconds) sets how often a github_* remote rescans releases
// and rebuilds its index; 0 falls back to mutable_ttl.
SyncInterval int `json:"sync_interval,omitempty"`
// Upstream HTTP timeouts in seconds. 0 means use the server default.
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
@@ -152,6 +156,51 @@ func (r *Remote) ValidateMirrorStrategy() error {
return nil
}
// SyncPeriod is the background sync cadence: sync_interval, else mutable_ttl,
// else def.
func (r Remote) SyncPeriod(def time.Duration) time.Duration {
if r.SyncInterval > 0 {
return time.Duration(r.SyncInterval) * time.Second
}
if r.MutableTTL > 0 {
return time.Duration(r.MutableTTL) * time.Second
}
return def
}
// SyncSchedule is the sync_interval and the stable per-remote phase offsetting
// its slots, so remotes sharing an interval spread across it; both zero when
// sync_interval is unset.
func (r Remote) SyncSchedule() (interval, phase time.Duration) {
if r.SyncInterval <= 0 {
return 0, 0
}
interval = time.Duration(r.SyncInterval) * time.Second
return interval, syncPhase(r.Name, interval)
}
func syncPhase(name string, period time.Duration) time.Duration {
h := fnv.New64a()
_, _ = h.Write([]byte(name))
return time.Duration(h.Sum64() % uint64(period))
}
// ValidateSyncInterval rejects a negative sync_interval, or one set on a type
// without a background syncer.
func (r *Remote) ValidateSyncInterval() error {
if r.SyncInterval < 0 {
return fmt.Errorf("sync_interval must be >= 0, got %d", r.SyncInterval)
}
switch r.PackageType {
case PackageGitHubRPM, PackageGitHubDeb, PackageGitHubAlpine:
default:
if r.SyncInterval > 0 {
return fmt.Errorf("sync_interval is only valid on github_rpm, github_deb and github_alpine remotes")
}
}
return nil
}
// DefaultQuarantineDays matches the remotes.quarantine_days column default.
const DefaultQuarantineDays = 3
+65
View File
@@ -2,8 +2,10 @@ package models
import (
"encoding/json"
"fmt"
"strings"
"testing"
"time"
)
func TestRemote_ValidatePatterns(t *testing.T) {
@@ -123,3 +125,66 @@ func TestValidateMirrorlist(t *testing.T) {
})
}
}
func TestSyncPeriod(t *testing.T) {
const def = 5 * time.Minute
for _, tc := range []struct {
interval, ttl int
want time.Duration
}{
{0, 0, def},
{0, 600, 10 * time.Minute},
{3600, 600, time.Hour},
} {
r := Remote{SyncInterval: tc.interval, MutableTTL: tc.ttl}
if got := r.SyncPeriod(def); got != tc.want {
t.Errorf("SyncPeriod(interval=%d, ttl=%d) = %v, want %v", tc.interval, tc.ttl, got, tc.want)
}
}
}
func TestValidateSyncInterval(t *testing.T) {
for _, tc := range []struct {
pt PackageType
interval int
ok bool
}{
{PackageGitHubRPM, 3600, true},
{PackageGitHubDeb, 3600, true},
{PackageGitHubAlpine, 3600, true},
{PackageGitHubRPM, -1, false},
{PackageRPM, 0, true},
{PackageRPM, 60, false},
} {
r := Remote{PackageType: tc.pt, SyncInterval: tc.interval}
if err := r.ValidateSyncInterval(); (err == nil) != tc.ok {
t.Errorf("%s interval=%d: err=%v, want ok=%v", tc.pt, tc.interval, err, tc.ok)
}
}
}
func TestSyncSchedule(t *testing.T) {
if i, p := (Remote{Name: "x", MutableTTL: 60}).SyncSchedule(); i != 0 || p != 0 {
t.Fatalf("unset sync_interval schedule = %v/%v, want 0/0", i, p)
}
const period = 5 * time.Minute
phases := map[time.Duration]bool{}
var lo, hi time.Duration = period, 0
for i := 0; i < 13; i++ {
r := Remote{Name: fmt.Sprintf("github-rpm-%02d", i), SyncInterval: int(period.Seconds())}
interval, p := r.SyncSchedule()
if interval != period || p < 0 || p >= period {
t.Fatalf("%s: schedule %v/%v outside [0, %v)", r.Name, interval, p, period)
}
if _, again := r.SyncSchedule(); again != p {
t.Fatalf("%s: phase not stable", r.Name)
}
p = p.Truncate(time.Second)
phases[p] = true
lo, hi = min(lo, p), max(hi, p)
}
if len(phases) != 13 || hi-lo < period/2 {
t.Fatalf("phases bunched: %d distinct, spread %v of %v", len(phases), hi-lo, period)
}
}
+1
View File
@@ -8,6 +8,7 @@ export interface Remote {
immutable_ttl: number;
mutable_ttl: number;
check_mutable: boolean;
sync_interval?: number;
patterns: string[];
blocklist: string[];
mutable_patterns: string[];