Add per-remote sync_interval and spread GitHub syncs #166
@@ -203,7 +203,7 @@ additive and idempotent for the same reason; a test enforces it.
|
||||
| `GITHUB_SYNC_RATE` | `1` | `github_rpm` syncer global GitHub request rate (req/s), shared across all remotes. `1`/s = 3600/hr, under an authenticated token's ~5000/hr; unauthenticated (~60/hr) relies on ETag/304 |
|
||||
| `GITHUB_SYNC_BURST` | `5` | Token-bucket burst for the shared limiter |
|
||||
| `GITHUB_SYNC_WORKERS` | `3` | Concurrent `github_rpm` scan workers |
|
||||
| `GITHUB_SYNC_POLL_INTERVAL` | `60` | Base scheduler tick in seconds; per-remote cadence is its `mutable_ttl`, enforced by the DB lease |
|
||||
| `GITHUB_SYNC_POLL_INTERVAL` | `60` | Base scheduler tick in seconds; per-remote cadence is its `sync_interval` (phase-spread), else `mutable_ttl`, enforced by the DB lease |
|
||||
| `GITHUB_TOKEN` | | Server-level GitHub PAT (fine-grained or classic), sent as `Authorization: Bearer`. Applies to every GitHub request; per-remote creds override it. See [GitHub authentication](#github-authentication) |
|
||||
| `GITHUB_APP_ID` | | GitHub App id (App auth mode; mutually exclusive with `GITHUB_TOKEN`) |
|
||||
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
|
||||
|
||||
@@ -32,8 +32,12 @@ derived metadata current off the client request path:
|
||||
The first `dnf` request is served from cache. If a request arrives before the
|
||||
prime lands, it returns a retryable `503` (with `Retry-After`) rather than
|
||||
serving an empty repo or blocking on a multi-minute derive.
|
||||
- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for
|
||||
new or changed releases no more often than its `mutable_ttl`. New/changed
|
||||
- **Periodic re-check, driven by `sync_interval` or `mutable_ttl`.** With
|
||||
`sync_interval` (seconds) set, a remote re-checks once per interval at a
|
||||
stable per-remote offset, so remotes on the same interval spread out instead
|
||||
of syncing together; its first scheduled sync waits for that slot (a
|
||||
cold-cache request still primes at once). Unset, it re-checks no more often
|
||||
than its `mutable_ttl`. New/changed
|
||||
assets are derived incrementally; assets already cached are never re-fetched,
|
||||
and assets that disappear upstream are pruned.
|
||||
- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
|
||||
@@ -50,7 +54,7 @@ derived metadata current off the client request path:
|
||||
periodic scan a replica must atomically claim a per-remote lease
|
||||
(`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
|
||||
`sync_lease_expires`); only the winner scans. This bounds total GitHub load to
|
||||
~once per `mutable_ttl` regardless of replica count, and the shared `etag`
|
||||
~once per `sync_interval` (else `mutable_ttl`) regardless of replica count, and the shared `etag`
|
||||
lets any replica issue the conditional request.
|
||||
|
||||
```hcl
|
||||
|
||||
@@ -104,6 +104,10 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateSyncInterval(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateQuarantine(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
@@ -149,6 +153,10 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateSyncInterval(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := remote.ValidateQuarantine(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
|
||||
@@ -56,6 +56,51 @@ func TestMutableTTLOmittedVsZero(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// sync_interval round-trips through the API and DB on github_* remotes and is
|
||||
// rejected when negative or on a type without a background syncer.
|
||||
func TestSyncIntervalAPI(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
h := NewRemotesHandler(db, nil, nil).Routes()
|
||||
interval := func(name string) int {
|
||||
t.Helper()
|
||||
r, err := db.GetRemote(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", name, err)
|
||||
}
|
||||
return r.SyncInterval
|
||||
}
|
||||
const gh = `"package_type":"github_deb","base_url":"https://api.github.com/repos/acme/tools","releases_remote":"github"`
|
||||
|
||||
if c := do(t, h, "POST", "/", `{"name":"si-gh",`+gh+`}`); c != 201 {
|
||||
t.Fatalf("create unset = %d", c)
|
||||
}
|
||||
if got := interval("si-gh"); got != 0 {
|
||||
t.Fatalf("unset sync_interval stored as %d", got)
|
||||
}
|
||||
if c := do(t, h, "PUT", "/si-gh", `{`+gh+`,"sync_interval":3600}`); c != 200 {
|
||||
t.Fatalf("update = %d", c)
|
||||
}
|
||||
if got := interval("si-gh"); got != 3600 {
|
||||
t.Fatalf("sync_interval = %d, want 3600", got)
|
||||
}
|
||||
|
||||
for _, body := range []string{
|
||||
`{"name":"si-neg",` + gh + `,"sync_interval":-1}`,
|
||||
`{"name":"si-rpm","package_type":"rpm","base_url":"https://example.com/repo","sync_interval":60}`,
|
||||
} {
|
||||
if c := do(t, h, "POST", "/", body); c != 400 {
|
||||
t.Errorf("%s = %d, want 400", body, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuarantineDaysDefault(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
|
||||
@@ -30,8 +30,8 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
|
||||
|
||||
// ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
|
||||
// lease. See claimSyncLease.
|
||||
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
|
||||
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease, interval, phase)
|
||||
}
|
||||
|
||||
// ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the
|
||||
|
||||
@@ -30,8 +30,8 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
|
||||
// ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
|
||||
// lease. See claimSyncLease.
|
||||
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
|
||||
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease, interval, phase)
|
||||
}
|
||||
|
||||
// ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the
|
||||
|
||||
@@ -29,7 +29,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
|
||||
const lease = 15 * time.Minute
|
||||
freshness := time.Hour
|
||||
|
||||
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
|
||||
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease, 0, 0)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
@@ -37,7 +37,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
|
||||
t.Fatalf("initial etag should be empty, got %q", etag)
|
||||
}
|
||||
|
||||
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 claim err: %v", err)
|
||||
}
|
||||
@@ -49,7 +49,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 recency claim err: %v", err)
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
|
||||
t.Fatal("periodic claim succeeded inside the freshness window")
|
||||
}
|
||||
|
||||
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
|
||||
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease, 0, 0)
|
||||
if err != nil || !claimed4 {
|
||||
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
|
||||
}
|
||||
|
||||
@@ -34,8 +34,8 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
|
||||
// ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
|
||||
// See claimSyncLease.
|
||||
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
|
||||
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
|
||||
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease, interval, phase)
|
||||
}
|
||||
|
||||
// ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
|
||||
@@ -48,22 +48,30 @@ func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner stri
|
||||
// (claimed=true) only when the remote is due and no live lease is held by
|
||||
// another replica. Due means: a pending retry after a failed scan has reached
|
||||
// next_retry_at, or, with no retry pending, the remote was never synced or was
|
||||
// synced longer than freshness ago. A zero freshness (prime scans) ignores the
|
||||
// recency gate but still honours a pending retry's backoff. The returned etag is
|
||||
// the stored releases-list ETag, shared across replicas so a conditional request
|
||||
// last synced before the current slot (interval set) or longer than freshness
|
||||
// ago (otherwise). With interval set, slots start at phase past each multiple
|
||||
// of interval and a remote created inside the current slot waits for the next.
|
||||
// Slots are computed from the DB clock so replicas agree whatever their own
|
||||
// clocks say. A zero freshness and interval (prime scans) ignores the recency
|
||||
// gate but still honours a pending retry's backoff. The returned etag is the
|
||||
// stored releases-list ETag, shared across replicas so a conditional request
|
||||
// can short-circuit an unchanged repo.
|
||||
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
|
||||
const slot = `to_timestamp(floor((extract(epoch FROM now())::float8 - $6::float8) / NULLIF($5::float8, 0)) * $5::float8 + $6::float8)`
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
SELECT r.name, $2, now() + make_interval(secs => $4)
|
||||
FROM remotes r
|
||||
WHERE r.name = $1
|
||||
AND ($5::float8 <= 0 OR COALESCE(r.created_at, '-infinity') < `+slot+`)
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
|
||||
ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
|
||||
ELSE s.last_synced_at IS NULL OR s.last_synced_at < COALESCE(`+slot+`, now() - make_interval(secs => $3)) END)
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds(), interval.Seconds(), phase.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -30,7 +34,7 @@ func TestGitHubSyncLease(t *testing.T) {
|
||||
freshness := time.Hour
|
||||
|
||||
// First claim on a never-synced remote wins; etag starts empty.
|
||||
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease)
|
||||
claimed, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-1", freshness, lease, 0, 0)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
@@ -39,7 +43,7 @@ func TestGitHubSyncLease(t *testing.T) {
|
||||
}
|
||||
|
||||
// A second replica cannot claim while the lease is held.
|
||||
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
claimed2, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 claim err: %v", err)
|
||||
}
|
||||
@@ -53,7 +57,7 @@ func TestGitHubSyncLease(t *testing.T) {
|
||||
}
|
||||
|
||||
// A periodic re-claim inside the freshness window is blocked by recency.
|
||||
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
claimed3, _, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", freshness, lease, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 recency claim err: %v", err)
|
||||
}
|
||||
@@ -62,7 +66,7 @@ func TestGitHubSyncLease(t *testing.T) {
|
||||
}
|
||||
|
||||
// A prime (freshness 0) bypasses recency and reads the persisted etag.
|
||||
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease)
|
||||
claimed4, etag4, err := testDB.ClaimGitHubSyncLease(ctx(), name, "replica-2", 0, lease, 0, 0)
|
||||
if err != nil || !claimed4 {
|
||||
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
|
||||
}
|
||||
@@ -108,7 +112,7 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
|
||||
freshness := time.Hour
|
||||
claim := func(f time.Duration) (bool, string) {
|
||||
t.Helper()
|
||||
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
|
||||
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease, 0, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
@@ -220,21 +224,21 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
|
||||
}{
|
||||
{"github_rpm_sync_state", models.PackageGitHubRPM,
|
||||
func(n string) (bool, string, error) {
|
||||
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
|
||||
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
|
||||
},
|
||||
func(n string, res provider.SyncResult) error {
|
||||
return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
|
||||
}},
|
||||
{"github_deb_sync_state", models.PackageGitHubDeb,
|
||||
func(n string) (bool, string, error) {
|
||||
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
|
||||
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
|
||||
},
|
||||
func(n string, res provider.SyncResult) error {
|
||||
return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
|
||||
}},
|
||||
{"github_alpine_sync_state", models.PackageGitHubAlpine,
|
||||
func(n string) (bool, string, error) {
|
||||
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
|
||||
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute, 0, 0)
|
||||
},
|
||||
func(n string, res provider.SyncResult) error {
|
||||
return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
|
||||
@@ -276,3 +280,123 @@ func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type syncTable struct {
|
||||
table string
|
||||
pt models.PackageType
|
||||
claim func(name, owner string, freshness, interval, phase time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
func syncTables() []syncTable {
|
||||
wrap := func(fn func(context.Context, string, string, time.Duration, time.Duration, time.Duration, time.Duration) (bool, string, error)) func(string, string, time.Duration, time.Duration, time.Duration) (bool, error) {
|
||||
return func(name, owner string, freshness, interval, phase time.Duration) (bool, error) {
|
||||
ok, _, err := fn(ctx(), name, owner, freshness, 15*time.Minute, interval, phase)
|
||||
return ok, err
|
||||
}
|
||||
}
|
||||
return []syncTable{
|
||||
{"github_rpm_sync_state", models.PackageGitHubRPM, wrap(testDB.ClaimGitHubSyncLease)},
|
||||
{"github_deb_sync_state", models.PackageGitHubDeb, wrap(testDB.ClaimGitHubDebSyncLease)},
|
||||
{"github_alpine_sync_state", models.PackageGitHubAlpine, wrap(testDB.ClaimGitHubAlpineSyncLease)},
|
||||
}
|
||||
}
|
||||
|
||||
var syncSeq atomic.Int64
|
||||
|
||||
// seedSynced creates a remote of tc's type created createdAgo before the DB
|
||||
// clock and, when syncedAgo >= 0, a released lease last synced syncedAgo before it.
|
||||
func seedSynced(t *testing.T, tc syncTable, createdAgo, syncedAgo time.Duration) string {
|
||||
t.Helper()
|
||||
name := fmt.Sprintf("sync-%s-%d", tc.table, syncSeq.Add(1))
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github",
|
||||
}); err != nil {
|
||||
t.Fatalf("create remote: %v", err)
|
||||
}
|
||||
if _, err := testDB.Pool.Exec(ctx(), `UPDATE remotes SET created_at = now() - make_interval(secs => $2) WHERE name = $1`, name, createdAgo.Seconds()); err != nil {
|
||||
t.Fatalf("seed created_at: %v", err)
|
||||
}
|
||||
if syncedAgo >= 0 {
|
||||
if _, err := testDB.Pool.Exec(ctx(), `INSERT INTO `+tc.table+` (remote_name, last_synced_at) VALUES ($1, now() - make_interval(secs => $2))`, name, syncedAgo.Seconds()); err != nil {
|
||||
t.Fatalf("seed sync state: %v", err)
|
||||
}
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func mustClaim(t *testing.T, tc syncTable, name, owner string, freshness, interval, phase time.Duration) bool {
|
||||
t.Helper()
|
||||
ok, err := tc.claim(name, owner, freshness, interval, phase)
|
||||
if err != nil {
|
||||
t.Fatalf("claim: %v", err)
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
const neverSynced = -1
|
||||
|
||||
// With no interval the claim keeps the freshness-since-last-sync predicate and
|
||||
// a new remote is due at once.
|
||||
func TestSyncLeaseUnsetIntervalFreshness(t *testing.T) {
|
||||
requireDB(t)
|
||||
const ttl = time.Hour
|
||||
for _, tc := range syncTables() {
|
||||
t.Run(tc.table, func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
ago time.Duration
|
||||
freshness time.Duration
|
||||
due bool
|
||||
}{
|
||||
{neverSynced, ttl, true},
|
||||
{ttl + time.Minute, ttl, true},
|
||||
{ttl - time.Minute, ttl, false},
|
||||
{time.Second, ttl, false},
|
||||
{time.Second, 0, true},
|
||||
} {
|
||||
name := seedSynced(t, tc, time.Second, c.ago)
|
||||
if got := mustClaim(t, tc, name, "r1", c.freshness, 0, 0); got != c.due {
|
||||
t.Errorf("synced %v ago, freshness %v: claimed=%v, want %v", c.ago, c.freshness, got, c.due)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// With an interval the slot comes from the DB clock: a remote is due only when
|
||||
// created and last synced before the current slot, regardless of freshness,
|
||||
// and still one replica at a time.
|
||||
func TestSyncLeaseSlot(t *testing.T) {
|
||||
requireDB(t)
|
||||
var epoch float64
|
||||
if err := testDB.Pool.QueryRow(ctx(), `SELECT extract(epoch FROM now())::float8`).Scan(&epoch); err != nil {
|
||||
t.Fatalf("db now: %v", err)
|
||||
}
|
||||
const interval = time.Hour
|
||||
// Pick the phase that puts the current slot's start 30m before the DB now.
|
||||
phase := time.Duration(math.Mod(epoch-1800, interval.Seconds()) * float64(time.Second))
|
||||
const day = 24 * time.Hour
|
||||
for _, tc := range syncTables() {
|
||||
t.Run(tc.table, func(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
desc string
|
||||
createdAgo, ago time.Duration
|
||||
freshness time.Duration
|
||||
due bool
|
||||
}{
|
||||
{"created inside the slot", 10 * time.Second, neverSynced, 0, false},
|
||||
{"created before the slot", 35 * time.Minute, neverSynced, 0, true},
|
||||
{"synced inside the slot", day, 25 * time.Minute, 0, false},
|
||||
{"synced before the slot", day, 35 * time.Minute, 10 * 365 * day, true},
|
||||
} {
|
||||
name := seedSynced(t, tc, c.createdAgo, c.ago)
|
||||
if got := mustClaim(t, tc, name, "r1", c.freshness, interval, phase); got != c.due {
|
||||
t.Errorf("%s: claimed=%v, want %v", c.desc, got, c.due)
|
||||
}
|
||||
if c.due && mustClaim(t, tc, name, "r2", 0, interval, phase) {
|
||||
t.Errorf("%s: second replica claimed a held lease", c.desc)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
)
|
||||
|
||||
const remoteCols = `name, package_type, repo_type, base_url, mirrorlist, mirror_strategy, description, username, password,
|
||||
immutable_ttl, mutable_ttl, check_mutable,
|
||||
immutable_ttl, mutable_ttl, check_mutable, sync_interval,
|
||||
patterns, blocklist, mutable_patterns, immutable_patterns,
|
||||
ban_tags_enabled, ban_tags,
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
@@ -29,7 +29,7 @@ func normalizeMirrorStrategy(s string) string {
|
||||
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
|
||||
return scanner.Scan(
|
||||
&r.Name, &r.PackageType, &r.RepoType, &r.BaseURL, &r.Mirrorlist, &r.MirrorStrategy, &r.Description, &r.Username, &r.Password,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
|
||||
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable, &r.SyncInterval,
|
||||
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
|
||||
&r.BanTagsEnabled, &r.BanTags,
|
||||
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
|
||||
@@ -76,8 +76,8 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
quarantine_enabled, quarantine_days, stale_on_error,
|
||||
releases_remote, managed_by,
|
||||
upstream_dial_timeout, upstream_tls_timeout, upstream_response_header_timeout,
|
||||
mirror_strategy, serve_stale
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27)
|
||||
mirror_strategy, sync_interval, serve_stale
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25,$26,$27,$28)
|
||||
`,
|
||||
r.Name, r.PackageType, r.RepoType, r.BaseURL, r.Mirrorlist, r.Description, r.Username, r.Password,
|
||||
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
|
||||
@@ -86,7 +86,7 @@ func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
|
||||
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy), r.ServeStale,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy), r.SyncInterval, r.ServeStale,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -101,7 +101,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
quarantine_enabled=$17, quarantine_days=$18, stale_on_error=$19,
|
||||
releases_remote=$20, managed_by=$21,
|
||||
upstream_dial_timeout=$22, upstream_tls_timeout=$23, upstream_response_header_timeout=$24,
|
||||
mirror_strategy=$26, serve_stale=$27,
|
||||
mirror_strategy=$26, sync_interval=$27, serve_stale=$28,
|
||||
updated_at=NOW()
|
||||
WHERE name=$1
|
||||
`,
|
||||
@@ -113,7 +113,7 @@ func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
|
||||
r.ReleasesRemote, r.ManagedBy,
|
||||
r.UpstreamDialTimeout, r.UpstreamTLSTimeout, r.UpstreamResponseHeaderTimeout,
|
||||
r.Mirrorlist,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy), r.ServeStale,
|
||||
normalizeMirrorStrategy(r.MirrorStrategy), r.SyncInterval, r.ServeStale,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ const (
|
||||
type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
|
||||
}
|
||||
|
||||
@@ -129,7 +129,8 @@ func (s *Syncer) Run(ctx context.Context) {
|
||||
}
|
||||
|
||||
// schedule enqueues a periodic check for every github_alpine remote. The DB lease
|
||||
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
|
||||
// enforces the per-remote sync_interval (else mutable_ttl) cadence and
|
||||
// cross-replica coordination.
|
||||
func (s *Syncer) schedule(ctx context.Context) {
|
||||
remotes, err := s.store.ListGitHubAlpineRemotes(ctx)
|
||||
if err != nil {
|
||||
@@ -142,8 +143,10 @@ func (s *Syncer) schedule(ctx context.Context) {
|
||||
}
|
||||
|
||||
// EnqueuePrime queues an immediate background prime for a freshly created remote.
|
||||
// A sync_interval remote skips it and syncs in its first slot, so remotes
|
||||
// created together do not burst.
|
||||
func (s *Syncer) EnqueuePrime(remote models.Remote) {
|
||||
if s == nil {
|
||||
if s == nil || remote.SyncInterval > 0 {
|
||||
return
|
||||
}
|
||||
s.enqueue(remote, true)
|
||||
@@ -189,16 +192,16 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
ttl := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = defaultSyncFreshness
|
||||
}
|
||||
ttl := job.remote.SyncPeriod(defaultSyncFreshness)
|
||||
freshness := ttl
|
||||
var interval, phase time.Duration
|
||||
if job.prime {
|
||||
freshness = 0
|
||||
} else {
|
||||
interval, phase = job.remote.SyncSchedule()
|
||||
}
|
||||
|
||||
claimed, etag, err := s.store.ClaimGitHubAlpineSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
|
||||
claimed, etag, err := s.store.ClaimGitHubAlpineSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration, interval, phase)
|
||||
if err != nil {
|
||||
slog.Error("github_alpine syncer: claim lease", "remote", job.remote.Name, "error", err)
|
||||
return
|
||||
|
||||
@@ -28,6 +28,10 @@ type fakeSyncStore struct {
|
||||
lastSynced map[string]time.Time
|
||||
etags map[string]string
|
||||
retryAt map[string]time.Time
|
||||
freshness time.Duration
|
||||
interval time.Duration
|
||||
phase time.Duration
|
||||
attempts int
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
@@ -47,9 +51,11 @@ func (f *fakeSyncStore) ListGitHubAlpineRemotes(_ context.Context) ([]models.Rem
|
||||
return append([]models.Remote(nil), f.remotes...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.freshness, f.interval, f.phase = freshness, interval, phase
|
||||
f.attempts++
|
||||
now := time.Now()
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
@@ -223,7 +229,7 @@ func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
claimed, _, err := store.ClaimGitHubAlpineSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
|
||||
claimed, _, err := store.ClaimGitHubAlpineSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration, 0, 0)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
@@ -363,3 +369,37 @@ func TestSyncerFailedAssetRetry(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A periodic check claims with sync_interval as its freshness and passes the
|
||||
// remote's slot schedule; unset, it falls back to mutable_ttl with no schedule.
|
||||
func TestSyncerHonoursSyncInterval(t *testing.T) {
|
||||
for _, interval := range []int{0, 7200} {
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
store.lastSynced["sched"] = time.Now()
|
||||
remote := models.Remote{Name: "sched", MutableTTL: 90, SyncInterval: interval}
|
||||
wantInterval, wantPhase := remote.SyncSchedule()
|
||||
s.process(context.Background(), syncJob{remote: remote})
|
||||
if want := remote.SyncPeriod(0); store.freshness != want || store.interval != wantInterval || store.phase != wantPhase {
|
||||
t.Errorf("sync_interval=%d: claim freshness/interval/phase = %v/%v/%v, want %v/%v/%v",
|
||||
interval, store.freshness, store.interval, store.phase, want, wantInterval, wantPhase)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create-time primes are skipped for a scheduled remote so remotes created
|
||||
// together spread out; a cold-cache prime still claims with no schedule.
|
||||
func TestSyncerScheduledPrime(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
remote := models.Remote{Name: "fresh", SyncInterval: 3600}
|
||||
|
||||
s.EnqueuePrime(remote)
|
||||
if len(s.jobs) != 0 {
|
||||
t.Fatal("create-time prime enqueued for a sync_interval remote")
|
||||
}
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: true})
|
||||
if store.attempts != 1 || store.freshness != 0 || store.interval != 0 {
|
||||
t.Fatalf("cold-cache prime: attempts=%d freshness=%v interval=%v, want 1/0/0", store.attempts, store.freshness, store.interval)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ const (
|
||||
type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
|
||||
}
|
||||
|
||||
@@ -129,7 +129,8 @@ func (s *Syncer) Run(ctx context.Context) {
|
||||
}
|
||||
|
||||
// schedule enqueues a periodic check for every github_deb remote. The DB lease
|
||||
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
|
||||
// enforces the per-remote sync_interval (else mutable_ttl) cadence and
|
||||
// cross-replica coordination.
|
||||
func (s *Syncer) schedule(ctx context.Context) {
|
||||
remotes, err := s.store.ListGitHubDebRemotes(ctx)
|
||||
if err != nil {
|
||||
@@ -142,8 +143,10 @@ func (s *Syncer) schedule(ctx context.Context) {
|
||||
}
|
||||
|
||||
// EnqueuePrime queues an immediate background prime for a freshly created remote.
|
||||
// A sync_interval remote skips it and syncs in its first slot, so remotes
|
||||
// created together do not burst.
|
||||
func (s *Syncer) EnqueuePrime(remote models.Remote) {
|
||||
if s == nil {
|
||||
if s == nil || remote.SyncInterval > 0 {
|
||||
return
|
||||
}
|
||||
s.enqueue(remote, true)
|
||||
@@ -189,16 +192,16 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
ttl := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = defaultSyncFreshness
|
||||
}
|
||||
ttl := job.remote.SyncPeriod(defaultSyncFreshness)
|
||||
freshness := ttl
|
||||
var interval, phase time.Duration
|
||||
if job.prime {
|
||||
freshness = 0
|
||||
} else {
|
||||
interval, phase = job.remote.SyncSchedule()
|
||||
}
|
||||
|
||||
claimed, etag, err := s.store.ClaimGitHubDebSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
|
||||
claimed, etag, err := s.store.ClaimGitHubDebSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration, interval, phase)
|
||||
if err != nil {
|
||||
slog.Error("github_deb syncer: claim lease", "remote", job.remote.Name, "error", err)
|
||||
return
|
||||
|
||||
@@ -28,6 +28,10 @@ type fakeSyncStore struct {
|
||||
lastSynced map[string]time.Time
|
||||
etags map[string]string
|
||||
retryAt map[string]time.Time
|
||||
freshness time.Duration
|
||||
interval time.Duration
|
||||
phase time.Duration
|
||||
attempts int
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
@@ -47,9 +51,11 @@ func (f *fakeSyncStore) ListGitHubDebRemotes(_ context.Context) ([]models.Remote
|
||||
return append([]models.Remote(nil), f.remotes...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.freshness, f.interval, f.phase = freshness, interval, phase
|
||||
f.attempts++
|
||||
now := time.Now()
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
@@ -223,7 +229,7 @@ func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
claimed, _, err := store.ClaimGitHubDebSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
|
||||
claimed, _, err := store.ClaimGitHubDebSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration, 0, 0)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
@@ -363,3 +369,37 @@ func TestSyncerFailedAssetRetry(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A periodic check claims with sync_interval as its freshness and passes the
|
||||
// remote's slot schedule; unset, it falls back to mutable_ttl with no schedule.
|
||||
func TestSyncerHonoursSyncInterval(t *testing.T) {
|
||||
for _, interval := range []int{0, 7200} {
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
store.lastSynced["sched"] = time.Now()
|
||||
remote := models.Remote{Name: "sched", MutableTTL: 90, SyncInterval: interval}
|
||||
wantInterval, wantPhase := remote.SyncSchedule()
|
||||
s.process(context.Background(), syncJob{remote: remote})
|
||||
if want := remote.SyncPeriod(0); store.freshness != want || store.interval != wantInterval || store.phase != wantPhase {
|
||||
t.Errorf("sync_interval=%d: claim freshness/interval/phase = %v/%v/%v, want %v/%v/%v",
|
||||
interval, store.freshness, store.interval, store.phase, want, wantInterval, wantPhase)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create-time primes are skipped for a scheduled remote so remotes created
|
||||
// together spread out; a cold-cache prime still claims with no schedule.
|
||||
func TestSyncerScheduledPrime(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
remote := models.Remote{Name: "fresh", SyncInterval: 3600}
|
||||
|
||||
s.EnqueuePrime(remote)
|
||||
if len(s.jobs) != 0 {
|
||||
t.Fatal("create-time prime enqueued for a sync_interval remote")
|
||||
}
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: true})
|
||||
if store.attempts != 1 || store.freshness != 0 || store.interval != 0 {
|
||||
t.Fatalf("cold-cache prime: attempts=%d freshness=%v interval=%v, want 1/0/0", store.attempts, store.freshness, store.interval)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ const (
|
||||
// scan never loses its lease, while a crashed replica's lease still expires.
|
||||
syncLeaseDuration = 15 * time.Minute
|
||||
// defaultSyncFreshness is the periodic re-check interval used when a remote's
|
||||
// mutable_ttl is unset.
|
||||
// sync_interval and mutable_ttl are unset.
|
||||
defaultSyncFreshness = 5 * time.Minute
|
||||
// jobQueueDepth bounds the pending work queue; enqueues past it are dropped
|
||||
// (a later poll re-enqueues), never blocking the caller.
|
||||
@@ -34,7 +34,7 @@ const (
|
||||
type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease, interval, phase time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@ type SyncConfig struct {
|
||||
RatePerSec float64 // global GitHub request rate (req/s)
|
||||
Burst int // token-bucket burst
|
||||
Workers int // concurrent scan workers
|
||||
PollInterval time.Duration // base scheduler tick; per-remote cadence is mutable_ttl
|
||||
PollInterval time.Duration // base scheduler tick; per-remote cadence is sync_interval or mutable_ttl
|
||||
}
|
||||
|
||||
type syncJob struct {
|
||||
@@ -139,9 +139,9 @@ func (s *Syncer) Run(ctx context.Context) {
|
||||
}
|
||||
|
||||
// schedule enqueues a periodic check for every github_rpm remote. The DB lease
|
||||
// (claimed in the worker) enforces the per-remote mutable_ttl cadence and cross
|
||||
// replica coordination, so enqueuing every tick is cheap: a not-yet-due remote
|
||||
// simply fails to claim and is skipped.
|
||||
// (claimed in the worker) enforces the per-remote sync_interval (else
|
||||
// mutable_ttl) cadence and cross replica coordination, so enqueuing every tick
|
||||
// is cheap: a not-yet-due remote simply fails to claim and is skipped.
|
||||
func (s *Syncer) schedule(ctx context.Context) {
|
||||
remotes, err := s.store.ListGitHubRPMRemotes(ctx)
|
||||
if err != nil {
|
||||
@@ -155,8 +155,10 @@ func (s *Syncer) schedule(ctx context.Context) {
|
||||
|
||||
// EnqueuePrime queues an immediate background prime for a freshly created
|
||||
// remote so its metadata is derived without blocking the create call.
|
||||
// A sync_interval remote skips it and syncs in its first slot, so remotes
|
||||
// created together do not burst.
|
||||
func (s *Syncer) EnqueuePrime(remote models.Remote) {
|
||||
if s == nil {
|
||||
if s == nil || remote.SyncInterval > 0 {
|
||||
return
|
||||
}
|
||||
s.enqueue(remote, true)
|
||||
@@ -205,16 +207,16 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
ttl := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = defaultSyncFreshness
|
||||
}
|
||||
ttl := job.remote.SyncPeriod(defaultSyncFreshness)
|
||||
freshness := ttl
|
||||
var interval, phase time.Duration
|
||||
if job.prime {
|
||||
freshness = 0 // prime ignores the recency gate but still respects a live lease
|
||||
} else {
|
||||
interval, phase = job.remote.SyncSchedule()
|
||||
}
|
||||
|
||||
claimed, etag, err := s.store.ClaimGitHubSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
|
||||
claimed, etag, err := s.store.ClaimGitHubSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration, interval, phase)
|
||||
if err != nil {
|
||||
slog.Error("github_rpm syncer: claim lease", "remote", job.remote.Name, "error", err)
|
||||
return
|
||||
|
||||
@@ -29,6 +29,10 @@ type fakeSyncStore struct {
|
||||
etags map[string]string
|
||||
retryAt map[string]time.Time
|
||||
lastResult provider.SyncResult
|
||||
freshness time.Duration
|
||||
interval time.Duration
|
||||
phase time.Duration
|
||||
attempts int
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
@@ -48,9 +52,11 @@ func (f *fakeSyncStore) ListGitHubRPMRemotes(_ context.Context) ([]models.Remote
|
||||
return append([]models.Remote(nil), f.remotes...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner string, freshness, lease, interval, phase time.Duration) (bool, string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.freshness, f.interval, f.phase = freshness, interval, phase
|
||||
f.attempts++
|
||||
now := time.Now()
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
@@ -230,7 +236,7 @@ func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
|
||||
remote := fx.remote()
|
||||
|
||||
// Replica 1 holds the lease.
|
||||
claimed, _, err := store.ClaimGitHubSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
|
||||
claimed, _, err := store.ClaimGitHubSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration, 0, 0)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
@@ -470,3 +476,37 @@ func TestSyncerFailedAssetRetry(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A periodic check claims with sync_interval as its freshness and passes the
|
||||
// remote's slot schedule; unset, it falls back to mutable_ttl with no schedule.
|
||||
func TestSyncerHonoursSyncInterval(t *testing.T) {
|
||||
for _, interval := range []int{0, 7200} {
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
store.lastSynced["sched"] = time.Now()
|
||||
remote := models.Remote{Name: "sched", MutableTTL: 90, SyncInterval: interval}
|
||||
wantInterval, wantPhase := remote.SyncSchedule()
|
||||
s.process(context.Background(), syncJob{remote: remote})
|
||||
if want := remote.SyncPeriod(0); store.freshness != want || store.interval != wantInterval || store.phase != wantPhase {
|
||||
t.Errorf("sync_interval=%d: claim freshness/interval/phase = %v/%v/%v, want %v/%v/%v",
|
||||
interval, store.freshness, store.interval, store.phase, want, wantInterval, wantPhase)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create-time primes are skipped for a scheduled remote so remotes created
|
||||
// together spread out; a cold-cache prime still claims with no schedule.
|
||||
func TestSyncerScheduledPrime(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
remote := models.Remote{Name: "fresh", SyncInterval: 3600}
|
||||
|
||||
s.EnqueuePrime(remote)
|
||||
if len(s.jobs) != 0 {
|
||||
t.Fatal("create-time prime enqueued for a sync_interval remote")
|
||||
}
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: true})
|
||||
if store.attempts != 1 || store.freshness != 0 || store.interval != 0 {
|
||||
t.Fatalf("cold-cache prime: attempts=%d freshness=%v interval=%v, want 1/0/0", store.attempts, store.freshness, store.interval)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS sync_interval INTEGER NOT NULL DEFAULT 0;
|
||||
@@ -2,6 +2,7 @@ package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"time"
|
||||
@@ -55,6 +56,9 @@ type Remote struct {
|
||||
ImmutableTTL int `json:"immutable_ttl"`
|
||||
MutableTTL int `json:"mutable_ttl"`
|
||||
CheckMutable bool `json:"check_mutable"`
|
||||
// SyncInterval (seconds) sets how often a github_* remote rescans releases
|
||||
// and rebuilds its index; 0 falls back to mutable_ttl.
|
||||
SyncInterval int `json:"sync_interval,omitempty"`
|
||||
|
||||
// Upstream HTTP timeouts in seconds. 0 means use the server default.
|
||||
UpstreamDialTimeout int `json:"upstream_dial_timeout,omitempty"`
|
||||
@@ -152,6 +156,51 @@ func (r *Remote) ValidateMirrorStrategy() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SyncPeriod is the background sync cadence: sync_interval, else mutable_ttl,
|
||||
// else def.
|
||||
func (r Remote) SyncPeriod(def time.Duration) time.Duration {
|
||||
if r.SyncInterval > 0 {
|
||||
return time.Duration(r.SyncInterval) * time.Second
|
||||
}
|
||||
if r.MutableTTL > 0 {
|
||||
return time.Duration(r.MutableTTL) * time.Second
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// SyncSchedule is the sync_interval and the stable per-remote phase offsetting
|
||||
// its slots, so remotes sharing an interval spread across it; both zero when
|
||||
// sync_interval is unset.
|
||||
func (r Remote) SyncSchedule() (interval, phase time.Duration) {
|
||||
if r.SyncInterval <= 0 {
|
||||
return 0, 0
|
||||
}
|
||||
interval = time.Duration(r.SyncInterval) * time.Second
|
||||
return interval, syncPhase(r.Name, interval)
|
||||
}
|
||||
|
||||
func syncPhase(name string, period time.Duration) time.Duration {
|
||||
h := fnv.New64a()
|
||||
_, _ = h.Write([]byte(name))
|
||||
return time.Duration(h.Sum64() % uint64(period))
|
||||
}
|
||||
|
||||
// ValidateSyncInterval rejects a negative sync_interval, or one set on a type
|
||||
// without a background syncer.
|
||||
func (r *Remote) ValidateSyncInterval() error {
|
||||
if r.SyncInterval < 0 {
|
||||
return fmt.Errorf("sync_interval must be >= 0, got %d", r.SyncInterval)
|
||||
}
|
||||
switch r.PackageType {
|
||||
case PackageGitHubRPM, PackageGitHubDeb, PackageGitHubAlpine:
|
||||
default:
|
||||
if r.SyncInterval > 0 {
|
||||
return fmt.Errorf("sync_interval is only valid on github_rpm, github_deb and github_alpine remotes")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DefaultQuarantineDays matches the remotes.quarantine_days column default.
|
||||
const DefaultQuarantineDays = 3
|
||||
|
||||
|
||||
@@ -2,8 +2,10 @@ package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRemote_ValidatePatterns(t *testing.T) {
|
||||
@@ -123,3 +125,66 @@ func TestValidateMirrorlist(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncPeriod(t *testing.T) {
|
||||
const def = 5 * time.Minute
|
||||
for _, tc := range []struct {
|
||||
interval, ttl int
|
||||
want time.Duration
|
||||
}{
|
||||
{0, 0, def},
|
||||
{0, 600, 10 * time.Minute},
|
||||
{3600, 600, time.Hour},
|
||||
} {
|
||||
r := Remote{SyncInterval: tc.interval, MutableTTL: tc.ttl}
|
||||
if got := r.SyncPeriod(def); got != tc.want {
|
||||
t.Errorf("SyncPeriod(interval=%d, ttl=%d) = %v, want %v", tc.interval, tc.ttl, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSyncInterval(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
pt PackageType
|
||||
interval int
|
||||
ok bool
|
||||
}{
|
||||
{PackageGitHubRPM, 3600, true},
|
||||
{PackageGitHubDeb, 3600, true},
|
||||
{PackageGitHubAlpine, 3600, true},
|
||||
{PackageGitHubRPM, -1, false},
|
||||
{PackageRPM, 0, true},
|
||||
{PackageRPM, 60, false},
|
||||
} {
|
||||
r := Remote{PackageType: tc.pt, SyncInterval: tc.interval}
|
||||
if err := r.ValidateSyncInterval(); (err == nil) != tc.ok {
|
||||
t.Errorf("%s interval=%d: err=%v, want ok=%v", tc.pt, tc.interval, err, tc.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncSchedule(t *testing.T) {
|
||||
if i, p := (Remote{Name: "x", MutableTTL: 60}).SyncSchedule(); i != 0 || p != 0 {
|
||||
t.Fatalf("unset sync_interval schedule = %v/%v, want 0/0", i, p)
|
||||
}
|
||||
|
||||
const period = 5 * time.Minute
|
||||
phases := map[time.Duration]bool{}
|
||||
var lo, hi time.Duration = period, 0
|
||||
for i := 0; i < 13; i++ {
|
||||
r := Remote{Name: fmt.Sprintf("github-rpm-%02d", i), SyncInterval: int(period.Seconds())}
|
||||
interval, p := r.SyncSchedule()
|
||||
if interval != period || p < 0 || p >= period {
|
||||
t.Fatalf("%s: schedule %v/%v outside [0, %v)", r.Name, interval, p, period)
|
||||
}
|
||||
if _, again := r.SyncSchedule(); again != p {
|
||||
t.Fatalf("%s: phase not stable", r.Name)
|
||||
}
|
||||
p = p.Truncate(time.Second)
|
||||
phases[p] = true
|
||||
lo, hi = min(lo, p), max(hi, p)
|
||||
}
|
||||
if len(phases) != 13 || hi-lo < period/2 {
|
||||
t.Fatalf("phases bunched: %d distinct, spread %v of %v", len(phases), hi-lo, period)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ export interface Remote {
|
||||
immutable_ttl: number;
|
||||
mutable_ttl: number;
|
||||
check_mutable: boolean;
|
||||
sync_interval?: number;
|
||||
patterns: string[];
|
||||
blocklist: string[];
|
||||
mutable_patterns: string[];
|
||||
|
||||
Reference in New Issue
Block a user