Document sync_interval for deb/alpine and fix created_at gate on existing rows (#176)

`sync_interval` is documented only for github_rpm, and the claim gate holding back remotes created inside the current slot also blocks remotes that already have a sync state row: the gate filters the INSERT SELECT, so ON CONFLICT never runs and a failed cold-cache prime waits a full slot instead of its backoff.

- document `sync_interval` for github_deb/github_alpine and in the caching field table
- apply the created_at gate only when no sync state row exists
- test 0→N and N→0 interval changes and the insert-only gate against Postgres
- test that a negative `sync_interval` on update returns 400

Reviewed-on: #176
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

docs/alpine/remote-github.md

@@ -5,3 +5,16 @@ Shared classification, TTL and eviction rules: [caching](../common/caching.md).
 | Type | Mutable (auto-detected) | Immutable (auto-detected) |
 |---|---|---|
 | `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) |
+
+## Release sync
+
+A background syncer re-scans the GitHub releases list per remote, sharing one
+lease per remote across replicas through `github_alpine_sync_state`.
+
+- **`sync_interval`** (seconds) re-checks once per interval at a stable
+  per-remote offset, so remotes on the same interval spread out instead of
+  syncing together; its first scheduled sync waits for that slot (a cold-cache
+  request still primes at once). Unset, it re-checks no more often than its
+  `mutable_ttl`.
+- A failed scan keeps serving the last good index and retries with exponential
+  backoff.

docs/common/caching.md

@@ -13,6 +13,7 @@ Every proxied request on a remote is classified as **denied**, **immutable** or
 | `mutable_patterns` | Regexes forcing a path mutable. |
 | `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. |
 | `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires. Omitted on create, it defaults to `3600`; omitted on update, the stored value is kept. |
+| `sync_interval` | github_rpm/github_deb/github_alpine only. Seconds between background release re-scans, at a stable per-remote offset. Unset (`0`) re-scans no more often than `mutable_ttl`. |
 | `check_mutable` | When a mutable path's `mutable_ttl` expires, sends a conditional `HEAD` (`If-None-Match`) with the stored upstream ETag. A `304` re-arms the TTL and serves the stored copy; anything else refetches in full. The ETag is kept for `max(10 × mutable_ttl, 7 days)`, re-armed on each `304`, so idle paths age out. It is only stored when `check_mutable` is set and `mutable_ttl` is non-zero, and never for immutable paths. |
 | `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. |
 | `serve_stale` | rpm/deb/alpine virtual members only. When a virtual re-merges, use this member's previous index at once and refresh it in the background; the next merge picks up the refresh. The first merge still waits, and so does any merge once the previous index is over an hour old, so a member that keeps failing surfaces its error. Default `false`. |

docs/deb/remote-github.md

@@ -5,3 +5,16 @@ Shared classification, TTL and eviction rules: [caching](../common/caching.md).
 | Type | Mutable (auto-detected) | Immutable (auto-detected) |
 |---|---|---|
 | `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) |
+
+## Release sync
+
+A background syncer re-scans the GitHub releases list per remote, sharing one
+lease per remote across replicas through `github_deb_sync_state`.
+
+- **`sync_interval`** (seconds) re-checks once per interval at a stable
+  per-remote offset, so remotes on the same interval spread out instead of
+  syncing together; its first scheduled sync waits for that slot (a cold-cache
+  request still primes at once). Unset, it re-checks no more often than its
+  `mutable_ttl`.
+- A failed scan keeps serving the last good index and retries with exponential
+  backoff.

docs/rpm/remote-github.md

@@ -37,9 +37,9 @@ derived metadata current off the client request path:
   stable per-remote offset, so remotes on the same interval spread out instead
   of syncing together; its first scheduled sync waits for that slot (a
   cold-cache request still primes at once). Unset, it re-checks no more often
-  than its `mutable_ttl`. New/changed
-  assets are derived incrementally; assets already cached are never re-fetched,
-  and assets that disappear upstream are pruned.
+  than its `mutable_ttl`. New/changed assets are derived incrementally; assets
+  already cached are never re-fetched, and assets that disappear upstream are
+  pruned.
 - **ETag / 304 conditional requests.** The releases-list `ETag` is stored per
   remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed
   and the syncer derives nothing. GitHub does not count `304` conditional
@@ -54,8 +54,8 @@ derived metadata current off the client request path:
   periodic scan a replica must atomically claim a per-remote lease
   (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`,
   `sync_lease_expires`); only the winner scans. This bounds total GitHub load to
-  ~once per `sync_interval` (else `mutable_ttl`) regardless of replica count, and the shared `etag`
-  lets any replica issue the conditional request.
+  ~once per `sync_interval` (else `mutable_ttl`) regardless of replica count,
+  and the shared `etag` lets any replica issue the conditional request.
 
 ```hcl
 # Backend that serves the actual .rpm bytes from github.com.

internal/api/v2/remotes_ttl_test.go

@@ -101,6 +101,12 @@ func TestSyncIntervalAPI(t *testing.T) {
 			t.Errorf("%s = %d, want 400", body, c)
 		}
 	}
+	if c := do(t, h, "PUT", "/si-gh", `{`+gh+`,"sync_interval":-1}`); c != 400 {
+		t.Errorf("update negative = %d, want 400", c)
+	}
+	if got := interval("si-gh"); got != 3600 {
+		t.Errorf("rejected update changed sync_interval to %d", got)
+	}
 }
 
 func TestQuarantineDaysDefault(t *testing.T) {

internal/database/github_sync.go

@@ -50,7 +50,8 @@ func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner stri
 // next_retry_at, or, with no retry pending, the remote was never synced or was
 // last synced before the current slot (interval set) or longer than freshness
 // ago (otherwise). With interval set, slots start at phase past each multiple
-// of interval and a remote created inside the current slot waits for the next.
+// of interval and a remote created inside the current slot with no state row
+// waits for the next.
 // Slots are computed from the DB clock so replicas agree whatever their own
 // clocks say. A zero freshness and interval (prime scans) ignores the recency
 // gate but still honours a pending retry's backoff. The returned etag is the
@@ -63,7 +64,8 @@ func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner strin
 		SELECT r.name, $2, now() + make_interval(secs => $4)
 		  FROM remotes r
 		 WHERE r.name = $1
-		   AND ($5::float8 <= 0 OR COALESCE(r.created_at, '-infinity') < `+slot+`)
+		   AND ($5::float8 <= 0 OR COALESCE(r.created_at, '-infinity') < `+slot+`
+		        OR EXISTS (SELECT 1 FROM `+table+` e WHERE e.remote_name = r.name))
 		ON CONFLICT (remote_name) DO UPDATE
 			SET sync_lease_owner   = $2,
 			    sync_lease_expires = now() + make_interval(secs => $4)

internal/database/github_sync_test.go

@@ -400,3 +400,90 @@ func TestSyncLeaseSlot(t *testing.T) {
 		})
 	}
 }
+
+// slotPhase returns the phase that puts the current interval slot's start
+// startAgo before the DB clock.
+func slotPhase(t *testing.T, interval, startAgo time.Duration) time.Duration {
+	t.Helper()
+	var epoch float64
+	if err := testDB.Pool.QueryRow(ctx(), `SELECT extract(epoch FROM now())::float8`).Scan(&epoch); err != nil {
+		t.Fatalf("db now: %v", err)
+	}
+	return time.Duration(math.Mod(epoch-startAgo.Seconds(), interval.Seconds()) * float64(time.Second))
+}
+
+func setSyncedAgo(t *testing.T, tc syncTable, name string, ago time.Duration) {
+	t.Helper()
+	if _, err := testDB.Pool.Exec(ctx(), `UPDATE `+tc.table+` SET last_synced_at = now() - make_interval(secs => $2), sync_lease_owner = '', sync_lease_expires = NULL WHERE remote_name = $1`, name, ago.Seconds()); err != nil {
+		t.Fatalf("set last_synced_at: %v", err)
+	}
+}
+
+// Toggling sync_interval on a remote with sync state neither re-syncs at once
+// nor leaves it waiting longer than one interval (or freshness) since its last
+// sync. Elapsed time is simulated by moving last_synced_at and the slot start
+// back together, relative to the DB clock.
+func TestSyncLeaseIntervalChange(t *testing.T) {
+	requireDB(t)
+	const (
+		interval  = time.Hour
+		freshness = time.Hour
+		day       = 24 * time.Hour
+	)
+	for _, tc := range syncTables() {
+		t.Run(tc.table, func(t *testing.T) {
+			t.Run("0 to N", func(t *testing.T) {
+				name := seedSynced(t, tc, day, 25*time.Minute)
+				if mustClaim(t, tc, name, "r1", freshness, 0, 0) {
+					t.Fatal("claimed inside freshness before the change")
+				}
+				if mustClaim(t, tc, name, "r1", freshness, interval, slotPhase(t, interval, 30*time.Minute)) {
+					t.Fatal("synced inside the current slot, yet claimed right after the change")
+				}
+				// 30m later the next slot opens; the last sync is 55m old.
+				setSyncedAgo(t, tc, name, 55*time.Minute)
+				if !mustClaim(t, tc, name, "r1", freshness, interval, slotPhase(t, interval, time.Second)) {
+					t.Fatal("not due at the first slot after the last sync")
+				}
+			})
+			t.Run("N to 0", func(t *testing.T) {
+				name := seedSynced(t, tc, day, 10*time.Minute)
+				if mustClaim(t, tc, name, "r1", freshness, interval, slotPhase(t, interval, 30*time.Minute)) {
+					t.Fatal("claimed inside the slot before the change")
+				}
+				if mustClaim(t, tc, name, "r1", freshness, 0, 0) {
+					t.Fatal("synced inside freshness, yet claimed right after the change")
+				}
+				setSyncedAgo(t, tc, name, freshness+time.Second)
+				if !mustClaim(t, tc, name, "r1", freshness, 0, 0) {
+					t.Fatal("not due once freshness elapsed after the change")
+				}
+			})
+		})
+	}
+}
+
+// The created_at gate only holds back the first insert of a remote created
+// inside the current slot; once a state row exists the row's own sync and
+// retry state decide.
+func TestSyncLeaseCreatedGateInsertOnly(t *testing.T) {
+	requireDB(t)
+	const interval = time.Hour
+	phase := slotPhase(t, interval, 30*time.Minute)
+	for _, tc := range syncTables() {
+		t.Run(tc.table, func(t *testing.T) {
+			fresh := seedSynced(t, tc, 10*time.Second, neverSynced)
+			if mustClaim(t, tc, fresh, "r1", 0, interval, phase) {
+				t.Fatal("remote created inside the slot claimed with no state row")
+			}
+
+			name := seedSynced(t, tc, 10*time.Second, neverSynced)
+			if _, err := testDB.Pool.Exec(ctx(), `INSERT INTO `+tc.table+` (remote_name, sync_failures, next_retry_at) VALUES ($1, 1, now() - interval '1 second')`, name); err != nil {
+				t.Fatalf("seed failed prime: %v", err)
+			}
+			if !mustClaim(t, tc, name, "r1", 0, interval, phase) {
+				t.Fatal("created_at gate blocked a remote with a state row whose retry is due")
+			}
+		})
+	}
+}