Treat a clean EOF with unknown total as the whole asset

Wrap package parse errors in provider.ErrInvalidPackage so only transient
asset and insert errors fail the scan; cover rpm, deb and alpine.

.woodpecker/docker.yaml

@@ -4,31 +4,45 @@ when:
 
 steps:
   - name: docker-api
-    image: woodpeckerci/plugin-docker-buildx
+    image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
     settings:
-      registry: git.unkin.net
-      repo: git.unkin.net/unkin/artifactapi
+      registry: artifactapi.k8s.syd1.au.unkin.net
+      repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi
       build_args:
         VERSION: ${CI_COMMIT_TAG}
-      username: droneci
-      password:
-        from_secret: DRONECI_PASSWORD
       tags:
         - ${CI_COMMIT_TAG}
         - latest
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1
+          limits:
+            memory: 4Gi
+            cpu: 2
 
   - name: docker-web
-    image: woodpeckerci/plugin-docker-buildx
+    image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
     settings:
-      registry: git.unkin.net
-      repo: git.unkin.net/unkin/artifactapi-ui
+      registry: artifactapi.k8s.syd1.au.unkin.net
+      repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui
       dockerfile: ui/Dockerfile.ui
       context: ui
       build_args:
         BASE_PATH: /ui
-      username: droneci
-      password:
-        from_secret: DRONECI_PASSWORD
       tags:
         - ${CI_COMMIT_TAG}
         - latest
+    backend_options:
+      kubernetes:
+        serviceAccountName: default
+        resources:
+          requests:
+            memory: 1Gi
+            cpu: 1
+          limits:
+            memory: 4Gi
+            cpu: 2

internal/database/alpine_github_sync.go

@@ -2,11 +2,9 @@ package database
 
 import (
 	"context"
-	"errors"
 	"time"
 
-	"github.com/jackc/pgx/v5"
-
+	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -30,42 +28,14 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
 	return remotes, rows.Err()
 }
 
-// ClaimGitHubAlpineSyncLease atomically claims the per-remote sync lease. It
-// succeeds only when the remote is due (never synced, or synced longer than
-// freshness ago) and no live lease is held by another replica. A zero freshness
-// (prime scans) ignores the recency gate. The returned etag is the stored
-// releases-list ETag, shared across replicas.
+// ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
+// lease. See claimSyncLease.
 func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
-	row := db.Pool.QueryRow(ctx, `
-		INSERT INTO github_alpine_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
-		VALUES ($1, $2, now() + make_interval(secs => $4))
-		ON CONFLICT (remote_name) DO UPDATE
-			SET sync_lease_owner   = $2,
-			    sync_lease_expires = now() + make_interval(secs => $4)
-			WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
-			  AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
-		RETURNING s.etag
-	`, remoteName, owner, freshness.Seconds(), lease.Seconds())
-
-	var etag string
-	if err := row.Scan(&etag); err != nil {
-		if errors.Is(err, pgx.ErrNoRows) {
-			return false, "", nil
-		}
-		return false, "", err
-	}
-	return true, etag, nil
+	return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
 }
 
-// ReleaseGitHubAlpineSyncLease records the completed scan and frees the lease.
-// Only the owning replica may release; last_synced_at advances so the next poll
-// waits a full freshness window, and etag is persisted for the next conditional
-// request.
-func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
-	_, err := db.Pool.Exec(ctx, `
-		UPDATE github_alpine_sync_state
-		   SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
-		 WHERE remote_name = $1 AND sync_lease_owner = $2
-	`, remoteName, owner, syncedAt, etag)
-	return err
+// ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the
+// lease. See releaseSyncLease.
+func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
+	return db.releaseSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, res)
 }

internal/database/deb_github_sync.go

@@ -2,11 +2,9 @@ package database
 
 import (
 	"context"
-	"errors"
 	"time"
 
-	"github.com/jackc/pgx/v5"
-
+	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -30,41 +28,14 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
 	return remotes, rows.Err()
 }
 
-// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
-// succeeds only when the remote is due (never synced, or synced longer than
-// freshness ago) and no live lease is held by another replica. A zero freshness
-// (prime scans) ignores the recency gate. The returned etag is the stored
-// releases-list ETag, shared across replicas.
+// ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
+// lease. See claimSyncLease.
 func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
-	row := db.Pool.QueryRow(ctx, `
-		INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
-		VALUES ($1, $2, now() + make_interval(secs => $4))
-		ON CONFLICT (remote_name) DO UPDATE
-			SET sync_lease_owner   = $2,
-			    sync_lease_expires = now() + make_interval(secs => $4)
-			WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
-			  AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
-		RETURNING s.etag
-	`, remoteName, owner, freshness.Seconds(), lease.Seconds())
-
-	var etag string
-	if err := row.Scan(&etag); err != nil {
-		if errors.Is(err, pgx.ErrNoRows) {
-			return false, "", nil
-		}
-		return false, "", err
-	}
-	return true, etag, nil
+	return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
 }
 
-// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
-// the owning replica may release; last_synced_at advances so the next poll waits
-// a full freshness window, and etag is persisted for the next conditional request.
-func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
-	_, err := db.Pool.Exec(ctx, `
-		UPDATE github_deb_sync_state
-		   SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
-		 WHERE remote_name = $1 AND sync_lease_owner = $2
-	`, remoteName, owner, syncedAt, etag)
-	return err
+// ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the
+// lease. See releaseSyncLease.
+func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
+	return db.releaseSyncLease(ctx, "github_deb_sync_state", remoteName, owner, res)
 }

internal/database/deb_github_sync_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -44,7 +45,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
 		t.Fatal("replica-2 claimed while replica-1 holds the lease")
 	}
 
-	if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
+	if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
 		t.Fatalf("release: %v", err)
 	}
 

internal/database/github_sync.go

@@ -7,6 +7,8 @@ import (
 
 	"github.com/jackc/pgx/v5"
 
+	"git.unkin.net/unkin/artifactapi/internal/provider"
+
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -30,22 +32,35 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
 	return remotes, rows.Err()
 }
 
-// ClaimGitHubSyncLease atomically claims the per-remote sync lease. It succeeds
-// (claimed=true) only when the remote is due — never synced, or synced longer
-// than freshness ago — and no live lease is held by another replica. This bounds
-// total GitHub load to roughly one scan per freshness window regardless of how
-// many replicas poll. The returned etag is the stored releases-list ETag, shared
-// across replicas so a conditional request can short-circuit an unchanged repo.
-// A zero freshness (used for prime scans) ignores the recency gate and claims
-// whenever no live lease is held.
+// ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
+// See claimSyncLease.
 func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
+	return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
+}
+
+// ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
+// See releaseSyncLease.
+func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
+	return db.releaseSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, res)
+}
+
+// claimSyncLease atomically claims a per-remote sync lease in table. It succeeds
+// (claimed=true) only when the remote is due and no live lease is held by
+// another replica. Due means: a pending retry after a failed scan has reached
+// next_retry_at, or, with no retry pending, the remote was never synced or was
+// synced longer than freshness ago. A zero freshness (prime scans) ignores the
+// recency gate but still honours a pending retry's backoff. The returned etag is
+// the stored releases-list ETag, shared across replicas so a conditional request
+// can short-circuit an unchanged repo.
+func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
 	row := db.Pool.QueryRow(ctx, `
-		INSERT INTO github_rpm_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
+		INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
 		VALUES ($1, $2, now() + make_interval(secs => $4))
 		ON CONFLICT (remote_name) DO UPDATE
 			SET sync_lease_owner   = $2,
 			    sync_lease_expires = now() + make_interval(secs => $4)
-			WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
+			WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
+			            ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
 			  AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
 		RETURNING s.etag
 	`, remoteName, owner, freshness.Seconds(), lease.Seconds())
@@ -60,14 +75,27 @@ func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string
 	return true, etag, nil
 }
 
-// ReleaseGitHubSyncLease records the completed scan and frees the lease. Only the
-// owning replica may release; last_synced_at advances so the next poll waits a
-// full freshness window, and etag is persisted for the next conditional request.
-func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
+// releaseSyncLease records a scan outcome and frees the lease; only the owning
+// replica may release. Success advances last_synced_at, persists the etag and
+// clears any retry. Failure leaves last_synced_at and etag untouched (the last
+// good metadata keeps serving) and schedules next_retry_at with exponential
+// backoff from the consecutive-failure count, never before res.RetryAt.
+func (db *DB) releaseSyncLease(ctx context.Context, table, remoteName, owner string, res provider.SyncResult) error {
+	var retryAt *time.Time
+	if !res.RetryAt.IsZero() {
+		retryAt = &res.RetryAt
+	}
 	_, err := db.Pool.Exec(ctx, `
-		UPDATE github_rpm_sync_state
-		   SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
+		UPDATE `+table+`
+		   SET last_synced_at = CASE WHEN $3 THEN last_synced_at ELSE now() END,
+		       etag           = CASE WHEN $3 THEN etag ELSE $4 END,
+		       sync_failures  = CASE WHEN $3 THEN sync_failures + 1 ELSE 0 END,
+		       next_retry_at  = CASE WHEN $3 THEN GREATEST(
+		                            now() + make_interval(secs => LEAST($5 * power(2, LEAST(sync_failures, 20)), $6)),
+		                            $7::timestamptz)
+		                        END,
+		       sync_lease_owner = '', sync_lease_expires = NULL
 		 WHERE remote_name = $1 AND sync_lease_owner = $2
-	`, remoteName, owner, syncedAt, etag)
+	`, remoteName, owner, res.Failed, res.Etag, res.Backoff.Seconds(), res.MaxBackoff.Seconds(), retryAt)
 	return err
 }

internal/database/github_sync_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 	"time"
 
+	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )
 
@@ -47,7 +48,7 @@ func TestGitHubSyncLease(t *testing.T) {
 	}
 
 	// Replica 1 finishes: record the sync and persist an etag.
-	if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
+	if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
 		t.Fatalf("release: %v", err)
 	}
 
@@ -93,3 +94,185 @@ func TestListGitHubRPMRemotes(t *testing.T) {
 		t.Fatalf("seeded remote %q not returned", name)
 	}
 }
+
+// A failed scan must not count as a sync: the etag and last_synced_at stay put,
+// every claim (prime included) is held off until the backoff elapses, the delay
+// doubles per consecutive failure up to the cap, an upstream retry hint pushes
+// it later, and a success clears the retry state.
+func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
+	requireDB(t)
+	name := "gh-retry-" + time.Now().Format("150405.000000")
+	seedGitHubRPMRemote(t, name)
+
+	const lease = 15 * time.Minute
+	freshness := time.Hour
+	claim := func(f time.Duration) (bool, string) {
+		t.Helper()
+		ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
+		if err != nil {
+			t.Fatalf("claim: %v", err)
+		}
+		return ok, etag
+	}
+	release := func(res provider.SyncResult) {
+		t.Helper()
+		if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "r1", res); err != nil {
+			t.Fatalf("release: %v", err)
+		}
+	}
+	state := func() (failures int, retryIn time.Duration, synced *time.Time, etag string) {
+		t.Helper()
+		var next *time.Time
+		var dbNow time.Time
+		if err := testDB.Pool.QueryRow(ctx(), `SELECT sync_failures, next_retry_at, last_synced_at, etag, now() FROM github_rpm_sync_state WHERE remote_name = $1`, name).
+			Scan(&failures, &next, &synced, &etag, &dbNow); err != nil {
+			t.Fatalf("read state: %v", err)
+		}
+		if next != nil {
+			retryIn = next.Sub(dbNow)
+		}
+		return
+	}
+	elapse := func() {
+		t.Helper()
+		if _, err := testDB.Pool.Exec(ctx(), `UPDATE github_rpm_sync_state SET next_retry_at = now() - interval '1 second' WHERE remote_name = $1`, name); err != nil {
+			t.Fatalf("elapse backoff: %v", err)
+		}
+	}
+	failed := provider.SyncResult{Failed: true, Etag: `"ignored"`, Backoff: time.Minute, MaxBackoff: 3 * time.Minute}
+
+	// Last good sync.
+	if ok, _ := claim(freshness); !ok {
+		t.Fatal("first claim failed")
+	}
+	release(provider.SyncResult{Etag: `"good"`})
+	_, _, goodSynced, _ := state()
+
+	// First failure: 60s backoff, sync time and etag untouched.
+	if ok, _ := claim(0); !ok {
+		t.Fatal("prime claim failed")
+	}
+	release(failed)
+	n, in, synced, etag := state()
+	if n != 1 || in < 55*time.Second || in > 65*time.Second {
+		t.Fatalf("after 1st failure: failures=%d retry in %v, want 1 and ~60s", n, in)
+	}
+	if etag != `"good"` || synced == nil || !synced.Equal(*goodSynced) {
+		t.Fatalf("failed scan overwrote sync state: etag=%q synced=%v", etag, synced)
+	}
+	if ok, _ := claim(0); ok {
+		t.Fatal("prime claimed inside the retry backoff")
+	}
+
+	// Backoff elapsed: claimable despite being inside the freshness window,
+	// and the second failure doubles the delay.
+	elapse()
+	if ok, _ := claim(freshness); !ok {
+		t.Fatal("periodic claim refused after backoff elapsed")
+	}
+	release(failed)
+	if n, in, _, _ := state(); n != 2 || in < 115*time.Second || in > 125*time.Second {
+		t.Fatalf("after 2nd failure: failures=%d retry in %v, want 2 and ~120s", n, in)
+	}
+
+	// Third failure caps at MaxBackoff (3m, not 4m).
+	elapse()
+	claim(freshness)
+	release(failed)
+	if n, in, _, _ := state(); n != 3 || in < 175*time.Second || in > 185*time.Second {
+		t.Fatalf("after 3rd failure: failures=%d retry in %v, want 3 and ~180s", n, in)
+	}
+
+	// An upstream rate-limit reset later than the backoff wins.
+	elapse()
+	claim(freshness)
+	hinted := failed
+	hinted.RetryAt = time.Now().Add(20 * time.Minute)
+	release(hinted)
+	if _, in, _, _ := state(); in < 19*time.Minute || in > 21*time.Minute {
+		t.Fatalf("retry hint ignored: retry in %v, want ~20m", in)
+	}
+
+	// Recovery: success persists the new etag and clears the retry, so the
+	// normal freshness window applies again.
+	elapse()
+	if ok, etag := claim(freshness); !ok || etag != `"good"` {
+		t.Fatalf("recovery claim ok=%v etag=%q, want last good etag", ok, etag)
+	}
+	release(provider.SyncResult{Etag: `"new"`})
+	if n, in, _, etag := state(); n != 0 || in != 0 || etag != `"new"` {
+		t.Fatalf("after success: failures=%d retry in %v etag=%q", n, in, etag)
+	}
+	if ok, _ := claim(freshness); ok {
+		t.Fatal("claimed inside freshness window after a successful sync")
+	}
+}
+
+// Deleting a remote drops its sync state, so a recreated remote of the same name
+// starts clean instead of inheriting the old ETag, sync time and retry state.
+func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
+	requireDB(t)
+	for _, tc := range []struct {
+		table   string
+		pt      models.PackageType
+		claim   func(name string) (bool, string, error)
+		release func(name string, res provider.SyncResult) error
+	}{
+		{"github_rpm_sync_state", models.PackageGitHubRPM,
+			func(n string) (bool, string, error) {
+				return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+			},
+			func(n string, res provider.SyncResult) error {
+				return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
+			}},
+		{"github_deb_sync_state", models.PackageGitHubDeb,
+			func(n string) (bool, string, error) {
+				return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+			},
+			func(n string, res provider.SyncResult) error {
+				return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
+			}},
+		{"github_alpine_sync_state", models.PackageGitHubAlpine,
+			func(n string) (bool, string, error) {
+				return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+			},
+			func(n string, res provider.SyncResult) error {
+				return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
+			}},
+	} {
+		t.Run(tc.table, func(t *testing.T) {
+			name := "gh-recreate-" + time.Now().Format("150405.000000")
+			create := func() {
+				t.Helper()
+				if err := testDB.CreateRemote(ctx(), &models.Remote{
+					Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
+					BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
+				}); err != nil {
+					t.Fatalf("create remote: %v", err)
+				}
+			}
+
+			create()
+			if ok, _, err := tc.claim(name); err != nil || !ok {
+				t.Fatalf("claim: ok=%v err=%v", ok, err)
+			}
+			if err := tc.release(name, provider.SyncResult{Etag: `"old"`}); err != nil {
+				t.Fatalf("release: %v", err)
+			}
+
+			if err := testDB.DeleteRemote(ctx(), name); err != nil {
+				t.Fatalf("delete remote: %v", err)
+			}
+			var rows int
+			if err := testDB.Pool.QueryRow(ctx(), `SELECT count(*) FROM `+tc.table+` WHERE remote_name = $1`, name).Scan(&rows); err != nil || rows != 0 {
+				t.Fatalf("sync state left after delete: rows=%d err=%v", rows, err)
+			}
+
+			create()
+			ok, etag, err := tc.claim(name)
+			if err != nil || !ok || etag != "" {
+				t.Fatalf("recreated remote claim ok=%v etag=%q err=%v, want a fresh claim with no etag", ok, etag, err)
+			}
+		})
+	}
+}

internal/provider/alpine/github.go

@@ -372,6 +372,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
+	// A transiently failed asset fails the scan so the ETag is not advanced and
+	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
+	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -400,10 +403,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				if !errors.Is(err, provider.ErrInvalidPackage) {
+					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
+				}
 				continue
 			}
 			if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
 				slog.Error("github_alpine: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_alpine: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -415,7 +422,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, nil
+	return newEtag, true, errors.Join(failed...)
 }
 
 type ghRelease struct {
@@ -470,7 +477,7 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote
 			return nil, "", false, err
 		}
 		if resp.StatusCode != http.StatusOK {
-			return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
+			return nil, "", false, provider.NewUpstreamStatusError("github releases API "+u, resp)
 		}
 		if page == 1 {
 			newEtag = respEtag
@@ -496,7 +503,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
 		return nil, err
 	}
 	if meta.Name == "" || meta.Arch == "" {
-		return nil, errors.New(".PKGINFO missing pkgname/arch")
+		return nil, fmt.Errorf("%w: .PKGINFO missing pkgname/arch", provider.ErrInvalidPackage)
 	}
 
 	meta.RepoName = remote.Name
@@ -526,13 +533,13 @@ func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote,
 		}
 		meta, complete, perr := pkginfoFromPrefix(body)
 		if perr != nil {
-			return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
+			return nil, fmt.Errorf("%w: parse apk .PKGINFO: %w", provider.ErrInvalidPackage, perr)
 		}
 		if complete {
 			return meta, nil
 		}
 		if full || n >= p.headerMax {
-			return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
+			return nil, fmt.Errorf("%w: .PKGINFO not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
 		}
 		n *= 2
 		if n > p.headerMax {
@@ -594,7 +601,7 @@ func pkginfoFromPrefix(prefix []byte) (meta *provider.AlpineMetadata, complete b
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response body was shorter than n (i.e. we already have the whole object).
+// response holds the whole object.
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -616,15 +623,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
+		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	full := int64(len(body)) < n
-	return body, full, nil
+	return provider.ReadPrefix(resp, n)
 }
 
 // assetPath is the package's location relative to github.com — the path the

internal/provider/alpine/github_test.go

@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -88,15 +89,17 @@ type githubFixture struct {
 	notModHit   int
 	releaseAuth string
 	assetAuth   string
+	assetFail   map[string]int // asset filename -> downloads left to answer 500
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		apkBytes: map[string][]byte{},
-		rangeHit: map[string]int{},
-		fullHit:  map[string]int{},
+		apkBytes:  map[string][]byte{},
+		rangeHit:  map[string]int{},
+		fullHit:   map[string]int{},
+		assetFail: map[string]int{},
 	}
 	f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
 
@@ -146,6 +149,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
+		if f.assetFail[name] > 0 {
+			f.assetFail[name]--
+			f.mu.Unlock()
+			http.Error(w, "boom", http.StatusInternalServerError)
+			return
+		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -495,3 +504,13 @@ func readAPKIndex(t *testing.T, gzBytes []byte) string {
 		}
 	}
 }
+
+func TestGitHubDeriveAssetMissingPkgnameIsInvalid(t *testing.T) {
+	fx := newGitHubFixture(t, true)
+	fx.apkBytes["nameless-1-r0.apk"] = testsupport.MinimalApk("", "1-r0", "x86_64")
+	asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2.3/nameless-1-r0.apk"}
+	_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
+	if !errors.Is(err, provider.ErrInvalidPackage) {
+		t.Fatalf("want ErrInvalidPackage, got %v", err)
+	}
+}

internal/provider/alpine/syncer.go

@@ -28,7 +28,7 @@ type SyncStore interface {
 	provider.RemoteMetadataStore
 	ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error)
 	ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
-	ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
+	ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
 }
 
 // SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
@@ -189,10 +189,11 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
 		s.mu.Unlock()
 	}()
 
-	freshness := time.Duration(job.remote.MutableTTL) * time.Second
-	if freshness <= 0 {
-		freshness = defaultSyncFreshness
+	ttl := time.Duration(job.remote.MutableTTL) * time.Second
+	if ttl <= 0 {
+		ttl = defaultSyncFreshness
 	}
+	freshness := ttl
 	if job.prime {
 		freshness = 0
 	}
@@ -210,16 +211,13 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
 	defer cancel()
 
 	newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
-	releaseEtag := etag
-	if scanErr == nil {
-		releaseEtag = newEtag
-	} else {
+	if scanErr != nil {
 		slog.Error("github_alpine syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
 	}
 
 	relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 	defer relCancel()
-	if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
+	if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, provider.NewSyncResult(newEtag, scanErr, ttl)); err != nil {
 		slog.Warn("github_alpine syncer: release lease", "remote", job.remote.Name, "error", err)
 	}
 

internal/provider/alpine/syncer_test.go

@@ -27,6 +27,7 @@ type fakeSyncStore struct {
 	leaseExp   map[string]time.Time
 	lastSynced map[string]time.Time
 	etags      map[string]string
+	retryAt    map[string]time.Time
 }
 
 func newFakeSyncStore() *fakeSyncStore {
@@ -36,6 +37,7 @@ func newFakeSyncStore() *fakeSyncStore {
 		leaseExp:   map[string]time.Time{},
 		lastSynced: map[string]time.Time{},
 		etags:      map[string]string{},
+		retryAt:    map[string]time.Time{},
 	}
 }
 
@@ -52,6 +54,9 @@ func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owne
 	ls, hasLS := f.lastSynced[name]
 	exp, hasExp := f.leaseExp[name]
 	freshOK := !hasLS || now.Sub(ls) >= freshness
+	if ra, pending := f.retryAt[name]; pending {
+		freshOK = !now.Before(ra)
+	}
 	leaseOK := !hasExp || exp.Before(now)
 	if freshOK && leaseOK {
 		f.leaseOwner[name] = owner
@@ -61,14 +66,19 @@ func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owne
 	return false, "", nil
 }
 
-func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
+func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner string, res provider.SyncResult) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	if f.leaseOwner[name] != owner {
 		return nil
 	}
-	f.lastSynced[name] = syncedAt
-	f.etags[name] = etag
+	if res.Failed {
+		f.retryAt[name] = time.Now().Add(res.Backoff)
+	} else {
+		f.lastSynced[name] = time.Now()
+		f.etags[name] = res.Etag
+		delete(f.retryAt, name)
+	}
 	delete(f.leaseOwner, name)
 	delete(f.leaseExp, name)
 	return nil
@@ -298,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
 		t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
 	}
 }
+
+// A transiently failed asset fails the scan: the ETag is not advanced, a retry
+// is scheduled, and the retry against the unchanged release list derives it. An
+// invalid package is skipped instead, since no retry can fix it.
+func TestSyncerFailedAssetRetry(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		corrupt   bool
+		wantRetry bool
+	}{
+		{name: "transient", wantRetry: true},
+		{name: "invalid package", corrupt: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			fx := newGitHubFixture(t, true)
+			fx.etag = `"v1"`
+			if tc.corrupt {
+				fx.apkBytes["other-9-r0.apk"] = []byte("not a package")
+			} else {
+				fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
+				fx.assetFail["other-9-r0.apk"] = 1
+			}
+			store := newFakeSyncStore()
+			s := newSyncer(store, newTestProvider(), testSyncConfig())
+			remote := fx.remote()
+			bg := context.Background()
+
+			s.process(bg, syncJob{remote: remote, prime: true})
+			_, pending := store.retryAt[remote.Name]
+			if pending != tc.wantRetry {
+				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
+			}
+			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
+				t.Fatalf("etag saved=%v after first scan", saved)
+			}
+			if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 1 {
+				t.Fatalf("healthy asset not served: %d rows", len(rows))
+			}
+			if !tc.wantRetry {
+				return
+			}
+
+			store.mu.Lock()
+			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
+			store.mu.Unlock()
+			s.process(bg, syncJob{remote: remote})
+			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
+				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
+			}
+			if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 2 {
+				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
+			}
+		})
+	}
+}

internal/provider/deb/github.go

@@ -333,6 +333,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
+	// A transiently failed asset fails the scan so the ETag is not advanced and
+	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
+	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -361,10 +364,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_deb: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				if !errors.Is(err, provider.ErrInvalidPackage) {
+					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
+				}
 				continue
 			}
 			if err := store.InsertDebMetadata(ctx, meta); err != nil {
 				slog.Error("github_deb: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_deb: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
@@ -376,7 +383,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, nil
+	return newEtag, true, errors.Join(failed...)
 }
 
 type ghRelease struct {
@@ -431,7 +438,7 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote
 			return nil, "", false, err
 		}
 		if resp.StatusCode != http.StatusOK {
-			return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
+			return nil, "", false, provider.NewUpstreamStatusError("github releases API "+u, resp)
 		}
 		if page == 1 {
 			newEtag = respEtag
@@ -468,7 +475,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
 		Size:         asset.Size,
 	}
 	if meta.Name == "" {
-		return nil, errors.New("control missing Package field")
+		return nil, fmt.Errorf("%w: control missing Package field", provider.ErrInvalidPackage)
 	}
 
 	// The Packages SHA256 must be the sha256 of the whole .deb. Prefer GitHub's
@@ -503,13 +510,13 @@ func (p *GitHubProvider) fetchControl(ctx context.Context, remote models.Remote,
 		}
 		control, complete, perr := controlFromPrefix(body)
 		if perr != nil {
-			return "", fmt.Errorf("parse deb control: %w", perr)
+			return "", fmt.Errorf("%w: parse deb control: %w", provider.ErrInvalidPackage, perr)
 		}
 		if complete {
 			return control, nil
 		}
 		if full || n >= p.headerMax {
-			return "", fmt.Errorf("control.tar not found within %d bytes of %s", n, downloadURL)
+			return "", fmt.Errorf("%w: control.tar not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
 		}
 		n *= 2
 		if n > p.headerMax {
@@ -567,7 +574,7 @@ func controlFromPrefix(prefix []byte) (control string, complete bool, err error)
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response body was shorter than n (i.e. we already have the whole object).
+// response holds the whole object.
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -589,15 +596,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
+		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	full := int64(len(body)) < n
-	return body, full, nil
+	return provider.ReadPrefix(resp, n)
 }
 
 func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -620,7 +621,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
+		return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
 
 	h := sha256.New()

internal/provider/deb/github_test.go

@@ -7,6 +7,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -81,15 +82,17 @@ type githubFixture struct {
 	notModHit   int
 	releaseAuth string
 	assetAuth   string
+	assetFail   map[string]int // asset filename -> downloads left to answer 500
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		debBytes: map[string][]byte{},
-		rangeHit: map[string]int{},
-		fullHit:  map[string]int{},
+		debBytes:  map[string][]byte{},
+		rangeHit:  map[string]int{},
+		fullHit:   map[string]int{},
+		assetFail: map[string]int{},
 	}
 	f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
 
@@ -139,6 +142,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
+		if f.assetFail[name] > 0 {
+			f.assetFail[name]--
+			f.mu.Unlock()
+			http.Error(w, "boom", http.StatusInternalServerError)
+			return
+		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -460,3 +469,13 @@ func TestGitHubAssetPatternFilter(t *testing.T) {
 		t.Fatalf("pattern filter failed, rows=%+v", rows)
 	}
 }
+
+func TestGitHubDeriveAssetMissingPackageIsInvalid(t *testing.T) {
+	fx := newGitHubFixture(t, true)
+	fx.debBytes["nameless_1_amd64.deb"] = testsupport.MinimalDeb("", "1", "amd64")
+	asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2-3/nameless_1_amd64.deb", Digest: "sha256:00"}
+	_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
+	if !errors.Is(err, provider.ErrInvalidPackage) {
+		t.Fatalf("want ErrInvalidPackage, got %v", err)
+	}
+}

internal/provider/deb/syncer.go

@@ -28,7 +28,7 @@ type SyncStore interface {
 	provider.RemoteMetadataStore
 	ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
 	ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
-	ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
+	ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
 }
 
 // SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
@@ -189,10 +189,11 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
 		s.mu.Unlock()
 	}()
 
-	freshness := time.Duration(job.remote.MutableTTL) * time.Second
-	if freshness <= 0 {
-		freshness = defaultSyncFreshness
+	ttl := time.Duration(job.remote.MutableTTL) * time.Second
+	if ttl <= 0 {
+		ttl = defaultSyncFreshness
 	}
+	freshness := ttl
 	if job.prime {
 		freshness = 0
 	}
@@ -210,16 +211,13 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
 	defer cancel()
 
 	newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
-	releaseEtag := etag
-	if scanErr == nil {
-		releaseEtag = newEtag
-	} else {
+	if scanErr != nil {
 		slog.Error("github_deb syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
 	}
 
 	relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 	defer relCancel()
-	if err := s.store.ReleaseGitHubDebSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
+	if err := s.store.ReleaseGitHubDebSyncLease(relCtx, job.remote.Name, s.owner, provider.NewSyncResult(newEtag, scanErr, ttl)); err != nil {
 		slog.Warn("github_deb syncer: release lease", "remote", job.remote.Name, "error", err)
 	}
 

internal/provider/deb/syncer_test.go

@@ -27,6 +27,7 @@ type fakeSyncStore struct {
 	leaseExp   map[string]time.Time
 	lastSynced map[string]time.Time
 	etags      map[string]string
+	retryAt    map[string]time.Time
 }
 
 func newFakeSyncStore() *fakeSyncStore {
@@ -36,6 +37,7 @@ func newFakeSyncStore() *fakeSyncStore {
 		leaseExp:   map[string]time.Time{},
 		lastSynced: map[string]time.Time{},
 		etags:      map[string]string{},
+		retryAt:    map[string]time.Time{},
 	}
 }
 
@@ -52,6 +54,9 @@ func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner s
 	ls, hasLS := f.lastSynced[name]
 	exp, hasExp := f.leaseExp[name]
 	freshOK := !hasLS || now.Sub(ls) >= freshness
+	if ra, pending := f.retryAt[name]; pending {
+		freshOK = !now.Before(ra)
+	}
 	leaseOK := !hasExp || exp.Before(now)
 	if freshOK && leaseOK {
 		f.leaseOwner[name] = owner
@@ -61,14 +66,19 @@ func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner s
 	return false, "", nil
 }
 
-func (f *fakeSyncStore) ReleaseGitHubDebSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
+func (f *fakeSyncStore) ReleaseGitHubDebSyncLease(_ context.Context, name, owner string, res provider.SyncResult) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	if f.leaseOwner[name] != owner {
 		return nil
 	}
-	f.lastSynced[name] = syncedAt
-	f.etags[name] = etag
+	if res.Failed {
+		f.retryAt[name] = time.Now().Add(res.Backoff)
+	} else {
+		f.lastSynced[name] = time.Now()
+		f.etags[name] = res.Etag
+		delete(f.retryAt, name)
+	}
 	delete(f.leaseOwner, name)
 	delete(f.leaseExp, name)
 	return nil
@@ -298,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
 		t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
 	}
 }
+
+// A transiently failed asset fails the scan: the ETag is not advanced, a retry
+// is scheduled, and the retry against the unchanged release list derives it. An
+// invalid package is skipped instead, since no retry can fix it.
+func TestSyncerFailedAssetRetry(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		corrupt   bool
+		wantRetry bool
+	}{
+		{name: "transient", wantRetry: true},
+		{name: "invalid package", corrupt: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			fx := newGitHubFixture(t, true)
+			fx.etag = `"v1"`
+			if tc.corrupt {
+				fx.debBytes["other_9-9_arm64.deb"] = []byte("not a package")
+			} else {
+				fx.debBytes["other_9-9_arm64.deb"] = testsupport.MinimalDeb("other", "9-9", "arm64")
+				fx.assetFail["other_9-9_arm64.deb"] = 1
+			}
+			store := newFakeSyncStore()
+			s := newSyncer(store, newTestProvider(), testSyncConfig())
+			remote := fx.remote()
+			bg := context.Background()
+
+			s.process(bg, syncJob{remote: remote, prime: true})
+			_, pending := store.retryAt[remote.Name]
+			if pending != tc.wantRetry {
+				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
+			}
+			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
+				t.Fatalf("etag saved=%v after first scan", saved)
+			}
+			if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 1 {
+				t.Fatalf("healthy asset not served: %d rows", len(rows))
+			}
+			if !tc.wantRetry {
+				return
+			}
+
+			store.mu.Lock()
+			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
+			store.mu.Unlock()
+			s.process(bg, syncJob{remote: remote})
+			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
+				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
+			}
+			if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 2 {
+				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
+			}
+		})
+	}
+}

internal/provider/rpm/github.go

@@ -343,6 +343,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
+	// A transiently failed asset fails the scan so the ETag is not advanced and
+	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
+	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -373,10 +376,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				if !errors.Is(err, provider.ErrInvalidPackage) {
+					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
+				}
 				continue
 			}
 			if err := store.InsertRPMMetadata(ctx, meta); err != nil {
 				slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -388,7 +395,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, nil
+	return newEtag, true, errors.Join(failed...)
 }
 
 type ghRelease struct {
@@ -447,7 +454,7 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote
 			return nil, "", false, err
 		}
 		if resp.StatusCode != http.StatusOK {
-			return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
+			return nil, "", false, provider.NewUpstreamStatusError("github releases API "+u, resp)
 		}
 		if page == 1 {
 			newEtag = respEtag
@@ -569,12 +576,12 @@ func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote,
 			}
 			continue
 		}
-		return nil, fmt.Errorf("parse rpm header: %w", perr)
+		return nil, fmt.Errorf("%w: parse rpm header: %w", provider.ErrInvalidPackage, perr)
 	}
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response body was shorter than n (i.e. we already have the whole object).
+// response holds the whole object.
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -596,15 +603,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
+		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	full := int64(len(body)) < n
-	return body, full, nil
+	return provider.ReadPrefix(resp, n)
 }
 
 func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -627,7 +628,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
+		return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
 
 	h := sha256.New()

internal/provider/rpm/github_test.go

@@ -1,11 +1,13 @@
 package rpm
 
 import (
+	"cmp"
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -24,15 +26,22 @@ import (
 // fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
 // mirroring the (repo_name, file_path) uniqueness of the real table.
 type fakeStore struct {
-	mu   sync.Mutex
-	rows map[string]provider.RPMMetadata
+	mu         sync.Mutex
+	rows       map[string]provider.RPMMetadata
+	insertFail map[string]int // package name -> inserts left to fail
 }
 
-func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
+func newFakeStore() *fakeStore {
+	return &fakeStore{rows: map[string]provider.RPMMetadata{}, insertFail: map[string]int{}}
+}
 
 func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	if f.insertFail[m.Name] > 0 {
+		f.insertFail[m.Name]--
+		return errors.New("insert failed")
+	}
 	if _, ok := f.rows[m.FilePath]; ok {
 		return nil // ON CONFLICT DO NOTHING
 	}
@@ -78,15 +87,21 @@ type githubFixture struct {
 	notModHit   int               // releases-list requests answered 304
 	releaseAuth string            // Authorization header seen on the last releases request
 	assetAuth   string            // Authorization header seen on the last asset request
+	failStatus  int               // when set, the releases list answers this status (rate-limit style)
+	assetFail   map[string]int    // asset filename -> downloads left to answer failCode
+	failCode    int               // status for assetFail downloads; 0 = 500
+	shortRead   map[string]int    // asset filename -> ranged GETs left to close early
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		rpmBytes: map[string][]byte{},
-		rangeHit: map[string]int{},
-		fullHit:  map[string]int{},
+		rpmBytes:  map[string][]byte{},
+		rangeHit:  map[string]int{},
+		fullHit:   map[string]int{},
+		assetFail: map[string]int{},
+		shortRead: map[string]int{},
 	}
 	f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
 
@@ -100,6 +115,14 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		f.mu.Lock()
 		f.releasesHit++
 		f.releaseAuth = r.Header.Get("Authorization")
+		if f.failStatus != 0 {
+			status := f.failStatus
+			f.mu.Unlock()
+			w.Header().Set("X-RateLimit-Remaining", "0")
+			w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(30*time.Second).Unix(), 10))
+			http.Error(w, "API rate limit exceeded", status)
+			return
+		}
 		etag := f.etag
 		if etag != "" && r.Header.Get("If-None-Match") == etag {
 			f.notModHit++
@@ -136,6 +159,17 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
+		if f.assetFail[name] > 0 {
+			f.assetFail[name]--
+			code := cmp.Or(f.failCode, http.StatusInternalServerError)
+			f.mu.Unlock()
+			http.Error(w, "boom", code)
+			return
+		}
+		short := rng != "" && f.shortRead[name] > 0
+		if short {
+			f.shortRead[name]--
+		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -156,6 +190,9 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 			end = len(body) - 1
 		}
 		w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
+		if short {
+			end /= 2
+		}
 		w.Header().Set("Content-Length", strconv.Itoa(end+1))
 		w.WriteHeader(http.StatusPartialContent)
 		w.Write(body[:end+1])

internal/provider/rpm/syncer.go

@@ -35,7 +35,7 @@ type SyncStore interface {
 	provider.RemoteMetadataStore
 	ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
 	ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
-	ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
+	ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
 }
 
 // SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
@@ -205,10 +205,11 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
 		s.mu.Unlock()
 	}()
 
-	freshness := time.Duration(job.remote.MutableTTL) * time.Second
-	if freshness <= 0 {
-		freshness = defaultSyncFreshness
+	ttl := time.Duration(job.remote.MutableTTL) * time.Second
+	if ttl <= 0 {
+		ttl = defaultSyncFreshness
 	}
+	freshness := ttl
 	if job.prime {
 		freshness = 0 // prime ignores the recency gate but still respects a live lease
 	}
@@ -226,18 +227,15 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
 	defer cancel()
 
 	newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
-	releaseEtag := etag
-	if scanErr == nil {
-		releaseEtag = newEtag
-	} else {
+	if scanErr != nil {
 		slog.Error("github_rpm syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
 	}
 
 	// Release on a detached context so a clean shutdown mid-scan still frees the
-	// lease and advances last_synced_at (otherwise it simply expires).
+	// lease and records the outcome (otherwise the lease simply expires).
 	relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 	defer relCancel()
-	if err := s.store.ReleaseGitHubSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
+	if err := s.store.ReleaseGitHubSyncLease(relCtx, job.remote.Name, s.owner, provider.NewSyncResult(newEtag, scanErr, ttl)); err != nil {
 		slog.Warn("github_rpm syncer: release lease", "remote", job.remote.Name, "error", err)
 	}
 

internal/provider/rpm/syncer_test.go

@@ -27,6 +27,8 @@ type fakeSyncStore struct {
 	leaseExp   map[string]time.Time
 	lastSynced map[string]time.Time
 	etags      map[string]string
+	retryAt    map[string]time.Time
+	lastResult provider.SyncResult
 }
 
 func newFakeSyncStore() *fakeSyncStore {
@@ -36,6 +38,7 @@ func newFakeSyncStore() *fakeSyncStore {
 		leaseExp:   map[string]time.Time{},
 		lastSynced: map[string]time.Time{},
 		etags:      map[string]string{},
+		retryAt:    map[string]time.Time{},
 	}
 }
 
@@ -52,6 +55,9 @@ func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner stri
 	ls, hasLS := f.lastSynced[name]
 	exp, hasExp := f.leaseExp[name]
 	freshOK := !hasLS || now.Sub(ls) >= freshness
+	if ra, pending := f.retryAt[name]; pending {
+		freshOK = !now.Before(ra)
+	}
 	leaseOK := !hasExp || exp.Before(now)
 	if freshOK && leaseOK {
 		f.leaseOwner[name] = owner
@@ -61,14 +67,20 @@ func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner stri
 	return false, "", nil
 }
 
-func (f *fakeSyncStore) ReleaseGitHubSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
+func (f *fakeSyncStore) ReleaseGitHubSyncLease(_ context.Context, name, owner string, res provider.SyncResult) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
 	if f.leaseOwner[name] != owner {
 		return nil
 	}
-	f.lastSynced[name] = syncedAt
-	f.etags[name] = etag
+	f.lastResult = res
+	if res.Failed {
+		f.retryAt[name] = time.Now().Add(res.Backoff)
+	} else {
+		f.lastSynced[name] = time.Now()
+		f.etags[name] = res.Etag
+		delete(f.retryAt, name)
+	}
 	delete(f.leaseOwner, name)
 	delete(f.leaseExp, name)
 	return nil
@@ -310,3 +322,151 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
 		t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
 	}
 }
+
+// A rate-limited scan is not recorded as a sync: the last good repodata keeps
+// serving, the retry honours the upstream reset hint and backoff, and once the
+// upstream recovers the next poll after the backoff re-syncs without waiting
+// out mutable_ttl.
+func TestSyncerFailedScanRetriesAfterBackoff(t *testing.T) {
+	fx := newGitHubFixture(t, true)
+	fx.etag = `"v1"`
+	store := newFakeSyncStore()
+	p := newTestProvider()
+	s := newSyncer(store, p, testSyncConfig())
+	remote := fx.remote()
+	bg := context.Background()
+
+	s.process(bg, syncJob{remote: remote, prime: true})
+	if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
+		t.Fatalf("prime did not derive: %d rows", len(rows))
+	}
+
+	// mutable_ttl elapses while GitHub is rate-limiting.
+	fx.mu.Lock()
+	fx.failStatus = http.StatusForbidden
+	fx.etag = `"v2"`
+	fx.mu.Unlock()
+	store.mu.Lock()
+	store.lastSynced[remote.Name] = time.Now().Add(-2 * time.Hour)
+	store.mu.Unlock()
+
+	s.process(bg, syncJob{remote: remote})
+	res := store.lastResult
+	if !res.Failed {
+		t.Fatal("403 scan was recorded as a successful sync")
+	}
+	if until := time.Until(res.RetryAt); until < 20*time.Second || until > 40*time.Second {
+		t.Fatalf("retry hint from X-RateLimit-Reset not carried: retry in %v", until)
+	}
+	if res.Backoff != time.Minute || res.MaxBackoff >= time.Duration(remote.MutableTTL)*time.Second {
+		t.Fatalf("backoff %v cap %v, want 1m first retry capped below mutable_ttl", res.Backoff, res.MaxBackoff)
+	}
+	if store.etags[remote.Name] != `"v1"` {
+		t.Fatalf("failed scan replaced the etag: %q", store.etags[remote.Name])
+	}
+
+	rec := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/repodata/repomd.xml", nil)
+	p.ServeRemote(rec, req, remote, "repodata/repomd.xml", "https://x", store)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("last good repodata not served during failure: %d %s", rec.Code, rec.Body.String())
+	}
+	if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
+		t.Fatalf("failed scan dropped cached metadata: %d rows", len(rows))
+	}
+
+	// Inside the backoff nothing is retried.
+	hits := fx.releasesHit
+	s.process(bg, syncJob{remote: remote})
+	if fx.releasesHit != hits {
+		t.Fatal("scan retried inside the backoff window")
+	}
+
+	// Upstream recovers and the backoff elapses: the next poll re-syncs.
+	fx.mu.Lock()
+	fx.failStatus = 0
+	fx.rpmBytes["other-9-9.aarch64.rpm"] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
+	fx.mu.Unlock()
+	store.mu.Lock()
+	store.retryAt[remote.Name] = time.Now().Add(-time.Second)
+	store.mu.Unlock()
+
+	s.process(bg, syncJob{remote: remote})
+	if store.lastResult.Failed || store.etags[remote.Name] != `"v2"` {
+		t.Fatalf("recovery scan not recorded: failed=%v etag=%q", store.lastResult.Failed, store.etags[remote.Name])
+	}
+	if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 2 {
+		t.Fatalf("recovery did not refresh metadata: %d rows", len(rows))
+	}
+}
+
+// A transiently failed asset fails the scan: the ETag is not advanced, a retry
+// is scheduled, and the retry against the unchanged release list derives it. An
+// invalid package is skipped instead, since no retry can fix it.
+func TestSyncerFailedAssetRetry(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		corrupt   bool
+		failCode  int
+		short     bool
+		insert    bool
+		wantRetry bool
+	}{
+		{name: "transient", wantRetry: true},
+		{name: "rate limited", failCode: http.StatusForbidden, wantRetry: true},
+		{name: "short read", short: true, wantRetry: true},
+		{name: "insert failure", insert: true, wantRetry: true},
+		{name: "invalid package", corrupt: true},
+		{name: "asset not found", failCode: http.StatusNotFound},
+		{name: "asset gone", failCode: http.StatusGone},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			const other = "other-9-9.aarch64.rpm"
+			fx := newGitHubFixture(t, true)
+			fx.etag = `"v1"`
+			fx.rpmBytes[other] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
+			store := newFakeSyncStore()
+			switch {
+			case tc.corrupt:
+				fx.rpmBytes[other] = []byte("not a package")
+			case tc.short:
+				fx.shortRead[other] = 1
+			case tc.insert:
+				store.insertFail["other"] = 1
+			case tc.failCode != 0 && !tc.wantRetry:
+				fx.failCode, fx.assetFail[other] = tc.failCode, 1<<30
+			default:
+				fx.failCode, fx.assetFail[other] = tc.failCode, 1
+			}
+			s := newSyncer(store, newTestProvider(), testSyncConfig())
+			remote := fx.remote()
+			bg := context.Background()
+
+			s.process(bg, syncJob{remote: remote, prime: true})
+			_, pending := store.retryAt[remote.Name]
+			if pending != tc.wantRetry {
+				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
+			}
+			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
+				t.Fatalf("etag saved=%v after first scan", saved)
+			}
+			if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
+				t.Fatalf("healthy asset not served: %d rows", len(rows))
+			}
+			if !tc.wantRetry {
+				return
+			}
+
+			store.mu.Lock()
+			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
+			store.mu.Unlock()
+			s.process(bg, syncJob{remote: remote})
+			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
+				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
+			}
+			if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 2 {
+				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
+			}
+		})
+	}
+}

internal/provider/syncretry.go

@@ -0,0 +1,120 @@
+package provider
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	syncRetryBase = time.Minute
+	syncRetryMax  = 10 * time.Minute
+)
+
+// UpstreamStatusError is a non-success upstream response. RetryAt is the
+// upstream's own retry hint (Retry-After, or X-RateLimit-Reset once the quota
+// is exhausted); zero when it gave none.
+type UpstreamStatusError struct {
+	URL     string
+	Status  int
+	RetryAt time.Time
+}
+
+func (e *UpstreamStatusError) Error() string {
+	return fmt.Sprintf("%s: status %d", e.URL, e.Status)
+}
+
+// NewUpstreamStatusError wraps a non-success response, capturing its retry hint.
+func NewUpstreamStatusError(url string, resp *http.Response) *UpstreamStatusError {
+	e := &UpstreamStatusError{URL: url, Status: resp.StatusCode}
+	if ra := resp.Header.Get("Retry-After"); ra != "" {
+		if secs, err := strconv.Atoi(ra); err == nil {
+			e.RetryAt = time.Now().Add(time.Duration(secs) * time.Second)
+		} else if t, err := http.ParseTime(ra); err == nil {
+			e.RetryAt = t
+		}
+	} else if resp.Header.Get("X-RateLimit-Remaining") == "0" {
+		if reset, err := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 64); err == nil {
+			e.RetryAt = time.Unix(reset, 0)
+		}
+	}
+	return e
+}
+
+// ErrInvalidPackage marks an asset whose bytes are not a valid package. It is
+// permanent, so a scan skips the asset instead of failing and retrying.
+var ErrInvalidPackage = errors.New("invalid package")
+
+// AssetStatusError classifies a non-success asset download. A missing or
+// unsatisfiable asset is permanent; anything else (403/429 rate limits, 5xx) is
+// transient.
+func AssetStatusError(url string, resp *http.Response) error {
+	switch resp.StatusCode {
+	case http.StatusNotFound, http.StatusGone, http.StatusRequestedRangeNotSatisfiable:
+		return fmt.Errorf("%w: %s: status %d", ErrInvalidPackage, url, resp.StatusCode)
+	}
+	return NewUpstreamStatusError(url, resp)
+}
+
+// ReadPrefix reads up to n bytes of a 200 or 206 asset response. full reports
+// that the body is the whole object (per Content-Length or the Content-Range
+// total, or a clean EOF before n when the total is unknown); a body ending before n bytes that is not the whole object is a
+// transient short read.
+func ReadPrefix(resp *http.Response, n int64) (body []byte, full bool, err error) {
+	body, err = io.ReadAll(io.LimitReader(resp.Body, n))
+	if err != nil {
+		return nil, false, err
+	}
+	total := resp.ContentLength
+	unknown := resp.StatusCode == http.StatusOK && total == -1
+	if resp.StatusCode == http.StatusPartialContent {
+		total = -1
+		if _, t, ok := strings.Cut(resp.Header.Get("Content-Range"), "/"); ok {
+			if v, perr := strconv.ParseInt(t, 10, 64); perr == nil {
+				total = v
+			}
+			unknown = t == "*"
+		}
+	}
+	// net/http errors on a premature close, so a clean EOF before n with an unknown total is the whole object.
+	full = total == int64(len(body)) || (unknown && int64(len(body)) < n)
+	if !full && int64(len(body)) < n {
+		return nil, false, fmt.Errorf("short read: got %d of %d bytes", len(body), n)
+	}
+	return body, full, nil
+}
+
+// SyncResult is a background scan's outcome, recorded when its sync lease is
+// released. A failed scan keeps the prior sync time and ETag and schedules a
+// retry after Backoff, doubled per consecutive failure up to MaxBackoff, and
+// never earlier than RetryAt.
+type SyncResult struct {
+	Etag       string
+	Failed     bool
+	RetryAt    time.Time
+	Backoff    time.Duration
+	MaxBackoff time.Duration
+}
+
+// NewSyncResult builds the result for a scan against a remote with the given
+// mutable_ttl. The retry cap stays well below ttl, and an upstream hint is
+// clamped to ttl so a bogus reset can never stall the remote longer than a
+// normal sync interval would.
+func NewSyncResult(etag string, scanErr error, ttl time.Duration) SyncResult {
+	if scanErr == nil {
+		return SyncResult{Etag: etag}
+	}
+	res := SyncResult{Failed: true, Backoff: syncRetryBase, MaxBackoff: min(syncRetryMax, max(syncRetryBase, ttl/4))}
+	var se *UpstreamStatusError
+	if errors.As(scanErr, &se) && !se.RetryAt.IsZero() {
+		res.RetryAt = se.RetryAt
+		if limit := time.Now().Add(ttl); res.RetryAt.After(limit) {
+			res.RetryAt = limit
+		}
+	}
+	return res
+}

internal/provider/syncretry_test.go

@@ -0,0 +1,138 @@
+package provider
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+	"time"
+)
+
+func TestNewUpstreamStatusErrorRetryHint(t *testing.T) {
+	reset := time.Now().Add(15 * time.Minute).Truncate(time.Second)
+	cases := []struct {
+		name   string
+		header map[string]string
+		want   time.Duration // 0 = no hint
+	}{
+		{"retry-after seconds", map[string]string{"Retry-After": "90"}, 90 * time.Second},
+		{"retry-after date", map[string]string{"Retry-After": reset.UTC().Format(http.TimeFormat)}, 15 * time.Minute},
+		{"quota exhausted", map[string]string{"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": strconv.FormatInt(reset.Unix(), 10)}, 15 * time.Minute},
+		{"quota left is not a hint", map[string]string{"X-RateLimit-Remaining": "12", "X-RateLimit-Reset": strconv.FormatInt(reset.Unix(), 10)}, 0},
+		{"no headers", nil, 0},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			resp := &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}}
+			for k, v := range c.header {
+				resp.Header.Set(k, v)
+			}
+			e := NewUpstreamStatusError("u", resp)
+			if c.want == 0 {
+				if !e.RetryAt.IsZero() {
+					t.Fatalf("unexpected hint %v", e.RetryAt)
+				}
+				return
+			}
+			if d := time.Until(e.RetryAt) - c.want; d < -2*time.Second || d > 2*time.Second {
+				t.Fatalf("retry in %v, want ~%v", time.Until(e.RetryAt), c.want)
+			}
+		})
+	}
+}
+
+func TestNewSyncResult(t *testing.T) {
+	if r := NewSyncResult(`"e"`, nil, time.Hour); r.Failed || r.Etag != `"e"` {
+		t.Fatalf("success result = %+v", r)
+	}
+
+	r := NewSyncResult(`"e"`, errors.New("boom"), time.Hour)
+	if !r.Failed || r.Backoff != time.Minute || r.MaxBackoff != 10*time.Minute || !r.RetryAt.IsZero() {
+		t.Fatalf("plain failure = %+v, want 1m backoff capped at 10m, no hint", r)
+	}
+	if r := NewSyncResult("", errors.New("boom"), 5*time.Minute); r.MaxBackoff != time.Minute+15*time.Second {
+		t.Fatalf("short ttl cap = %v, want ttl/4", r.MaxBackoff)
+	}
+
+	far := &UpstreamStatusError{Status: 403, RetryAt: time.Now().Add(3 * time.Hour)}
+	r = NewSyncResult("", fmt.Errorf("scan: %w", far), time.Hour)
+	if until := time.Until(r.RetryAt); until > time.Hour || until < 59*time.Minute {
+		t.Fatalf("wrapped hint not clamped to ttl: retry in %v", until)
+	}
+}
+
+func TestAssetStatusErrorClassification(t *testing.T) {
+	for status, permanent := range map[int]bool{
+		http.StatusNotFound: true, http.StatusGone: true, http.StatusRequestedRangeNotSatisfiable: true,
+		http.StatusForbidden: false, http.StatusTooManyRequests: false, http.StatusInternalServerError: false, http.StatusBadGateway: false,
+	} {
+		err := AssetStatusError("u", &http.Response{StatusCode: status, Header: http.Header{}})
+		if errors.Is(err, ErrInvalidPackage) != permanent {
+			t.Errorf("status %d: permanent=%v, want %v", status, !permanent, permanent)
+		}
+	}
+}
+
+func TestReadPrefix(t *testing.T) {
+	cases := []struct {
+		name     string
+		status   int
+		cl       int64
+		crange   string
+		body     string
+		n        int64
+		wantFull bool
+		wantErr  bool
+	}{
+		{name: "200 whole object", status: 200, cl: 5, body: "hello", n: 32, wantFull: true},
+		{name: "200 larger than n", status: 200, cl: 5, body: "hello", n: 4},
+		{name: "200 chunked shorter than n", status: 200, cl: -1, body: "hello", n: 32, wantFull: true},
+		{name: "200 chunked larger than n", status: 200, cl: -1, body: "hello", n: 4},
+		{name: "206 whole object", status: 206, crange: "bytes 0-4/5", body: "hello", n: 32, wantFull: true},
+		{name: "206 prefix", status: 206, crange: "bytes 0-3/100", body: "hell", n: 4},
+		{name: "206 short read", status: 206, crange: "bytes 0-31/100", body: "hello", n: 32, wantErr: true},
+		{name: "206 no content-range short", status: 206, body: "hello", n: 32, wantErr: true},
+		{name: "206 unknown total shorter than n", status: 206, cl: -1, crange: "bytes 0-4/*", body: "hello", n: 32, wantFull: true},
+		{name: "206 unknown total prefix", status: 206, cl: -1, crange: "bytes 0-3/*", body: "hell", n: 4},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				if tc.cl > 0 {
+					w.Header().Set("Content-Length", strconv.FormatInt(tc.cl, 10))
+				}
+				if tc.crange != "" {
+					w.Header().Set("Content-Range", tc.crange)
+				}
+				w.WriteHeader(tc.status)
+				_, _ = io.WriteString(w, tc.body)
+				w.(http.Flusher).Flush()
+			}))
+			defer srv.Close()
+			resp, err := http.Get(srv.URL)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer resp.Body.Close()
+			if tc.cl == -1 && resp.ContentLength != -1 {
+				t.Fatalf("response not chunked: ContentLength=%d", resp.ContentLength)
+			}
+			body, full, err := ReadPrefix(resp, tc.n)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err=%v, wantErr=%v", err, tc.wantErr)
+			}
+			if err != nil {
+				if errors.Is(err, ErrInvalidPackage) {
+					t.Fatalf("short read must be transient: %v", err)
+				}
+				return
+			}
+			if full != tc.wantFull || int64(len(body)) > tc.n {
+				t.Fatalf("full=%v len=%d, want full=%v", full, len(body), tc.wantFull)
+			}
+		})
+	}
+}

migrations/0002_github_sync_retry.sql

@@ -0,0 +1,9 @@
+-- Failed GitHub release scans retry with backoff instead of waiting out a full
+-- mutable_ttl: sync_failures counts consecutive failures, next_retry_at gates
+-- the next claim while a retry is pending.
+ALTER TABLE github_rpm_sync_state    ADD COLUMN IF NOT EXISTS sync_failures INT NOT NULL DEFAULT 0;
+ALTER TABLE github_rpm_sync_state    ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ;
+ALTER TABLE github_deb_sync_state    ADD COLUMN IF NOT EXISTS sync_failures INT NOT NULL DEFAULT 0;
+ALTER TABLE github_deb_sync_state    ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ;
+ALTER TABLE github_alpine_sync_state ADD COLUMN IF NOT EXISTS sync_failures INT NOT NULL DEFAULT 0;
+ALTER TABLE github_alpine_sync_state ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ;

migrations/0003_github_sync_state_cascade.sql

@@ -0,0 +1,12 @@
+-- Sync state is dropped with its remote, so a recreated remote of the same name
+-- starts with no inherited ETag or timing. NOT VALID skips checking rows left by
+-- remotes deleted before this migration, which would otherwise fail it.
+ALTER TABLE github_rpm_sync_state    DROP CONSTRAINT IF EXISTS github_rpm_sync_state_remote_fk;
+ALTER TABLE github_rpm_sync_state    ADD CONSTRAINT github_rpm_sync_state_remote_fk
+	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;
+ALTER TABLE github_deb_sync_state    DROP CONSTRAINT IF EXISTS github_deb_sync_state_remote_fk;
+ALTER TABLE github_deb_sync_state    ADD CONSTRAINT github_deb_sync_state_remote_fk
+	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;
+ALTER TABLE github_alpine_sync_state DROP CONSTRAINT IF EXISTS github_alpine_sync_state_remote_fk;
+ALTER TABLE github_alpine_sync_state ADD CONSTRAINT github_alpine_sync_state_remote_fk
+	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;