Skip missing GitHub assets and treat short reads as transient

internal/provider/rpm/github.go

@@ -581,7 +581,7 @@ func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote,
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response body was shorter than n (i.e. we already have the whole object).
+// response holds the whole object.
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -603,15 +603,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
+		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	full := int64(len(body)) < n
-	return body, full, nil
+	return provider.ReadPrefix(resp, n)
 }
 
 func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -634,7 +628,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
+		return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
 
 	h := sha256.New()

internal/provider/rpm/github_test.go

@@ -1,11 +1,13 @@
 package rpm
 
 import (
+	"cmp"
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -24,15 +26,22 @@ import (
 // fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
 // mirroring the (repo_name, file_path) uniqueness of the real table.
 type fakeStore struct {
-	mu   sync.Mutex
-	rows map[string]provider.RPMMetadata
+	mu         sync.Mutex
+	rows       map[string]provider.RPMMetadata
+	insertFail map[string]int // package name -> inserts left to fail
 }
 
-func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
+func newFakeStore() *fakeStore {
+	return &fakeStore{rows: map[string]provider.RPMMetadata{}, insertFail: map[string]int{}}
+}
 
 func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	if f.insertFail[m.Name] > 0 {
+		f.insertFail[m.Name]--
+		return errors.New("insert failed")
+	}
 	if _, ok := f.rows[m.FilePath]; ok {
 		return nil // ON CONFLICT DO NOTHING
 	}
@@ -79,7 +88,9 @@ type githubFixture struct {
 	releaseAuth string            // Authorization header seen on the last releases request
 	assetAuth   string            // Authorization header seen on the last asset request
 	failStatus  int               // when set, the releases list answers this status (rate-limit style)
-	assetFail   map[string]int    // asset filename -> downloads left to answer 500
+	assetFail   map[string]int    // asset filename -> downloads left to answer failCode
+	failCode    int               // status for assetFail downloads; 0 = 500
+	shortRead   map[string]int    // asset filename -> ranged GETs left to close early
 	mu          sync.Mutex
 }
 
@@ -90,6 +101,7 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		rangeHit:  map[string]int{},
 		fullHit:   map[string]int{},
 		assetFail: map[string]int{},
+		shortRead: map[string]int{},
 	}
 	f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
 
@@ -149,10 +161,15 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		f.mu.Lock()
 		if f.assetFail[name] > 0 {
 			f.assetFail[name]--
+			code := cmp.Or(f.failCode, http.StatusInternalServerError)
 			f.mu.Unlock()
-			http.Error(w, "boom", http.StatusInternalServerError)
+			http.Error(w, "boom", code)
 			return
 		}
+		short := rng != "" && f.shortRead[name] > 0
+		if short {
+			f.shortRead[name]--
+		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -173,6 +190,9 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 			end = len(body) - 1
 		}
 		w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
+		if short {
+			end /= 2
+		}
 		w.Header().Set("Content-Length", strconv.Itoa(end+1))
 		w.WriteHeader(http.StatusPartialContent)
 		w.Write(body[:end+1])

internal/provider/rpm/syncer_test.go

@@ -407,21 +407,37 @@ func TestSyncerFailedAssetRetry(t *testing.T) {
 	for _, tc := range []struct {
 		name      string
 		corrupt   bool
+		failCode  int
+		short     bool
+		insert    bool
 		wantRetry bool
 	}{
 		{name: "transient", wantRetry: true},
+		{name: "rate limited", failCode: http.StatusForbidden, wantRetry: true},
+		{name: "short read", short: true, wantRetry: true},
+		{name: "insert failure", insert: true, wantRetry: true},
 		{name: "invalid package", corrupt: true},
+		{name: "asset not found", failCode: http.StatusNotFound},
+		{name: "asset gone", failCode: http.StatusGone},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
+			const other = "other-9-9.aarch64.rpm"
 			fx := newGitHubFixture(t, true)
 			fx.etag = `"v1"`
-			if tc.corrupt {
-				fx.rpmBytes["other-9-9.aarch64.rpm"] = []byte("not a package")
-			} else {
-				fx.rpmBytes["other-9-9.aarch64.rpm"] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
-				fx.assetFail["other-9-9.aarch64.rpm"] = 1
-			}
+			fx.rpmBytes[other] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
 			store := newFakeSyncStore()
+			switch {
+			case tc.corrupt:
+				fx.rpmBytes[other] = []byte("not a package")
+			case tc.short:
+				fx.shortRead[other] = 1
+			case tc.insert:
+				store.insertFail["other"] = 1
+			case tc.failCode != 0 && !tc.wantRetry:
+				fx.failCode, fx.assetFail[other] = tc.failCode, 1<<30
+			default:
+				fx.failCode, fx.assetFail[other] = tc.failCode, 1
+			}
 			s := newSyncer(store, newTestProvider(), testSyncConfig())
 			remote := fx.remote()
 			bg := context.Background()