Skip invalid GitHub packages instead of failing the scan
Wrap package parse errors in provider.ErrInvalidPackage so only transient asset and insert errors fail the scan; cover rpm, deb and alpine.
This commit is contained in:
@@ -372,8 +372,8 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
// A failed asset fails the scan so the ETag is not advanced and the retry
|
||||
// re-derives it; assets that did derive are kept and served meanwhile.
|
||||
// A transiently failed asset fails the scan so the ETag is not advanced and
|
||||
// the retry re-derives it; an invalid package is skipped as no retry fixes it.
|
||||
var failed []error
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
@@ -403,7 +403,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
if !errors.Is(err, provider.ErrInvalidPackage) {
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
|
||||
@@ -501,7 +503,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
|
||||
return nil, err
|
||||
}
|
||||
if meta.Name == "" || meta.Arch == "" {
|
||||
return nil, errors.New(".PKGINFO missing pkgname/arch")
|
||||
return nil, fmt.Errorf("%w: .PKGINFO missing pkgname/arch", provider.ErrInvalidPackage)
|
||||
}
|
||||
|
||||
meta.RepoName = remote.Name
|
||||
@@ -531,13 +533,13 @@ func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote,
|
||||
}
|
||||
meta, complete, perr := pkginfoFromPrefix(body)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
|
||||
return nil, fmt.Errorf("%w: parse apk .PKGINFO: %w", provider.ErrInvalidPackage, perr)
|
||||
}
|
||||
if complete {
|
||||
return meta, nil
|
||||
}
|
||||
if full || n >= p.headerMax {
|
||||
return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
|
||||
return nil, fmt.Errorf("%w: .PKGINFO not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
|
||||
}
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
|
||||
@@ -88,15 +88,17 @@ type githubFixture struct {
|
||||
notModHit int
|
||||
releaseAuth string
|
||||
assetAuth string
|
||||
assetFail map[string]int // asset filename -> downloads left to answer 500
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
apkBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
apkBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
assetFail: map[string]int{},
|
||||
}
|
||||
f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
|
||||
|
||||
@@ -146,6 +148,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
if f.assetFail[name] > 0 {
|
||||
f.assetFail[name]--
|
||||
f.mu.Unlock()
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
|
||||
@@ -308,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
|
||||
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
|
||||
}
|
||||
}
|
||||
|
||||
// A transiently failed asset fails the scan: the ETag is not advanced, a retry
|
||||
// is scheduled, and the retry against the unchanged release list derives it. An
|
||||
// invalid package is skipped instead, since no retry can fix it.
|
||||
func TestSyncerFailedAssetRetry(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
corrupt bool
|
||||
wantRetry bool
|
||||
}{
|
||||
{name: "transient", wantRetry: true},
|
||||
{name: "invalid package", corrupt: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
if tc.corrupt {
|
||||
fx.apkBytes["other-9-r0.apk"] = []byte("not a package")
|
||||
} else {
|
||||
fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
|
||||
fx.assetFail["other-9-r0.apk"] = 1
|
||||
}
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
remote := fx.remote()
|
||||
bg := context.Background()
|
||||
|
||||
s.process(bg, syncJob{remote: remote, prime: true})
|
||||
_, pending := store.retryAt[remote.Name]
|
||||
if pending != tc.wantRetry {
|
||||
t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
|
||||
}
|
||||
if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
|
||||
t.Fatalf("etag saved=%v after first scan", saved)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("healthy asset not served: %d rows", len(rows))
|
||||
}
|
||||
if !tc.wantRetry {
|
||||
return
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
|
||||
store.mu.Unlock()
|
||||
s.process(bg, syncJob{remote: remote})
|
||||
if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
|
||||
t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 2 {
|
||||
t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user