Files
artifactapi/internal/api/v2/errorpaths_test.go
T
unkinben 6dc72920da
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful
feat: background syncer for github_rpm remotes
Lazy per-replica scans re-derived RPM metadata on the client request path
and, run independently on every replica, multiplied GitHub queries by the
replica count. A single background syncer with a shared rate limit, ETag
conditional checks, and a DB lease keeps metadata fresh off the request path
while bounding GitHub load to ~once per mutable_ttl across the fleet.

- Add a single per-process syncer (started at boot, stopped on shutdown) that
  owns a deduped/coalescing work queue, a worker pool, and one global
  token-bucket rate limiter bound onto the github provider so every GitHub call
  (releases list + each ranged asset GET) acquires a token first.
- Check each github_rpm remote for new/changed releases on its mutable_ttl
  cadence; derive only new/changed assets incrementally and prune assets that
  disappear upstream, so repodata is served from primed DB rows.
- Prime metadata in the background on remote creation; the create call never
  blocks on a derive.
- Send the stored releases-list ETag as If-None-Match; a 304 derives nothing
  (and does not count against GitHub's rate limit), making an unchanged repo
  nearly free.
- Coordinate replicas through a github_rpm_sync_state row (last_synced_at,
  etag, sync_lease_owner, sync_lease_expires): a periodic scan runs only for
  the replica that atomically claims the lease, bounding total GitHub load to
  ~once per mutable_ttl regardless of replica count.
- Keep the request path fast: serve current cache, enqueue a prime on an empty
  cache, and return a bounded wait then a retryable 503 rather than blocking on
  a cold derive.
- Add GITHUB_SYNC_RATE/BURST/WORKERS/POLL_INTERVAL config (conservative
  defaults) and document the syncer in the README.
2026-08-10 21:08:24 +10:00

131 lines
3.5 KiB
Go

package v2
import (
"context"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
var testDSN string
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, terminate, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
testDSN = dsn
code := m.Run()
terminate()
if code != 0 {
os.Exit(code)
}
}
// closedDB returns a DB whose pool has been closed, so every query fails —
// used to drive the handlers' error branches.
func closedDB(t *testing.T) *database.DB {
t.Helper()
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatalf("new db: %v", err)
}
db.Close()
return db
}
func do(t *testing.T, h http.Handler, method, path, body string) int {
t.Helper()
var r io.Reader
if body != "" {
r = strings.NewReader(body)
}
req := httptest.NewRequest(method, path, r)
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
return w.Code
}
func TestRemotesErrorPaths(t *testing.T) {
h := NewRemotesHandler(closedDB(t), nil).Routes()
if c := do(t, h, "GET", "/", ""); c != 500 {
t.Errorf("list with dead db = %d, want 500", c)
}
if c := do(t, h, "POST", "/", `{"name":"x","package_type":"generic","repo_type":"remote","base_url":"https://x"}`); c != 500 {
t.Errorf("create with dead db = %d, want 500", c)
}
if c := do(t, h, "PUT", "/x", `{"package_type":"generic","base_url":"https://x"}`); c != 500 {
t.Errorf("update with dead db = %d, want 500", c)
}
if c := do(t, h, "GET", "/x", ""); c != 404 {
t.Errorf("get missing = %d, want 404", c)
}
if c := do(t, h, "DELETE", "/x", ""); c != 500 {
t.Errorf("delete with dead db = %d, want 500", c)
}
// Bad request bodies never reach the db.
if c := do(t, h, "POST", "/", `not json`); c != 400 {
t.Errorf("invalid json = %d, want 400", c)
}
}
func TestVirtualsErrorPaths(t *testing.T) {
h := NewVirtualsHandler(closedDB(t)).Routes()
if c := do(t, h, "GET", "/", ""); c != 500 {
t.Errorf("list = %d, want 500", c)
}
if c := do(t, h, "GET", "/x", ""); c != 404 {
t.Errorf("get missing = %d, want 404", c)
}
if c := do(t, h, "POST", "/", `{"name":"v","package_type":"helm","members":["a"]}`); c != 500 {
t.Errorf("create = %d, want 500", c)
}
if c := do(t, h, "PUT", "/v", `{"package_type":"helm","members":["a"]}`); c != 500 {
t.Errorf("update = %d, want 500", c)
}
if c := do(t, h, "DELETE", "/v", ""); c != 500 {
t.Errorf("delete = %d, want 500", c)
}
}
func TestStatsErrorPaths(t *testing.T) {
h := NewStatsHandler(closedDB(t)).Routes()
for _, p := range []string{"/", "/top-remotes", "/top-files-by-hits", "/top-files-by-bandwidth"} {
if c := do(t, h, "GET", p, ""); c != 500 {
t.Errorf("stats %s = %d, want 500", p, c)
}
}
}
func TestLocalErrorPaths(t *testing.T) {
h := NewLocalHandler(closedDB(t), nil).Routes()
// GetRemote fails on the closed db -> not found.
if c := do(t, h, "PUT", "/x/files/a.bin", "data"); c != 404 {
t.Errorf("upload unknown repo = %d, want 404", c)
}
// download / remove hit the db and 500.
if c := do(t, h, "GET", "/x/files/a.bin", ""); c != 500 {
t.Errorf("download = %d, want 500", c)
}
if c := do(t, h, "DELETE", "/x/files/a.bin", ""); c != 500 {
t.Errorf("remove = %d, want 500", c)
}
}
func TestLocalHandlerDBAccessor(t *testing.T) {
db := closedDB(t)
if NewLocalHandler(db, nil).DB() != db {
t.Error("DB() should return the handler's database")
}
}