Add cache_repodata toggle for github_rpm remotes (#168)
github_rpm remotes regenerate repomd and all three gz files from every metadata row on each request, which is expensive for large repos under repeated dnf traffic. - add a per-remote `cache_repodata` toggle (column, model, v2 API, docs) - cache built repodata per replica, rebuilt via singleflight when the `count(*)`+`max(id)` row fingerprint moves - store a build only if no newer build has landed; evict idle entries hourly - map cached-path DB errors like the uncached path: 503 + Retry-After on cancel/deadline, generic 500 otherwise - check `cacheEmpty` via the fingerprint for all github_rpm remotes, not only cached ones Reviewed-on: #168 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #168.
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
func TestCacheRepodataRoundTrip(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
h := NewRemotesHandler(db, nil, nil).Routes()
|
||||
get := func() bool {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest("GET", "/cache-rd", nil))
|
||||
var got struct {
|
||||
CacheRepodata *bool `json:"cache_repodata"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil || got.CacheRepodata == nil {
|
||||
t.Fatalf("get = %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
return *got.CacheRepodata
|
||||
}
|
||||
const body = `"package_type":"github_rpm","base_url":"https://api.github.com"`
|
||||
|
||||
steps := []struct {
|
||||
method, path, json string
|
||||
want bool
|
||||
}{
|
||||
{"POST", "/", `{"name":"cache-rd",` + body + `}`, false},
|
||||
{"PUT", "/cache-rd", `{` + body + `,"cache_repodata":true}`, true},
|
||||
{"PUT", "/cache-rd", `{` + body + `,"cache_repodata":false}`, false},
|
||||
}
|
||||
for i, s := range steps {
|
||||
if c := do(t, h, s.method, s.path, s.json); c/100 != 2 {
|
||||
t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c)
|
||||
}
|
||||
if got := get(); got != s.want {
|
||||
t.Errorf("step %d: cache_repodata = %v, want %v", i, got, s.want)
|
||||
}
|
||||
}
|
||||
if c := do(t, h, "POST", "/", `{"name":"cache-rd-on",`+body+`,"cache_repodata":true}`); c != 201 {
|
||||
t.Fatalf("create with cache_repodata = %d", c)
|
||||
}
|
||||
if r, err := db.GetRemote(t.Context(), "cache-rd-on"); err != nil || !r.CacheRepodata {
|
||||
t.Fatalf("create did not store cache_repodata: %v %v", r, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user