unkinben
6dc72920da
Lazy per-replica scans re-derived RPM metadata on the client request path and, run independently on every replica, multiplied GitHub queries by the replica count. A single background syncer with a shared rate limit, ETag conditional checks, and a DB lease keeps metadata fresh off the request path while bounding GitHub load to ~once per mutable_ttl across the fleet. - Add a single per-process syncer (started at boot, stopped on shutdown) that owns a deduped/coalescing work queue, a worker pool, and one global token-bucket rate limiter bound onto the github provider so every GitHub call (releases list + each ranged asset GET) acquires a token first. - Check each github_rpm remote for new/changed releases on its mutable_ttl cadence; derive only new/changed assets incrementally and prune assets that disappear upstream, so repodata is served from primed DB rows. - Prime metadata in the background on remote creation; the create call never blocks on a derive. - Send the stored releases-list ETag as If-None-Match; a 304 derives nothing (and does not count against GitHub's rate limit), making an unchanged repo nearly free. - Coordinate replicas through a github_rpm_sync_state row (last_synced_at, etag, sync_lease_owner, sync_lease_expires): a periodic scan runs only for the replica that atomically claims the lease, bounding total GitHub load to ~once per mutable_ttl regardless of replica count. - Keep the request path fast: serve current cache, enqueue a prime on an empty cache, and return a bounded wait then a retryable 503 rather than blocking on a cold derive. - Add GITHUB_SYNC_RATE/BURST/WORKERS/POLL_INTERVAL config (conservative defaults) and document the syncer in the README.
128 lines
3.6 KiB
Go
128 lines
3.6 KiB
Go
package v2
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
|
|
"git.unkin.net/unkin/artifactapi/internal/database"
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
// Primer enqueues a background metadata prime for a newly created remote so the
|
|
// create call never blocks on a derive. *rpm.Syncer satisfies it.
|
|
type Primer interface {
|
|
EnqueuePrime(remote models.Remote)
|
|
}
|
|
|
|
type RemotesHandler struct {
|
|
db *database.DB
|
|
primer Primer
|
|
}
|
|
|
|
func NewRemotesHandler(db *database.DB, primer Primer) *RemotesHandler {
|
|
return &RemotesHandler{db: db, primer: primer}
|
|
}
|
|
|
|
func (h *RemotesHandler) Routes() chi.Router {
|
|
r := chi.NewRouter()
|
|
r.Get("/", h.list)
|
|
r.Post("/", h.create)
|
|
r.Get("/{name}", h.get)
|
|
r.Put("/{name}", h.update)
|
|
r.Delete("/{name}", h.del)
|
|
return r
|
|
}
|
|
|
|
func (h *RemotesHandler) list(w http.ResponseWriter, r *http.Request) {
|
|
remotes, err := h.db.ListRemotes(r.Context())
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, remotes)
|
|
}
|
|
|
|
func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
|
|
name := chi.URLParam(r, "name")
|
|
remote, err := h.db.GetRemote(r.Context(), name)
|
|
if err != nil {
|
|
http.Error(w, fmt.Sprintf("remote %q not found", name), http.StatusNotFound)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, remote)
|
|
}
|
|
|
|
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
|
var remote models.Remote
|
|
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
|
http.Error(w, "invalid json", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if !remote.PackageType.Valid() {
|
|
http.Error(w, fmt.Sprintf("invalid package type: %q", remote.PackageType), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if remote.RepoType == "" {
|
|
remote.RepoType = models.RepoTypeRemote
|
|
}
|
|
if !remote.RepoType.Valid() {
|
|
http.Error(w, fmt.Sprintf("invalid repo type: %q", remote.RepoType), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if remote.RepoType == models.RepoTypeRemote && remote.BaseURL == "" {
|
|
http.Error(w, "base_url is required for remote repositories", http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := remote.ValidatePatterns(); err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
// Prime a github_rpm remote's metadata in the background so its first
|
|
// repodata request is served from cache instead of a cold on-demand derive.
|
|
if h.primer != nil && remote.PackageType == models.PackageGitHubRPM {
|
|
h.primer.EnqueuePrime(remote)
|
|
}
|
|
writeJSON(w, http.StatusCreated, remote)
|
|
}
|
|
|
|
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
|
name := chi.URLParam(r, "name")
|
|
var remote models.Remote
|
|
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
|
http.Error(w, "invalid json", http.StatusBadRequest)
|
|
return
|
|
}
|
|
remote.Name = name
|
|
if err := remote.ValidatePatterns(); err != nil {
|
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
|
return
|
|
}
|
|
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, remote)
|
|
}
|
|
|
|
func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
|
|
name := chi.URLParam(r, "name")
|
|
if err := h.db.DeleteRemote(r.Context(), name); err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusNoContent)
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(status)
|
|
json.NewEncoder(w).Encode(v)
|
|
}
|