Merge origin/master into benvin/virtual-npm

This commit is contained in:
2026-10-17 21:40:54 +11:00
35 changed files with 1012 additions and 89 deletions
+2 -8
View File
@@ -13,6 +13,7 @@ import (
"github.com/go-chi/chi/v5"
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
"git.unkin.net/unkin/artifactapi/internal/database"
tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
@@ -290,12 +291,5 @@ func writeJSON(w http.ResponseWriter, v any) {
}
func baseURL(r *http.Request) string {
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
scheme = fwd
}
return scheme + "://" + r.Host
return v1.Scheme(r) + "://" + r.Host
}
+1 -1
View File
@@ -254,7 +254,7 @@ func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request,
return
}
read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", scheme(r), r.Host, remote.Name)
tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", Scheme(r), r.Host, remote.Name)
doc, err := buildNPMPackument(pkg, files, read, tarballBase)
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
+109 -32
View File
@@ -6,7 +6,9 @@ import (
"fmt"
"io"
"log/slog"
"net"
"net/http"
"strconv"
"strings"
"github.com/go-chi/chi/v5"
@@ -43,6 +45,7 @@ func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *da
func (h *ProxyHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/remote/{remoteName}/*", h.handleProxy)
r.Head("/remote/{remoteName}/*", h.handleProxyHead)
r.Get("/local/{localName}/*", h.handleLocal)
r.Put("/local/{localName}/*", h.handleLocalPut)
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
@@ -71,31 +74,43 @@ func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusOK)
}
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
// remoteFor resolves the remote and provider named in the request, writing the
// error response itself when either is missing.
func (h *ProxyHandler) remoteFor(w http.ResponseWriter, r *http.Request) (*models.Remote, provider.Provider, bool) {
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
return
return nil, nil, false
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
return nil, nil, false
}
return remote, prov, true
}
// serveRemote lets metadata-only remotes (e.g. github_rpm) synthesize their own
// responses and redirect package downloads instead of proxying bytes.
func (h *ProxyHandler) serveRemote(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) bool {
rs, ok := prov.(provider.RemoteServer)
return ok && rs.ServeRemote(w, r, *remote, path, fmt.Sprintf("%s://%s", Scheme(r), r.Host), h.db)
}
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
remote, prov, ok := h.remoteFor(w, r)
if !ok {
return
}
// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
// redirect package downloads to a backend remote instead of proxying bytes.
if rs, ok := prov.(provider.RemoteServer); ok {
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
return
}
path := chi.URLParam(r, "*")
if h.serveRemote(w, r, remote, prov, path) {
return
}
h.serveFetch(w, r, remote, prov, path)
}
func (h *ProxyHandler) serveFetch(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) {
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
if err != nil {
var proxyErr *proxy.ProxyError
@@ -103,7 +118,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
slog.Error("proxy fetch failed", "remote", remote.Name, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
@@ -111,37 +126,43 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
var body io.Reader = result.Reader
if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
raw, err := io.ReadAll(result.Reader)
if err != nil {
http.Error(w, "bad gateway", http.StatusBadGateway)
raw, ok := rewriteServed(w, r, sr, *remote, result.Reader)
if !ok {
return
}
raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
body, result.Size = bytes.NewReader(raw), int64(len(raw))
}
w.Header().Set("Content-Type", result.ContentType)
w.Header().Set("X-Artifact-Source", result.Source)
if result.Size > 0 {
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
}
w.WriteHeader(http.StatusOK)
io.Copy(w, body)
}
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
func rewritesMutable(prov provider.Provider, path string) bool {
_, sr := prov.(provider.ServeRewriter)
_, rr := prov.(provider.ResponseRewriter)
return (sr || rr) && prov.Classify(path) == provider.Mutable
}
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
// handleProxyHead answers HEAD like GET without a body. Responses a provider
// synthesizes or rewrites per request go through the GET path so their headers
// match; everything else is answered from store metadata or an upstream HEAD.
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
remote, prov, ok := h.remoteFor(w, r)
if !ok {
return
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
path := chi.URLParam(r, "*")
if h.serveRemote(w, r, remote, prov, path) {
return
}
if rewritesMutable(prov, path) {
h.serveFetch(w, r, remote, prov, path)
return
}
@@ -152,7 +173,7 @@ func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
slog.Error("proxy head failed", "remote", remote.Name, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
@@ -175,7 +196,7 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
return
}
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
proxyBaseURL := fmt.Sprintf("%s://%s", Scheme(r), r.Host)
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
if err != nil {
@@ -266,12 +287,68 @@ func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, re
io.Copy(w, reader)
}
func scheme(r *http.Request) string {
// Scheme returns the client-facing scheme: TLS, else the first
// X-Forwarded-Proto value when it is http or https, else http.
func Scheme(r *http.Request) string {
if r.TLS != nil {
return "https"
}
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
fwd, _, _ := strings.Cut(r.Header.Get("X-Forwarded-Proto"), ",")
if fwd = strings.ToLower(strings.TrimSpace(fwd)); fwd == "https" {
return fwd
}
return "http"
}
// rewriteServed rewrites a cached body for the requesting host and marks the
// response uncacheable by shared caches, since it varies per Host.
func rewriteServed(w http.ResponseWriter, r *http.Request, sr provider.ServeRewriter, remote models.Remote, src io.Reader) ([]byte, bool) {
if !validHost(r.Host) {
http.Error(w, "invalid host header", http.StatusBadRequest)
return nil, false
}
raw, err := io.ReadAll(src)
if err != nil {
http.Error(w, "bad gateway", http.StatusBadGateway)
return nil, false
}
raw = sr.RewriteServed(raw, remote, fmt.Sprintf("%s://%s", Scheme(r), r.Host))
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
return raw, true
}
// validHost accepts host[:port] where host is a DNS name or IP literal.
func validHost(hostport string) bool {
host, port := hostport, ""
if h, p, err := net.SplitHostPort(hostport); err == nil {
if p == "" {
return false
}
host, port = h, p
} else if strings.HasPrefix(hostport, "[") {
return strings.HasSuffix(hostport, "]") && net.ParseIP(hostport[1:len(hostport)-1]) != nil
}
if port != "" {
if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
return false
}
}
if net.ParseIP(host) != nil {
return true
}
if strings.HasPrefix(hostport, "[") {
return false
}
if host == "" || len(host) > 253 {
return false
}
for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' ||
strings.Trim(strings.ToLower(label), "abcdefghijklmnopqrstuvwxyz0123456789-_") != "" {
return false
}
}
return true
}
+74
View File
@@ -0,0 +1,74 @@
package v1
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestValidHost(t *testing.T) {
for _, h := range []string{"aa.example", "aa.example:8443", "artifactapi.k8s.syd1.au.unkin.net", "10.0.0.1:80", "[::1]:8080", "[::1]", "::1", "localhost", "my_host.local"} {
if !validHost(h) {
t.Errorf("validHost(%q) = false, want true", h)
}
}
for _, h := range []string{"", "evil\".example", "a b", "aa.example:0", "aa.example:99999", "aa.example:x", "aa.example:", ":8080", "-bad.example", "a..b", "[::1", "[nope]:80", "a/b", "a<script>", strings.Repeat("a", 64) + ".example"} {
if validHost(h) {
t.Errorf("validHost(%q) = true, want false", h)
}
}
}
var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
const serviceIndex = `{"resources":[{"@id":"https://api.nuget.org/v3-flatcontainer/"}]}`
func rewrite(t *testing.T, host, proto string) (*httptest.ResponseRecorder, string, bool) {
t.Helper()
r := httptest.NewRequest(http.MethodGet, "/api/v1/remote/nuget-org/index.json", nil)
r.Host = host
if proto != "" {
r.Header.Set("X-Forwarded-Proto", proto)
}
w := httptest.NewRecorder()
raw, ok := rewriteServed(w, r, &nuget.Provider{}, nugetOrg, strings.NewReader(serviceIndex))
return w, string(raw), ok
}
func TestRewriteServedHeaders(t *testing.T) {
w, body, ok := rewrite(t, "aa.example:8443", "https")
if !ok {
t.Fatal("valid request rejected")
}
if want := `{"resources":[{"@id":"https://aa.example:8443/api/v1/remote/nuget-org/v3-flatcontainer/"}]}`; body != want {
t.Errorf("body = %s, want %s", body, want)
}
if got := w.Header().Get("Cache-Control"); got != "no-store" {
t.Errorf("Cache-Control = %q, want no-store", got)
}
if got := w.Header().Get("Vary"); got != "Host, X-Forwarded-Proto" {
t.Errorf("Vary = %q", got)
}
if got := w.Header().Get("Content-Length"); got != "91" || len(body) != 91 {
t.Errorf("Content-Length = %q, body len %d", got, len(body))
}
}
func TestRewriteServedIgnoresBogusProto(t *testing.T) {
_, plain, _ := rewrite(t, "aa.example", "")
_, bogus, _ := rewrite(t, "aa.example", "javascript")
if bogus != plain || !strings.Contains(bogus, `"http://aa.example/`) {
t.Errorf("bogus X-Forwarded-Proto changed the body: %s", bogus)
}
}
func TestRewriteServedRejectsInvalidHost(t *testing.T) {
w, _, ok := rewrite(t, `evil"},{"x":"`, "")
if ok || w.Code != http.StatusBadRequest {
t.Errorf("invalid host: ok=%v code=%d, want rejected with 400", ok, w.Code)
}
}
+41
View File
@@ -0,0 +1,41 @@
package v1
import (
"net/http"
"testing"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/npm"
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
)
func TestRoutesRemoteHead(t *testing.T) {
r := (&ProxyHandler{}).Routes()
for _, m := range []string{http.MethodGet, http.MethodHead} {
if !r.Match(chi.NewRouteContext(), m, "/remote/myremote/some/path") {
t.Errorf("%s /remote/myremote/some/path not routed", m)
}
}
}
func TestRewritesMutable(t *testing.T) {
tests := []struct {
name string
prov provider.Provider
path string
want bool
}{
{"npm packument", &npm.Provider{}, "left-pad", true},
{"npm tarball", &npm.Provider{}, "left-pad/-/left-pad-1.3.0.tgz", false},
{"nuget index", &nuget.Provider{}, "index.json", true},
{"rpm repodata", &rpm.Provider{}, "repodata/repomd.xml", false},
}
for _, tc := range tests {
if got := rewritesMutable(tc.prov, tc.path); got != tc.want {
t.Errorf("%s: rewritesMutable = %v, want %v", tc.name, got, tc.want)
}
}
}
+17 -3
View File
@@ -7,14 +7,28 @@ import (
)
func TestScheme(t *testing.T) {
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
if got := Scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
t.Errorf("TLS request scheme = %q, want https", got)
}
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
if got := scheme(r); got != "https" {
if got := Scheme(r); got != "https" {
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
}
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
for fwd, want := range map[string]string{
"https, http": "https",
"http, https": "http",
"HTTPS": "https",
" https ": "https",
"javascript": "http",
"ftp": "http",
"https;evil": "http",
} {
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {fwd}}}
if got := Scheme(r); got != want {
t.Errorf("X-Forwarded-Proto %q scheme = %q, want %s", fwd, got, want)
}
}
if got := Scheme(&http.Request{Header: http.Header{}}); got != "http" {
t.Errorf("default scheme = %q, want http", got)
}
}
+20 -10
View File
@@ -67,8 +67,12 @@ func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, remote)
}
// defaultMutableTTL matches the remotes.mutable_ttl column default; a create
// that omits mutable_ttl gets it, an explicit 0 still means never expire.
const defaultMutableTTL = 3600
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
var remote models.Remote
remote := models.Remote{MutableTTL: defaultMutableTTL}
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
@@ -114,7 +118,15 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
// Capture the current backend before the update so we can tell whether the
// remote's base_url (its upstream) changed. A read failure just means we
// skip the freshness flush; it must not block the update.
existing, err := h.db.GetRemote(r.Context(), name)
oldKnown := err == nil
var remote models.Remote
if oldKnown {
remote.MutableTTL = existing.MutableTTL
}
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
@@ -132,13 +144,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Capture the current backend before the update so we can tell whether the
// remote's base_url (its upstream) changed. A read failure just means we
// skip the freshness flush; it must not block the update.
oldBaseURL, oldKnown := "", false
if existing, err := h.db.GetRemote(r.Context(), name); err == nil {
oldBaseURL, oldKnown = existing.BaseURL, true
}
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@@ -147,13 +152,13 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
// Release / APKINDEX): purge it so the next request re-fetches from the new
// upstream instead of serving stale data until TTL expiry. A flush failure
// is logged but does not fail the request — the DB update already landed.
if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil {
if oldKnown && existing.BaseURL != remote.BaseURL && h.cache != nil {
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
slog.Warn("flush cached metadata after base_url change failed",
"remote", name, "error", err)
} else {
slog.Info("flushed cached metadata after base_url change",
"remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL)
"remote", name, "old_base_url", existing.BaseURL, "new_base_url", remote.BaseURL)
}
}
writeJSON(w, http.StatusOK, remote)
@@ -165,6 +170,11 @@ func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if h.cache != nil {
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
slog.Warn("flush cached metadata after delete failed", "remote", name, "error", err)
}
}
w.WriteHeader(http.StatusNoContent)
}
+25
View File
@@ -94,3 +94,28 @@ func TestUpdateFlushFailureStillSucceeds(t *testing.T) {
t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls)
}
}
// Deleting a remote flushes its cached state so a re-created remote of the
// same name starts cold; a flush error does not fail the delete.
func TestDeleteFlushesCache(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
for _, ff := range []*fakeFlusher{{}, {err: errors.New("redis down")}} {
const name = "rpm-flush-delete"
seedRemote(t, db, name, "https://example.com/repo")
h := NewRemotesHandler(db, ff, nil).Routes()
if c := do(t, h, "DELETE", "/"+name, ""); c != 204 {
t.Fatalf("delete = %d, want 204", c)
}
if len(ff.calls) != 1 || ff.calls[0] != name {
t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name)
}
}
}
+57
View File
@@ -0,0 +1,57 @@
package v2
import (
"context"
"testing"
"git.unkin.net/unkin/artifactapi/internal/database"
)
// An omitted mutable_ttl defaults to 3600 on create and keeps the stored value
// on update; an explicit 0 is stored as 0 in both.
func TestMutableTTLOmittedVsZero(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
h := NewRemotesHandler(db, nil, nil).Routes()
ttl := func(name string) int {
t.Helper()
r, err := db.GetRemote(context.Background(), name)
if err != nil {
t.Fatalf("get %s: %v", name, err)
}
return r.MutableTTL
}
const body = `"package_type":"rpm","base_url":"https://example.com/repo"`
steps := []struct {
method, path, json string
want int
}{
{"POST", "/", `{"name":"ttl-omit",` + body + `}`, 3600},
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":120}`, 120},
{"PUT", "/ttl-omit", `{` + body + `}`, 120},
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":0}`, 0},
{"PUT", "/ttl-omit", `{` + body + `}`, 0},
}
for i, s := range steps {
if c := do(t, h, s.method, s.path, s.json); c/100 != 2 {
t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c)
}
if got := ttl("ttl-omit"); got != s.want {
t.Errorf("step %d: mutable_ttl = %d, want %d", i, got, s.want)
}
}
if c := do(t, h, "POST", "/", `{"name":"ttl-zero",`+body+`,"mutable_ttl":0}`); c != 201 {
t.Fatalf("create explicit 0 = %d", c)
}
if got := ttl("ttl-zero"); got != 0 {
t.Errorf("explicit 0 stored as %d", got)
}
}