unkin/artifactapi
3
0
Fork 0
Code Issues 5 Pull Requests 4 Actions Packages Projects Releases 1 Wiki Activity

Merge benvin/remote-query-forwarder into benvin/remote-ansible
ci/woodpecker/pr/pre-commit Pipeline was successful
Details
ci/woodpecker/pr/build Pipeline was successful
Details
ci/woodpecker/pr/test Pipeline was successful
Details

Browse Source
This commit is contained in:
Unkin Agent
2026-10-18 03:16:10 +11:00
parent 285e8674ca 71e6458c77
commit f4d3093952
190 changed files with 8593 additions and 1107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/terraform/registry.go
+2 -8
View File
@@ -13,6 +13,7 @@ import (
"github.com/go-chi/chi/v5"
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
"git.unkin.net/unkin/artifactapi/internal/database"
tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
@@ -290,12 +291,5 @@ func writeJSON(w http.ResponseWriter, v any) {
}
func baseURL(r *http.Request) string {
scheme := "http"
if r.TLS != nil {
scheme = "https"
}
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
scheme = fwd
}
return scheme + "://" + r.Host
return v1.Scheme(r) + "://" + r.Host
}
internal/api/v1/gems.go
+308
View File
@@ -0,0 +1,308 @@
package v1
import (
"cmp"
"context"
"errors"
"fmt"
"io"
"log/slog"
"maps"
"net/http"
"net/url"
"slices"
"strings"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider/gems"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Local gems repos speak the RubyGems push/yank API and compact index. Per gem:
// gems/<full>.gem package (immutable)
// quick/Marshal.4.8/<full>.gemspec.rz marshalled spec (immutable)
// _index/<name>/<full> compact index /info line; gates visibility
// _yanked/<name>/<full> yank tombstone; blocks re-push of the version
var (
gemsMaxPushBytes int64 = 256 << 20
gemsLockWait = 30 * time.Second
)
const (
gemsIndexPrefix = "_index/"
gemsYankedPrefix = "_yanked/"
)
func gemsText(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
w.WriteHeader(status)
_, _ = io.WriteString(w, msg)
}
func (h *ProxyHandler) localGemsRemote(w http.ResponseWriter, r *http.Request) (*models.Remote, bool) {
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageGems {
gemsText(w, http.StatusMethodNotAllowed, "only supported on local gems repositories")
return nil, false
}
return remote, true
}
// lockGemVersion serialises push and yank of one gem version across replicas.
func (h *ProxyHandler) lockGemVersion(w http.ResponseWriter, r *http.Request, repo, full string) (func(), bool) {
unlock, err := h.engine.Lock(r.Context(), repo, "_gemlock/"+full, gemsLockWait, 5*time.Minute)
if err != nil {
gemsText(w, http.StatusServiceUnavailable, full+" is being pushed or yanked, retry")
return nil, false
}
return unlock, true
}
func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) {
if chi.URLParam(r, "*") != "api/v1/gems" {
gemsText(w, http.StatusNotFound, "not found")
return
}
remote, ok := h.localGemsRemote(w, r)
if !ok {
return
}
gem, err := io.ReadAll(http.MaxBytesReader(w, r.Body, gemsMaxPushBytes))
if err != nil {
gemsText(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("gem exceeds %d bytes", gemsMaxPushBytes))
return
}
spec, err := gems.ParseGem(gem)
if err != nil {
gemsText(w, http.StatusBadRequest, err.Error())
return
}
quick, err := gems.QuickSpec(spec)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
full := spec.FullName()
unlock, ok := h.lockGemVersion(w, r, remote.Name, full)
if !ok {
return
}
defer unlock()
conflict := func() {
gemsText(w, http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
}
index := gemsIndexPrefix + spec.Name + "/" + full
for _, p := range []string{gemsYankedPrefix + spec.Name + "/" + full, index} {
f, err := h.db.GetLocalFile(r.Context(), remote.Name, p)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if f != nil && p == index {
conflict()
return
}
if f != nil {
gemsText(w, http.StatusConflict, fmt.Sprintf("%s was yanked; re-pushing a yanked version is not allowed", full))
return
}
}
// The version is not live, so on failure every path is an invisible leftover and safe to drop.
paths := []string{"gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz", index}
fail := func(status int, msg string) {
for _, p := range paths {
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, p); err != nil {
slog.Error("gem push cleanup failed", "repo", remote.Name, "path", p, "error", err)
}
}
gemsText(w, status, msg)
}
// The index line gates visibility, so it is written last.
for _, f := range []struct {
path string
data []byte
}{
{paths[0], gem},
{paths[1], quick},
} {
ok, err := h.npmStoreImmutable(r.Context(), remote.Name, f.path, f.data, "application/octet-stream")
if err != nil {
fail(http.StatusInternalServerError, err.Error())
return
}
if !ok {
fail(http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
return
}
}
line := gems.InfoLine(spec, gem)
if _, err := h.npmStore(r.Context(), remote.Name, index, []byte(line), "text/plain", false); err != nil {
fail(http.StatusInternalServerError, err.Error())
return
}
slog.Info("local gem pushed", "repo", remote.Name, "gem", full)
gemsText(w, http.StatusOK, "Successfully registered gem: "+spec.Name+" ("+spec.Version+")")
}
func (h *ProxyHandler) handleLocalDelete(w http.ResponseWriter, r *http.Request) {
if chi.URLParam(r, "*") != "api/v1/gems/yank" {
gemsText(w, http.StatusNotFound, "not found")
return
}
remote, ok := h.localGemsRemote(w, r)
if !ok {
return
}
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
if err != nil {
gemsText(w, http.StatusRequestEntityTooLarge, "yank body too large")
return
}
form, err := url.ParseQuery(string(body))
if err != nil {
gemsText(w, http.StatusBadRequest, "invalid form body")
return
}
for k, v := range r.URL.Query() {
if form.Get(k) == "" {
form[k] = v
}
}
spec := &gems.Spec{Name: form.Get("gem_name"), Version: form.Get("version"), Platform: form.Get("platform")}
if spec.Platform == "" {
spec.Platform = "ruby"
}
if !gems.NameRe.MatchString(spec.Name) || !gems.VersionRe.MatchString(spec.Version) || !gems.PlatformRe.MatchString(spec.Platform) {
gemsText(w, http.StatusBadRequest, "invalid gem_name, version or platform")
return
}
full := spec.FullName()
unlock, ok := h.lockGemVersion(w, r, remote.Name, full)
if !ok {
return
}
defer unlock()
index := gemsIndexPrefix + spec.Name + "/" + full
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, index)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if existing == nil {
gemsText(w, http.StatusNotFound, "The version "+spec.Version+" does not exist")
return
}
// The tombstone keeps the yanked line and is written before any delete.
line, err := h.readLocalBlob(r.Context(), existing.ContentHash)
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
if _, err := h.npmStore(r.Context(), remote.Name, gemsYankedPrefix+spec.Name+"/"+full, line, "text/plain", false); err != nil && !errors.Is(err, database.ErrAlreadyExists) {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
for _, p := range []string{index, "gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz"} {
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, p); err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
}
slog.Info("local gem yanked", "repo", remote.Name, "gem", full)
gemsText(w, http.StatusOK, "Successfully deleted gem: "+spec.Name+" ("+spec.Version+")")
}
// isGemsIndexPath reports whether path is a compact index document.
func isGemsIndexPath(path string) bool {
return path == "versions" || path == "names" || strings.HasPrefix(path, "info/")
}
func (h *ProxyHandler) serveLocalGemsIndex(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) {
body, err := h.LocalGemsIndex(r.Context(), remote.Name, path)
if errors.Is(err, virtual.ErrNotFound) {
gemsText(w, http.StatusNotFound, "not found")
return
}
if err != nil {
gemsText(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("X-Artifact-Source", "local")
gemsText(w, http.StatusOK, string(body))
}
// LocalGemsIndex builds a local gems repo's versions, names or info/<gem>
// document, or returns virtual.ErrNotFound.
func (h *ProxyHandler) LocalGemsIndex(ctx context.Context, repo, path string) ([]byte, error) {
if name, ok := strings.CutPrefix(path, "info/"); ok {
if !gems.NameRe.MatchString(name) {
return nil, virtual.ErrNotFound
}
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix+name+"/")
if err != nil {
return nil, err
}
if len(lines[name]) == 0 {
return nil, virtual.ErrNotFound
}
return gems.InfoBody(lines[name]), nil
}
if path != "versions" && path != "names" {
return nil, virtual.ErrNotFound
}
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix)
if err != nil {
return nil, err
}
names := slices.Sorted(maps.Keys(lines))
var b strings.Builder
if path == "names" {
b.WriteString("---\n")
for _, n := range names {
b.WriteString(n + "\n")
}
return []byte(b.String()), nil
}
b.WriteString(gems.VersionsHeader(time.Now()))
for _, n := range names {
b.WriteString(gems.VersionsLine(n, gems.InfoBody(lines[n])) + "\n")
}
return []byte(b.String()), nil
}
// localGemsLines returns /info lines per gem name in push order.
func (h *ProxyHandler) localGemsLines(ctx context.Context, repo, prefix string) (map[string][]string, error) {
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
if err != nil {
return nil, err
}
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(a.ID, b.ID) })
out := map[string][]string{}
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
rest, ok := strings.CutPrefix(f.FilePath, prefix)
if !ok {
continue
}
name := strings.TrimPrefix(prefix, gemsIndexPrefix)
if name == "" {
name, _, _ = strings.Cut(rest, "/")
} else {
name = strings.TrimSuffix(name, "/")
}
line, err := h.readLocalBlob(ctx, f.ContentHash)
if err != nil {
return nil, err
}
out[name] = append(out[name], string(line))
}
return out, nil
}
internal/api/v1/gems_handler_test.go
+254
View File
@@ -0,0 +1,254 @@
package v1
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"fmt"
"net/http"
"strings"
"sync"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func testGemFile(t *testing.T, name, version, platform string) []byte {
t.Helper()
meta := fmt.Sprintf(`--- !ruby/object:Gem::Specification
name: %s
version: !ruby/object:Gem::Version
version: %s
platform: %s
dependencies:
- !ruby/object:Gem::Dependency
name: rack
requirement: !ruby/object:Gem::Requirement
requirements:
- - "~>"
- !ruby/object:Gem::Version
version: '3.0'
type: :runtime
prerelease: false
rubygems_version: 3.5.22
specification_version: 4
summary: test
`, name, version, platform)
var gz bytes.Buffer
zw := gzip.NewWriter(&gz)
_, _ = zw.Write([]byte(meta))
_ = zw.Close()
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
_ = tw.WriteHeader(&tar.Header{Name: "metadata.gz", Mode: 0o444, Size: int64(gz.Len())})
_, _ = tw.Write(gz.Bytes())
_ = tw.Close()
return buf.Bytes()
}
func sha256Hex(b []byte) string {
s := sha256.Sum256(b)
return hex.EncodeToString(s[:])
}
func TestGemsPushIndexYank(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageGems)
base := srv.URL + "/api/v1/local/" + repo + "/"
g1 := testGemFile(t, "hello_gem", "0.1.0", "ruby")
g2 := testGemFile(t, "hello_gem", "0.2.0", "ruby")
gj := testGemFile(t, "hello_gem", "0.2.0", "java")
other := testGemFile(t, "hello-gem", "1.0.0", "ruby")
for _, g := range [][]byte{g1, g2, gj, other} {
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g)); code != http.StatusOK {
t.Fatalf("push = %d: %s", code, body)
}
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)+"x"); code != http.StatusConflict {
t.Fatalf("repush with different bytes = %d, want 409", code)
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)); code != http.StatusConflict {
t.Fatalf("identical repush = %d, want 409", code)
}
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", "not a gem"); code != http.StatusBadRequest {
t.Fatalf("garbage push = %d, want 400", code)
}
code, info := npmDo(t, http.MethodGet, base+"info/hello_gem", "")
want := "---\n0.1.0 rack:~> 3.0|checksum:" + sha256Hex(g1) + "\n0.2.0 rack:~> 3.0|checksum:" + sha256Hex(g2) +
"\n0.2.0-java rack:~> 3.0|checksum:" + sha256Hex(gj) + "\n"
if code != http.StatusOK || info != want {
t.Fatalf("info = %d\n%q\nwant\n%q", code, info, want)
}
if code, _ := npmDo(t, http.MethodHead, base+"versions", ""); code != http.StatusOK {
t.Fatalf("HEAD versions = %d; gem probes the compact index with HEAD", code)
}
_, versions := npmDo(t, http.MethodGet, base+"versions", "")
head, body, ok := strings.Cut(versions, "\n---\n")
if !ok || !strings.HasPrefix(head, "created_at: ") {
t.Fatalf("versions header = %q", versions)
}
_, otherInfo := npmDo(t, http.MethodGet, base+"info/hello-gem", "")
wantBody := fmt.Sprintf("hello-gem 1.0.0 %x\nhello_gem 0.1.0,0.2.0,0.2.0-java %x\n", md5.Sum([]byte(otherInfo)), md5.Sum([]byte(info)))
if body != wantBody {
t.Fatalf("versions body =\n%q\nwant\n%q", body, wantBody)
}
if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello-gem\nhello_gem\n" {
t.Fatalf("names = %q", names)
}
if code, got := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusOK || got != string(gj) {
t.Fatalf("download = %d", code)
}
if code, _ := npmDo(t, http.MethodGet, base+"quick/Marshal.4.8/hello_gem-0.1.0.gemspec.rz", ""); code != http.StatusOK {
t.Fatalf("quick spec = %d", code)
}
if code, _ := npmDo(t, http.MethodGet, base+"info/missing", ""); code != http.StatusNotFound {
t.Fatalf("missing info = %d", code)
}
yank := func(form string) int {
req, _ := http.NewRequest(http.MethodDelete, base+"api/v1/gems/yank", strings.NewReader(form))
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
_ = resp.Body.Close()
return resp.StatusCode
}
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusOK {
t.Fatalf("yank = %d", c)
}
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusNotFound {
t.Fatalf("second yank = %d, want 404", c)
}
if c := yank("gem_name=../x&version=1"); c != http.StatusBadRequest {
t.Fatalf("bad yank = %d, want 400", c)
}
if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") || !strings.Contains(info, "0.2.0 ") {
t.Fatalf("info after yank = %q", info)
}
if code, _ := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusNotFound {
t.Fatalf("yanked download = %d, want 404", code)
}
if _, v := npmDo(t, http.MethodGet, base+"versions", ""); strings.Contains(v, "java") || !strings.Contains(v, "hello_gem 0.1.0,0.2.0 ") {
t.Fatalf("versions after yank = %q", v)
}
for _, g := range []string{string(gj), string(gj) + "x"} {
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", g); code != http.StatusConflict || !strings.Contains(body, "yanked") {
t.Fatalf("push after yank = %d %q, want 409", code, body)
}
}
if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") {
t.Fatalf("info after rejected re-push = %q", info)
}
if c := yank("gem_name=hello-gem&version=1.0.0"); c != http.StatusOK {
t.Fatalf("yank last version = %d", c)
}
if code, _ := npmDo(t, http.MethodGet, base+"info/hello-gem", ""); code != http.StatusNotFound {
t.Fatalf("info for fully yanked gem = %d, want 404", code)
}
if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello_gem\n" {
t.Fatalf("names after yanking every version = %q", names)
}
}
func TestGemsPushRejects(t *testing.T) {
h, srv := npmTestServer(t)
npmLocal := npmRepo(t, h, models.PackageNPM)
g := string(testGemFile(t, "g", "1.0.0", "ruby"))
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+npmLocal+"/api/v1/gems", g); code != http.StatusMethodNotAllowed {
t.Fatalf("push to npm local = %d, want 405", code)
}
repo := npmRepo(t, h, models.PackageGems)
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/other", g); code != http.StatusNotFound {
t.Fatalf("push to other path = %d, want 404", code)
}
old := gemsMaxPushBytes
gemsMaxPushBytes = 64
defer func() { gemsMaxPushBytes = old }()
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/api/v1/gems", g); code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized push = %d, want 413", code)
}
}
func TestGemsPushYankSerialised(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageGems)
base := srv.URL + "/api/v1/local/" + repo + "/"
g := string(testGemFile(t, "race", "1.0.0", "ruby"))
var wg sync.WaitGroup
for i := 0; i < 16; i++ {
wg.Add(2)
go func() {
defer wg.Done()
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", g); code != http.StatusOK && code != http.StatusConflict {
t.Errorf("push = %d: %s", code, body)
}
}()
go func() {
defer wg.Done()
if code, body := npmDo(t, http.MethodDelete, base+"api/v1/gems/yank?gem_name=race&version=1.0.0", ""); code != http.StatusOK && code != http.StatusNotFound {
t.Errorf("yank = %d: %s", code, body)
}
}()
}
wg.Wait()
present := func(p string) bool {
f, err := h.db.GetLocalFile(context.Background(), repo, p)
if err != nil {
t.Fatal(err)
}
return f != nil
}
live := []string{"_index/race/race-1.0.0", "gems/race-1.0.0.gem", "quick/Marshal.4.8/race-1.0.0.gemspec.rz"}
tomb := present("_yanked/race/race-1.0.0")
for _, p := range live {
if present(p) == tomb {
t.Errorf("tombstoned=%v but %s present=%v", tomb, p, !tomb)
}
}
_, info := npmDo(t, http.MethodGet, base+"info/race", "")
if strings.Contains(info, "1.0.0") == tomb {
t.Errorf("tombstoned=%v but info = %q", tomb, info)
}
}
func TestGemsLockTimeoutIs503(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageGems)
base := srv.URL + "/api/v1/local/" + repo + "/"
redis, err := cache.NewRedis(testRedis)
if err != nil {
t.Fatal(err)
}
token, err := redis.AcquireLock(context.Background(), repo, "_gemlock/busy-1.0.0", time.Minute)
if token == "" || err != nil {
t.Fatalf("hold lock = %q %v", token, err)
}
prev := gemsLockWait
gemsLockWait = 100 * time.Millisecond
t.Cleanup(func() { gemsLockWait = prev })
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(testGemFile(t, "busy", "1.0.0", "ruby"))); code != http.StatusServiceUnavailable {
t.Fatalf("push under held lock = %d, want 503", code)
}
if code, _ := npmDo(t, http.MethodDelete, base+"api/v1/gems/yank?gem_name=busy&version=1.0.0", ""); code != http.StatusServiceUnavailable {
t.Fatalf("yank under held lock = %d, want 503", code)
}
if err := redis.ReleaseLock(context.Background(), repo, "_gemlock/busy-1.0.0", token); err != nil {
t.Fatal(err)
}
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(testGemFile(t, "busy", "1.0.0", "ruby"))); code != http.StatusOK {
t.Fatalf("push after release = %d: %s", code, body)
}
}
internal/api/v1/gems_virtual.go
+39
View File
@@ -0,0 +1,39 @@
package v1
import (
"errors"
"log/slog"
"net/http"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// serveVirtualGems serves the merged compact index and 302s downloads to the
// member owning that version.
func (h *ProxyHandler) serveVirtualGems(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) {
var body []byte
var loc string
var err error
if isGemsIndexPath(path) {
body, _, err = h.virtualEngine.FetchGems(r.Context(), virt, path)
} else {
loc, err = h.virtualEngine.GemsRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL)
}
switch {
case errors.Is(err, virtual.ErrNotFound):
gemsText(w, http.StatusNotFound, "not found")
case errors.Is(err, virtual.ErrBadPath):
gemsText(w, http.StatusBadRequest, err.Error())
case err != nil:
slog.Error("virtual gems fetch failed", "virtual", virt.Name, "path", path, "error", err)
gemsText(w, http.StatusBadGateway, "bad gateway")
case loc != "":
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
http.Redirect(w, r, loc, http.StatusFound)
default:
w.Header().Set("X-Artifact-Source", "virtual")
gemsText(w, http.StatusOK, string(body))
}
}
internal/api/v1/npm.go
+334
View File
@@ -0,0 +1,334 @@
package v1
import (
"bytes"
"context"
"crypto/sha1"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"regexp"
"strings"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// Local npm repos speak the npm registry publish/read protocol. Per package:
// <pkg>/-/<base>-<version>.tgz tarball (immutable)
// <pkg>/-/<base>-<version>.json version manifest (immutable)
// <pkg>/-/dist-tags/<tag> version string (mutable)
var (
npmNameRe = regexp.MustCompile(`^(@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*$`)
npmRefRe = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z.+_-]*$`)
npmSemverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`)
npmMaxPublishBytes int64 = 256 << 20
)
func (h *ProxyHandler) localNPMRemote(r *http.Request, name string) (*models.Remote, bool) {
remote, err := h.db.GetRemote(r.Context(), name)
if err != nil {
return nil, false
}
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageNPM
}
// npmPackageName returns the package a packument/publish path addresses, or ""
// when the path is something else (e.g. a tarball under <pkg>/-/).
func npmPackageName(raw string) string {
p, err := url.PathUnescape(raw)
if err != nil {
return ""
}
p = strings.TrimSuffix(p, "/")
if !npmNameRe.MatchString(p) {
return ""
}
return p
}
type npmRelease struct {
version string
manifest []byte
tarball []byte
tags []string
}
func parseNPMPublish(body io.Reader, pkg string) (*npmRelease, error) {
var doc struct {
Name string `json:"name"`
DistTags map[string]string `json:"dist-tags"`
Versions map[string]json.RawMessage `json:"versions"`
Attachments map[string]struct {
Data string `json:"data"`
} `json:"_attachments"`
}
if err := json.NewDecoder(body).Decode(&doc); err != nil {
return nil, fmt.Errorf("invalid publish body: %w", err)
}
if doc.Name != pkg {
return nil, fmt.Errorf("body name %q does not match %q", doc.Name, pkg)
}
if len(doc.Versions) != 1 || len(doc.Attachments) != 1 {
return nil, errors.New("publish must carry exactly one version and one tarball")
}
rel := &npmRelease{}
for v, m := range doc.Versions {
rel.version, rel.manifest = v, m
}
if !npmSemverRe.MatchString(rel.version) {
return nil, fmt.Errorf("invalid version %q", rel.version)
}
var meta struct {
Name string `json:"name"`
Version string `json:"version"`
Dist struct {
Integrity string `json:"integrity"`
Shasum string `json:"shasum"`
} `json:"dist"`
}
if err := json.Unmarshal(rel.manifest, &meta); err != nil {
return nil, fmt.Errorf("invalid version manifest: %w", err)
}
if meta.Name != pkg || meta.Version != rel.version {
return nil, errors.New("version manifest name/version mismatch")
}
for _, a := range doc.Attachments {
tgz, err := base64.StdEncoding.DecodeString(a.Data)
if err != nil {
return nil, fmt.Errorf("invalid tarball encoding: %w", err)
}
rel.tarball = tgz
}
if err := verifyNPMDist(rel.tarball, meta.Dist.Integrity, meta.Dist.Shasum); err != nil {
return nil, err
}
for tag, v := range doc.DistTags {
if !npmRefRe.MatchString(tag) || v != rel.version {
return nil, fmt.Errorf("dist-tag %q must point at the published version", tag)
}
rel.tags = append(rel.tags, tag)
}
return rel, nil
}
func verifyNPMDist(tgz []byte, integrity, shasum string) error {
if b64, ok := strings.CutPrefix(integrity, "sha512-"); ok {
sum := sha512.Sum512(tgz)
if base64.StdEncoding.EncodeToString(sum[:]) != b64 {
return errors.New("tarball does not match dist.integrity")
}
return nil
}
if shasum != "" {
sum := sha1.Sum(tgz)
if hex.EncodeToString(sum[:]) != shasum {
return errors.New("tarball does not match dist.shasum")
}
return nil
}
return errors.New("version manifest carries no sha512 integrity or shasum")
}
func npmError(w http.ResponseWriter, status int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
}
func (h *ProxyHandler) handleLocalPut(w http.ResponseWriter, r *http.Request) {
remote, ok := h.localNPMRemote(r, chi.URLParam(r, "localName"))
if !ok {
npmError(w, http.StatusMethodNotAllowed, "publish is only supported on local npm repositories")
return
}
pkg := npmPackageName(chi.URLParam(r, "*"))
if pkg == "" {
npmError(w, http.StatusBadRequest, "invalid package name")
return
}
rel, err := parseNPMPublish(http.MaxBytesReader(w, r.Body, npmMaxPublishBytes), pkg)
if err != nil {
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
npmError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("publish body exceeds %d bytes", npmMaxPublishBytes))
return
}
npmError(w, http.StatusBadRequest, err.Error())
return
}
base := pkg[strings.LastIndex(pkg, "/")+1:]
stem := pkg + "/-/" + base + "-" + rel.version
// The manifest gates visibility and tags may only name visible versions, so
// the order is tarball, manifest, tags. A tarball left by a failed publish is
// accepted again only with identical bytes; an existing manifest always conflicts.
conflict := func() {
npmError(w, http.StatusConflict, fmt.Sprintf("%s@%s already exists; overwrites are not allowed", pkg, rel.version))
}
ok, err = h.npmStoreImmutable(r.Context(), remote.Name, stem+".tgz", rel.tarball, "application/octet-stream")
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
if !ok {
conflict()
return
}
if _, err := h.npmStore(r.Context(), remote.Name, stem+".json", rel.manifest, "application/json", false); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
conflict()
return
}
npmError(w, http.StatusInternalServerError, err.Error())
return
}
for _, tag := range rel.tags {
if _, err := h.npmStore(r.Context(), remote.Name, pkg+"/-/dist-tags/"+tag, []byte(rel.version), "text/plain", true); err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
}
slog.Info("local npm package published", "repo", remote.Name, "package", pkg, "version", rel.version)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "id": pkg})
}
// npmStoreImmutable reports false when path already holds different bytes.
func (h *ProxyHandler) npmStoreImmutable(ctx context.Context, repo, path string, data []byte, contentType string) (bool, error) {
hash, err := h.npmStore(ctx, repo, path, data, contentType, false)
if !errors.Is(err, database.ErrAlreadyExists) {
return err == nil, err
}
existing, err := h.db.GetLocalFile(ctx, repo, path)
if err != nil || existing == nil {
return false, fmt.Errorf("read existing %s: %v", path, err)
}
return existing.ContentHash == hash, nil
}
func (h *ProxyHandler) npmStore(ctx context.Context, repo, path string, data []byte, contentType string, mutable bool) (string, error) {
result, err := h.cas.Store(ctx, bytes.NewReader(data), contentType)
if err != nil {
return "", err
}
if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
return "", err
}
if mutable {
return result.ContentHash, h.db.UpsertLocalFile(ctx, repo, path, result.ContentHash)
}
return result.ContentHash, h.db.CreateLocalFile(ctx, repo, path, result.ContentHash)
}
func (h *ProxyHandler) readLocalBlob(ctx context.Context, contentHash string) ([]byte, error) {
reader, _, err := h.store.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
if err != nil {
return nil, err
}
defer reader.Close()
return io.ReadAll(reader)
}
func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request, remote *models.Remote, pkg string) {
files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, pkg+"/-/")
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
base, ok := requestBaseURL(w, r)
if !ok {
return
}
read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
tarballBase := base + "/api/v1/local/" + remote.Name + "/"
doc, err := buildNPMPackument(pkg, files, read, tarballBase)
if err != nil {
npmError(w, http.StatusInternalServerError, err.Error())
return
}
if doc == nil {
npmError(w, http.StatusNotFound, "not found")
return
}
setNPMPerHostHeaders(w)
w.Header().Set("Content-Type", "application/json")
w.Header().Set("X-Artifact-Source", "local")
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(doc)
}
// setNPMPerHostHeaders keeps shared caches off packuments whose tarball URLs
// are built from the request host and Accept (abbreviated vs full).
func setNPMPerHostHeaders(w http.ResponseWriter) {
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto, Accept")
}
// buildNPMPackument returns nil when pkg has no published versions.
func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) {
prefix := pkg + "/-/"
versions := map[string]any{}
tags := map[string]string{}
for _, f := range files {
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
rest, ok := strings.CutPrefix(f.FilePath, prefix)
if !ok {
continue
}
if tag, isTag := strings.CutPrefix(rest, "dist-tags/"); isTag {
v, err := read(f.ContentHash)
if err != nil {
return nil, err
}
tags[tag] = string(v)
continue
}
stem, isManifest := strings.CutSuffix(rest, ".json")
if !isManifest {
continue
}
raw, err := read(f.ContentHash)
if err != nil {
return nil, err
}
var m map[string]any
if err := json.Unmarshal(raw, &m); err != nil {
return nil, fmt.Errorf("manifest %s: %w", f.FilePath, err)
}
dist, _ := m["dist"].(map[string]any)
if dist == nil {
dist = map[string]any{}
m["dist"] = dist
}
dist["tarball"] = tarballBase + prefix + stem + ".tgz"
v, _ := m["version"].(string)
versions[v] = m
}
if len(versions) == 0 {
return nil, nil
}
for tag, v := range tags {
if _, ok := versions[v]; !ok {
delete(tags, tag)
}
}
return map[string]any{"_id": pkg, "name": pkg, "dist-tags": tags, "versions": versions}, nil
}
internal/api/v1/npm_handler_test.go
+347
View File
@@ -0,0 +1,347 @@
package v1
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var (
testDSN string
testMinio testsupport.MinioConn
testRedis string
)
func TestMain(m *testing.M) {
ctx := context.Background()
dsn, termPG, err := testsupport.StartPostgres(ctx)
if err != nil {
os.Exit(m.Run())
}
conn, termMinio, err := testsupport.StartMinio(ctx)
if err != nil {
termPG()
os.Exit(m.Run())
}
redisURL, termRedis, err := testsupport.StartRedis(ctx)
if err != nil {
termMinio()
termPG()
os.Exit(m.Run())
}
testDSN, testMinio, testRedis = dsn, conn, redisURL
code := m.Run()
termRedis()
termMinio()
termPG()
os.Exit(code)
}
func npmTestServer(t *testing.T) (*ProxyHandler, *httptest.Server) {
t.Helper()
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
t.Cleanup(db.Close)
var store *storage.S3
for i := 0; i < 20; i++ {
if store, err = storage.NewS3(testMinio.Endpoint, testMinio.AccessKey, testMinio.SecretKey, "npm", false, ""); err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
if err != nil {
t.Fatal(err)
}
redis, err := cache.NewRedis(testRedis)
if err != nil {
t.Fatal(err)
}
h := NewProxyHandler(proxy.NewEngine(db, redis, store), nil, db, store, nil)
r := chi.NewRouter()
r.Mount("/api/v1", h.Routes())
srv := httptest.NewServer(r)
t.Cleanup(srv.Close)
return h, srv
}
func npmRepo(t *testing.T, h *ProxyHandler, pt models.PackageType) string {
t.Helper()
name := fmt.Sprintf("npm-%s-%d", pt, time.Now().UnixNano())
if err := h.db.CreateRemote(context.Background(), &models.Remote{Name: name, PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
t.Fatal(err)
}
return name
}
func npmDo(t *testing.T, method, url, body string) (int, string) {
t.Helper()
req, _ := http.NewRequest(method, url, strings.NewReader(body))
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
b, _ := io.ReadAll(resp.Body)
return resp.StatusCode, string(b)
}
func packumentVersions(t *testing.T, url string) map[string]any {
t.Helper()
code, body := npmDo(t, http.MethodGet, url, "")
if code != http.StatusOK {
t.Fatalf("GET %s = %d: %s", url, code, body)
}
var doc struct {
Versions map[string]any `json:"versions"`
}
if err := json.Unmarshal([]byte(body), &doc); err != nil {
t.Fatal(err)
}
return doc.Versions
}
func packumentTags(t *testing.T, url string) map[string]string {
t.Helper()
_, body := npmDo(t, http.MethodGet, url, "")
var doc struct {
Tags map[string]string `json:"dist-tags"`
}
if err := json.Unmarshal([]byte(body), &doc); err != nil {
t.Fatal(err)
}
return doc.Tags
}
func concurrentPuts(t *testing.T, url string, bodies ...string) []int {
t.Helper()
var wg sync.WaitGroup
codes := make([]int, len(bodies))
for i, b := range bodies {
wg.Add(1)
go func() {
defer wg.Done()
codes[i], _ = npmDo(t, http.MethodPut, url, b)
}()
}
wg.Wait()
return codes
}
func TestNPMPublishHandler(t *testing.T) {
h, srv := npmTestServer(t)
repo := npmRepo(t, h, models.PackageNPM)
base := srv.URL + "/api/v1/local/" + repo + "/"
t.Run("scoped name via %2f", func(t *testing.T) {
body := publishBody(t, "@acme/widget", []byte("scoped tgz"), publishOpts{})
if code, b := npmDo(t, http.MethodPut, base+"@acme%2fwidget", body); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
v := packumentVersions(t, base+"@acme%2fwidget")
dist := v["1.0.0"].(map[string]any)["dist"].(map[string]any)
tarball := dist["tarball"].(string)
if !strings.HasPrefix(tarball, base+"@acme/widget/-/widget-1.0.0.tgz") {
t.Fatalf("tarball = %s", tarball)
}
if code, b := npmDo(t, http.MethodGet, tarball, ""); code != http.StatusOK || b != "scoped tgz" {
t.Fatalf("tarball fetch = %d %q", code, b)
}
})
t.Run("republish", func(t *testing.T) {
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("first publish = %d: %s", code, b)
}
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("different"), publishOpts{})); code != http.StatusConflict {
t.Errorf("different bytes = %d, want 409", code)
}
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v2"), publishOpts{version: "1.1.0"})); code != http.StatusCreated {
t.Fatalf("publish 1.1.0 = %d: %s", code, b)
}
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusConflict {
t.Errorf("identical republish = %d, want 409", code)
}
if tags := packumentTags(t, base+"repub"); tags["latest"] != "1.1.0" {
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
}
})
t.Run("rejected republish leaves tags", func(t *testing.T) {
for _, v := range []string{"1.0.0", "1.1.0"} {
if code, b := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz "+v), publishOpts{version: v})); code != http.StatusCreated {
t.Fatalf("publish %s = %d: %s", v, code, b)
}
}
if code, _ := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz 1.0.0"), publishOpts{shasum: "x"})); code != http.StatusConflict {
t.Fatalf("re-PUT 1.0.0 = %d, want 409", code)
}
if tags := packumentTags(t, base+"rej"); tags["latest"] != "1.1.0" {
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
}
})
t.Run("mid-publish window", func(t *testing.T) {
if code, b := npmDo(t, http.MethodPut, base+"mid", publishBody(t, "mid", []byte("v1"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
ctx := context.Background()
if _, err := h.npmStore(ctx, repo, "mid/-/mid-1.0.1.tgz", []byte("v2"), "application/octet-stream", false); err != nil {
t.Fatal(err)
}
if tags := packumentTags(t, base+"mid"); tags["latest"] != "1.0.0" {
t.Errorf("tarball-only 1.0.1: latest = %q", tags["latest"])
}
if _, err := h.npmStore(ctx, repo, "mid/-/dist-tags/latest", []byte("1.0.1"), "text/plain", true); err != nil {
t.Fatal(err)
}
if tags := packumentTags(t, base+"mid"); tags["latest"] == "1.0.1" {
t.Error("latest points at invisible 1.0.1")
}
})
t.Run("concurrent identical", func(t *testing.T) {
body := publishBody(t, "same", []byte("same tgz"), publishOpts{})
codes := concurrentPuts(t, base+"same", body, body)
if codes[0]+codes[1] != http.StatusCreated+http.StatusConflict || codes[0] == codes[1] {
t.Fatalf("codes = %v, want one 201 and one 409", codes)
}
if tags := packumentTags(t, base+"same"); len(tags) != 1 || tags["latest"] != "1.0.0" {
t.Errorf("tags = %v", tags)
}
})
t.Run("concurrent different manifests", func(t *testing.T) {
tgz := []byte("race tgz")
codes := concurrentPuts(t, base+"race",
publishBody(t, "race", tgz, publishOpts{shasum: "a", tags: map[string]string{"a": "1.0.0"}}),
publishBody(t, "race", tgz, publishOpts{shasum: "b", tags: map[string]string{"b": "1.0.0"}}))
won := map[int]string{}
for i, c := range codes {
won[c] = []string{"a", "b"}[i]
}
if len(won) != 2 || won[http.StatusCreated] == "" || won[http.StatusConflict] == "" {
t.Fatalf("codes = %v, want one 201 and one 409", codes)
}
v := packumentVersions(t, base+"race")["1.0.0"].(map[string]any)
if got := v["dist"].(map[string]any)["shasum"]; got != won[http.StatusCreated] {
t.Errorf("manifest shasum = %v, winner %s", got, won[http.StatusCreated])
}
tags := packumentTags(t, base+"race")
if len(tags) != 1 || tags[won[http.StatusCreated]] != "1.0.0" {
t.Errorf("tags = %v, winner %s", tags, won[http.StatusCreated])
}
})
t.Run("retry after tarball-only write", func(t *testing.T) {
tgz := []byte("half published")
if _, err := h.npmStore(context.Background(), repo, "retry/-/retry-1.0.0.tgz", tgz, "application/octet-stream", false); err != nil {
t.Fatal(err)
}
if code, _ := npmDo(t, http.MethodGet, base+"retry", ""); code != http.StatusNotFound {
t.Fatalf("tarball-only version visible: %d", code)
}
if code, _ := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", []byte("other bytes"), publishOpts{})); code != http.StatusConflict {
t.Errorf("retry with different bytes = %d, want 409", code)
}
if code, b := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", tgz, publishOpts{})); code != http.StatusCreated {
t.Fatalf("retry with identical bytes = %d: %s", code, b)
}
if _, ok := packumentVersions(t, base+"retry")["1.0.0"]; !ok {
t.Error("retried version missing from packument")
}
})
t.Run("concurrent versions", func(t *testing.T) {
codes := concurrentPuts(t, base+"conc",
publishBody(t, "conc", []byte("tgz 1.0.0"), publishOpts{version: "1.0.0"}),
publishBody(t, "conc", []byte("tgz 2.0.0"), publishOpts{version: "2.0.0"}))
if codes[0] != http.StatusCreated || codes[1] != http.StatusCreated {
t.Fatalf("codes = %v", codes)
}
v := packumentVersions(t, base+"conc")
if _, ok := v["1.0.0"]; !ok {
t.Error("1.0.0 missing")
}
if _, ok := v["2.0.0"]; !ok {
t.Error("2.0.0 missing")
}
})
t.Run("body too large", func(t *testing.T) {
old := npmMaxPublishBytes
npmMaxPublishBytes = 64
defer func() { npmMaxPublishBytes = old }()
if code, _ := npmDo(t, http.MethodPut, base+"big", publishBody(t, "big", []byte("x"), publishOpts{})); code != http.StatusRequestEntityTooLarge {
t.Errorf("oversized body = %d, want 413", code)
}
})
t.Run("non-npm local", func(t *testing.T) {
generic := npmRepo(t, h, models.PackageGeneric)
if code, _ := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+generic+"/pkg", publishBody(t, "pkg", []byte("x"), publishOpts{})); code != http.StatusMethodNotAllowed {
t.Errorf("publish to generic local = %d, want 405", code)
}
})
}
func TestNPMPackumentPerHost(t *testing.T) {
h, srv := npmTestServer(t)
h.virtualEngine = virtual.NewEngine(h.db, nil)
h.virtualEngine.NPMLocal = h.LocalNPMPackument
repo := npmRepo(t, h, models.PackageNPM)
virt := "v" + repo
if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageNPM, Members: []string{repo}}); err != nil {
t.Fatal(err)
}
if code, b := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+repo+"/hostpkg", publishBody(t, "hostpkg", []byte("tgz"), publishOpts{})); code != http.StatusCreated {
t.Fatalf("publish = %d: %s", code, b)
}
routes := h.Routes()
get := func(path, host, proto string) *httptest.ResponseRecorder {
r := httptest.NewRequest(http.MethodGet, path, nil)
r.Host = host
if proto != "" {
r.Header.Set("X-Forwarded-Proto", proto)
}
w := httptest.NewRecorder()
routes.ServeHTTP(w, r)
return w
}
for _, path := range []string{"/local/" + repo + "/hostpkg", "/virtual/" + virt + "/hostpkg"} {
for _, host := range []string{"x@evil.com", "evil.example:"} {
if w := get(path, host, ""); w.Code != http.StatusBadRequest {
t.Errorf("%s Host %q = %d, want 400", path, host, w.Code)
}
}
w := get(path, "aa.example", "https")
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"tarball":"https://aa.example/api/v1/`) {
t.Fatalf("%s = %d: %s", path, w.Code, w.Body)
}
if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto, Accept" {
t.Errorf("%s Cache-Control %q Vary %q", path, cc, vary)
}
}
}
internal/api/v1/npm_test.go
+166
View File
@@ -0,0 +1,166 @@
package v1
import (
"crypto/sha1"
"crypto/sha512"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/database"
)
func TestNPMPackageName(t *testing.T) {
tests := map[string]string{
"left-pad": "left-pad",
"left-pad/": "left-pad",
"@acme%2fwidget": "@acme/widget",
"@acme%2Fwidget": "@acme/widget",
"@acme/widget": "@acme/widget",
"left-pad/-/left-pad.tgz": "",
"@acme/widget/-/w-1.tgz": "",
"a/b": "",
"../etc": "",
"Upper": "",
"": "",
}
for in, want := range tests {
if got := npmPackageName(in); got != want {
t.Errorf("npmPackageName(%q) = %q, want %q", in, got, want)
}
}
}
type publishOpts struct {
bodyName, manifestName, version, integrity, shasum string
tags map[string]string
extraVersion bool
}
func publishBody(t *testing.T, pkg string, tgz []byte, o publishOpts) string {
t.Helper()
sha := sha512.Sum512(tgz)
sh1 := sha1.Sum(tgz)
if o.bodyName == "" {
o.bodyName = pkg
}
if o.manifestName == "" {
o.manifestName = pkg
}
if o.version == "" {
o.version = "1.0.0"
}
if o.integrity == "" {
o.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha[:])
}
if o.shasum == "" {
o.shasum = hex.EncodeToString(sh1[:])
}
if o.tags == nil {
o.tags = map[string]string{"latest": o.version}
}
versions := map[string]any{o.version: map[string]any{
"name": o.manifestName, "version": o.version, "main": "index.js",
"dist": map[string]any{"integrity": o.integrity, "shasum": o.shasum, "tarball": "http://elsewhere/x.tgz"},
}}
if o.extraVersion {
versions["2.0.0"] = map[string]any{"name": pkg, "version": "2.0.0"}
}
b, _ := json.Marshal(map[string]any{
"_id": pkg, "name": o.bodyName, "dist-tags": o.tags, "versions": versions,
"_attachments": map[string]any{pkg + "-" + o.version + ".tgz": map[string]any{
"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz),
}},
})
return string(b)
}
func TestParseNPMPublish(t *testing.T) {
tgz := []byte("fake tarball bytes")
rel, err := parseNPMPublish(strings.NewReader(publishBody(t, "@acme/widget", tgz, publishOpts{tags: map[string]string{"latest": "1.0.0", "beta": "1.0.0"}})), "@acme/widget")
if err != nil {
t.Fatalf("valid scoped publish rejected: %v", err)
}
if rel.version != "1.0.0" || string(rel.tarball) != string(tgz) || len(rel.tags) != 2 {
t.Fatalf("unexpected release: %+v", rel)
}
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none", shasum: hex.EncodeToString(make([]byte, 20))})), "left-pad"); err == nil {
t.Fatal("shasum fallback should reject a mismatched tarball")
}
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none"})), "left-pad"); err != nil {
t.Fatalf("matching shasum fallback rejected: %v", err)
}
bad := map[string]publishOpts{
"body name mismatch": {bodyName: "other"},
"manifest name mismatch": {manifestName: "other"},
"integrity mismatch": {integrity: "sha512-AAAA"},
"path-like version": {version: "../1.0.0"},
"non-semver version": {version: "1..2"},
"leading-zero version": {version: "01.0.0"},
"tag at another version": {tags: map[string]string{"latest": "9.9.9"}},
"path-like tag": {tags: map[string]string{"../x": "1.0.0"}},
"multiple versions at once": {extraVersion: true},
}
for name, o := range bad {
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, o)), "left-pad"); err == nil {
t.Errorf("%s: expected rejection", name)
}
}
if _, err := parseNPMPublish(strings.NewReader(`{"name":"left-pad","versions":{}}`), "left-pad"); err == nil {
t.Error("deprecate-style body without attachments should be rejected")
}
}
func TestBuildNPMPackument(t *testing.T) {
blobs := map[string]string{
"h1": `{"name":"my_pkg","version":"1.0.0","dist":{"shasum":"aa","tarball":"http://elsewhere/x.tgz"}}`,
"h2": `{"name":"my_pkg","version":"1.1.0"}`,
"h3": "1.1.0",
"h4": "2.0.0",
"hx": `{"name":"myXpkg","version":"6.6.6"}`,
}
read := func(h string) ([]byte, error) {
b, ok := blobs[h]
if !ok {
return nil, errors.New("missing blob")
}
return []byte(b), nil
}
files := []database.LocalFile{
{FilePath: "my_pkg/-/dist-tags/latest", ContentHash: "h3"},
{FilePath: "my_pkg/-/dist-tags/next", ContentHash: "h4"},
{FilePath: "my_pkg/-/my_pkg-1.0.0.json", ContentHash: "h1"},
{FilePath: "my_pkg/-/my_pkg-1.0.0.tgz", ContentHash: "nope"},
{FilePath: "my_pkg/-/my_pkg-1.1.0.json", ContentHash: "h2"},
{FilePath: "myXpkg/-/myXpkg-6.6.6.json", ContentHash: "hx"},
}
doc, err := buildNPMPackument("my_pkg", files, read, "http://reg/api/v1/local/npm/")
if err != nil {
t.Fatal(err)
}
if tags := doc["dist-tags"].(map[string]string); tags["latest"] != "1.1.0" || len(tags) != 1 {
t.Errorf("dist-tags = %v", tags)
}
versions := doc["versions"].(map[string]any)
if len(versions) != 2 {
t.Fatalf("versions = %v (LIKE over-match must be dropped)", versions)
}
v1 := versions["1.0.0"].(map[string]any)["dist"].(map[string]any)
if v1["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.0.0.tgz" || v1["shasum"] != "aa" {
t.Errorf("1.0.0 dist = %v", v1)
}
if versions["1.1.0"].(map[string]any)["dist"].(map[string]any)["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.1.0.tgz" {
t.Error("1.1.0 missing rewritten tarball")
}
if doc, err := buildNPMPackument("absent", files, read, "x/"); err != nil || doc != nil {
t.Errorf("absent package: doc=%v err=%v", doc, err)
}
}
internal/api/v1/npm_virtual.go
+26
View File
@@ -0,0 +1,26 @@
package v1
import (
"context"
"encoding/json"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
// LocalNPMPackument serves a local npm repo's packument to npm virtuals, with
// tarball paths relative to the repo.
func (h *ProxyHandler) LocalNPMPackument(ctx context.Context, repo, pkg string) ([]byte, error) {
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, pkg+"/-/")
if err != nil {
return nil, err
}
read := func(hash string) ([]byte, error) { return h.readLocalBlob(ctx, hash) }
doc, err := buildNPMPackument(pkg, files, read, "")
if err != nil {
return nil, err
}
if doc == nil {
return nil, virtual.ErrNotFound
}
return json.Marshal(doc)
}
internal/api/v1/proxy.go
+162 -33
View File
@@ -6,8 +6,11 @@ import (
"fmt"
"io"
"log/slog"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/go-chi/chi/v5"
@@ -17,6 +20,7 @@ import (
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type ProxyHandler struct {
@@ -42,8 +46,14 @@ func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *da
func (h *ProxyHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/remote/{remoteName}/*", h.handleProxy)
r.Head("/remote/{remoteName}/*", h.handleProxyHead)
r.Get("/local/{localName}/*", h.handleLocal)
r.Head("/local/{localName}/*", h.handleLocal)
r.Put("/local/{localName}/*", h.handleLocalPut)
r.Post("/local/{localName}/*", h.handleLocalPost)
r.Delete("/local/{localName}/*", h.handleLocalDelete)
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
r.Head("/virtual/{virtualName}/*", h.handleVirtual)
return r
}
@@ -69,31 +79,43 @@ func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request)
w.WriteHeader(http.StatusOK)
}
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
// remoteFor resolves the remote and provider named in the request, writing the
// error response itself when either is missing.
func (h *ProxyHandler) remoteFor(w http.ResponseWriter, r *http.Request) (*models.Remote, provider.Provider, bool) {
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
return
return nil, nil, false
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
return nil, nil, false
}
return remote, prov, true
}
// serveRemote lets metadata-only remotes (e.g. github_rpm) synthesize their own
// responses and redirect package downloads instead of proxying bytes.
func (h *ProxyHandler) serveRemote(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) bool {
rs, ok := prov.(provider.RemoteServer)
return ok && rs.ServeRemote(w, r, *remote, path, fmt.Sprintf("%s://%s", Scheme(r), r.Host), h.db)
}
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
remote, prov, ok := h.remoteFor(w, r)
if !ok {
return
}
// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
// redirect package downloads to a backend remote instead of proxying bytes.
if rs, ok := prov.(provider.RemoteServer); ok {
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
return
}
path := chi.URLParam(r, "*")
if h.serveRemote(w, r, remote, prov, path) {
return
}
h.serveFetch(w, r, remote, prov, path)
}
func (h *ProxyHandler) serveFetch(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) {
result, err := h.engine.Fetch(r.Context(), *remote, forwardQuery(prov, path, r.URL.Query()), prov, r.Header)
if err != nil {
var proxyErr *proxy.ProxyError
@@ -101,7 +123,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
slog.Error("proxy fetch failed", "remote", remote.Name, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
@@ -109,37 +131,43 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
var body io.Reader = result.Reader
if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
raw, err := io.ReadAll(result.Reader)
if err != nil {
http.Error(w, "bad gateway", http.StatusBadGateway)
raw, ok := rewriteServed(w, r, sr, *remote, result.Reader)
if !ok {
return
}
raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
body, result.Size = bytes.NewReader(raw), int64(len(raw))
}
w.Header().Set("Content-Type", result.ContentType)
w.Header().Set("X-Artifact-Source", result.Source)
if result.Size > 0 {
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
}
w.WriteHeader(http.StatusOK)
io.Copy(w, body)
}
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
func rewritesMutable(prov provider.Provider, path string) bool {
_, sr := prov.(provider.ServeRewriter)
_, rr := prov.(provider.ResponseRewriter)
return (sr || rr) && prov.Classify(path) == provider.Mutable
}
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
// handleProxyHead answers HEAD like GET without a body. Responses a provider
// synthesizes or rewrites per request go through the GET path so their headers
// match; everything else is answered from store metadata or an upstream HEAD.
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
remote, prov, ok := h.remoteFor(w, r)
if !ok {
return
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
path := chi.URLParam(r, "*")
if h.serveRemote(w, r, remote, prov, path) {
return
}
if rewritesMutable(prov, path) {
h.serveFetch(w, r, remote, prov, path)
return
}
@@ -150,7 +178,7 @@ func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
slog.Error("proxy head failed", "remote", remote.Name, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
@@ -173,7 +201,15 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
return
}
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
proxyBaseURL, ok := requestBaseURL(w, r)
if !ok {
return
}
if virt.PackageType == models.PackageGems {
h.serveVirtualGems(w, r, *virt, path, proxyBaseURL)
return
}
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
if err != nil {
@@ -185,7 +221,14 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
return
}
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
var body []byte
var contentType string
if virt.PackageType == models.PackageNPM {
abbreviated := strings.Contains(r.Header.Get("Accept"), virtual.NPMAbbreviatedType)
body, contentType, err = h.virtualEngine.FetchNPM(r.Context(), *virt, path, proxyBaseURL, abbreviated)
} else {
body, contentType, err = h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
}
if errors.Is(err, virtual.ErrNotFound) {
http.Error(w, "not found", http.StatusNotFound)
return
@@ -196,6 +239,9 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
return
}
if virt.PackageType == models.PackageNPM {
setNPMPerHostHeaders(w)
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("X-Artifact-Source", "virtual")
w.WriteHeader(http.StatusOK)
@@ -212,6 +258,23 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
return
}
if remote.PackageType == models.PackageNPM {
if pkg := npmPackageName(path); pkg != "" {
h.serveNPMPackument(w, r, remote, pkg)
return
}
}
if remote.PackageType == models.PackageGems {
if isGemsIndexPath(path) {
h.serveLocalGemsIndex(w, r, remote, path)
return
}
if strings.HasPrefix(path, gemsIndexPrefix) || strings.HasPrefix(path, gemsYankedPrefix) {
http.Error(w, "not found", http.StatusNotFound)
return
}
}
prov, _ := provider.Get(remote.PackageType)
if indexer, ok := prov.(provider.LocalIndexer); ok {
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
@@ -250,11 +313,14 @@ func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, re
io.Copy(w, reader)
}
func scheme(r *http.Request) string {
// Scheme returns the client-facing scheme: TLS, else the first
// X-Forwarded-Proto value when it is http or https, else http.
func Scheme(r *http.Request) string {
if r.TLS != nil {
return "https"
}
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
fwd, _, _ := strings.Cut(r.Header.Get("X-Forwarded-Proto"), ",")
if fwd = strings.ToLower(strings.TrimSpace(fwd)); fwd == "https" {
return fwd
}
return "http"
@@ -278,3 +344,66 @@ func forwardQuery(prov provider.Provider, path string, query url.Values) string
}
return path + "?" + kept.Encode()
}
// rewriteServed rewrites a cached body for the requesting host and marks the
// response uncacheable by shared caches, since it varies per Host.
func rewriteServed(w http.ResponseWriter, r *http.Request, sr provider.ServeRewriter, remote models.Remote, src io.Reader) ([]byte, bool) {
base, ok := requestBaseURL(w, r)
if !ok {
return nil, false
}
raw, err := io.ReadAll(src)
if err != nil {
http.Error(w, "bad gateway", http.StatusBadGateway)
return nil, false
}
raw = sr.RewriteServed(raw, remote, base)
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
return raw, true
}
// requestBaseURL returns the client-facing scheme://host, answering 400 on an
// invalid Host.
func requestBaseURL(w http.ResponseWriter, r *http.Request) (string, bool) {
if !validHost(r.Host) {
http.Error(w, "invalid host header", http.StatusBadRequest)
return "", false
}
return Scheme(r) + "://" + r.Host, true
}
// validHost accepts host[:port] where host is a DNS name or IP literal.
func validHost(hostport string) bool {
host, port := hostport, ""
if h, p, err := net.SplitHostPort(hostport); err == nil {
if p == "" {
return false
}
host, port = h, p
} else if strings.HasPrefix(hostport, "[") {
return strings.HasSuffix(hostport, "]") && net.ParseIP(hostport[1:len(hostport)-1]) != nil
}
if port != "" {
if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
return false
}
}
if net.ParseIP(host) != nil {
return true
}
if strings.HasPrefix(hostport, "[") {
return false
}
if host == "" || len(host) > 253 {
return false
}
for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' ||
strings.Trim(strings.ToLower(label), "abcdefghijklmnopqrstuvwxyz0123456789-_") != "" {
return false
}
}
return true
}
internal/api/v1/rewrite_test.go
+74
View File
@@ -0,0 +1,74 @@
package v1
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestValidHost(t *testing.T) {
for _, h := range []string{"aa.example", "aa.example:8443", "artifactapi.k8s.syd1.au.unkin.net", "10.0.0.1:80", "[::1]:8080", "[::1]", "::1", "localhost", "my_host.local"} {
if !validHost(h) {
t.Errorf("validHost(%q) = false, want true", h)
}
}
for _, h := range []string{"", "evil\".example", "a b", "aa.example:0", "aa.example:99999", "aa.example:x", "aa.example:", ":8080", "-bad.example", "a..b", "[::1", "[nope]:80", "a/b", "a<script>", strings.Repeat("a", 64) + ".example"} {
if validHost(h) {
t.Errorf("validHost(%q) = true, want false", h)
}
}
}
var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
const serviceIndex = `{"resources":[{"@id":"https://api.nuget.org/v3-flatcontainer/"}]}`
func rewrite(t *testing.T, host, proto string) (*httptest.ResponseRecorder, string, bool) {
t.Helper()
r := httptest.NewRequest(http.MethodGet, "/api/v1/remote/nuget-org/index.json", nil)
r.Host = host
if proto != "" {
r.Header.Set("X-Forwarded-Proto", proto)
}
w := httptest.NewRecorder()
raw, ok := rewriteServed(w, r, &nuget.Provider{}, nugetOrg, strings.NewReader(serviceIndex))
return w, string(raw), ok
}
func TestRewriteServedHeaders(t *testing.T) {
w, body, ok := rewrite(t, "aa.example:8443", "https")
if !ok {
t.Fatal("valid request rejected")
}
if want := `{"resources":[{"@id":"https://aa.example:8443/api/v1/remote/nuget-org/v3-flatcontainer/"}]}`; body != want {
t.Errorf("body = %s, want %s", body, want)
}
if got := w.Header().Get("Cache-Control"); got != "no-store" {
t.Errorf("Cache-Control = %q, want no-store", got)
}
if got := w.Header().Get("Vary"); got != "Host, X-Forwarded-Proto" {
t.Errorf("Vary = %q", got)
}
if got := w.Header().Get("Content-Length"); got != "91" || len(body) != 91 {
t.Errorf("Content-Length = %q, body len %d", got, len(body))
}
}
func TestRewriteServedIgnoresBogusProto(t *testing.T) {
_, plain, _ := rewrite(t, "aa.example", "")
_, bogus, _ := rewrite(t, "aa.example", "javascript")
if bogus != plain || !strings.Contains(bogus, `"http://aa.example/`) {
t.Errorf("bogus X-Forwarded-Proto changed the body: %s", bogus)
}
}
func TestRewriteServedRejectsInvalidHost(t *testing.T) {
w, _, ok := rewrite(t, `evil"},{"x":"`, "")
if ok || w.Code != http.StatusBadRequest {
t.Errorf("invalid host: ok=%v code=%d, want rejected with 400", ok, w.Code)
}
}
internal/api/v1/routes_test.go
+41
View File
@@ -0,0 +1,41 @@
package v1
import (
"net/http"
"testing"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/npm"
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
)
func TestRoutesRemoteHead(t *testing.T) {
r := (&ProxyHandler{}).Routes()
for _, m := range []string{http.MethodGet, http.MethodHead} {
if !r.Match(chi.NewRouteContext(), m, "/remote/myremote/some/path") {
t.Errorf("%s /remote/myremote/some/path not routed", m)
}
}
}
func TestRewritesMutable(t *testing.T) {
tests := []struct {
name string
prov provider.Provider
path string
want bool
}{
{"npm packument", &npm.Provider{}, "left-pad", true},
{"npm tarball", &npm.Provider{}, "left-pad/-/left-pad-1.3.0.tgz", false},
{"nuget index", &nuget.Provider{}, "index.json", true},
{"rpm repodata", &rpm.Provider{}, "repodata/repomd.xml", false},
}
for _, tc := range tests {
if got := rewritesMutable(tc.prov, tc.path); got != tc.want {
t.Errorf("%s: rewritesMutable = %v, want %v", tc.name, got, tc.want)
}
}
}
internal/api/v1/scheme_test.go
+17 -3
View File
@@ -7,14 +7,28 @@ import (
)
func TestScheme(t *testing.T) {
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
if got := Scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
t.Errorf("TLS request scheme = %q, want https", got)
}
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
if got := scheme(r); got != "https" {
if got := Scheme(r); got != "https" {
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
}
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
for fwd, want := range map[string]string{
"https, http": "https",
"http, https": "http",
"HTTPS": "https",
" https ": "https",
"javascript": "http",
"ftp": "http",
"https;evil": "http",
} {
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {fwd}}}
if got := Scheme(r); got != want {
t.Errorf("X-Forwarded-Proto %q scheme = %q, want %s", fwd, got, want)
}
}
if got := Scheme(&http.Request{Header: http.Header{}}); got != "http" {
t.Errorf("default scheme = %q, want http", got)
}
}
internal/api/v2/remotes.go
+20 -10
View File
@@ -67,8 +67,12 @@ func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, remote)
}
// defaultMutableTTL matches the remotes.mutable_ttl column default; a create
// that omits mutable_ttl gets it, an explicit 0 still means never expire.
const defaultMutableTTL = 3600
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
var remote models.Remote
remote := models.Remote{MutableTTL: defaultMutableTTL}
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
@@ -114,7 +118,15 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
// Capture the current backend before the update so we can tell whether the
// remote's base_url (its upstream) changed. A read failure just means we
// skip the freshness flush; it must not block the update.
existing, err := h.db.GetRemote(r.Context(), name)
oldKnown := err == nil
var remote models.Remote
if oldKnown {
remote.MutableTTL = existing.MutableTTL
}
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
@@ -132,13 +144,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Capture the current backend before the update so we can tell whether the
// remote's base_url (its upstream) changed. A read failure just means we
// skip the freshness flush; it must not block the update.
oldBaseURL, oldKnown := "", false
if existing, err := h.db.GetRemote(r.Context(), name); err == nil {
oldBaseURL, oldKnown = existing.BaseURL, true
}
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@@ -147,13 +152,13 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
// Release / APKINDEX): purge it so the next request re-fetches from the new
// upstream instead of serving stale data until TTL expiry. A flush failure
// is logged but does not fail the request — the DB update already landed.
if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil {
if oldKnown && existing.BaseURL != remote.BaseURL && h.cache != nil {
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
slog.Warn("flush cached metadata after base_url change failed",
"remote", name, "error", err)
} else {
slog.Info("flushed cached metadata after base_url change",
"remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL)
"remote", name, "old_base_url", existing.BaseURL, "new_base_url", remote.BaseURL)
}
}
writeJSON(w, http.StatusOK, remote)
@@ -165,6 +170,11 @@ func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if h.cache != nil {
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
slog.Warn("flush cached metadata after delete failed", "remote", name, "error", err)
}
}
w.WriteHeader(http.StatusNoContent)
}
internal/api/v2/remotes_flush_test.go
+25
View File
@@ -94,3 +94,28 @@ func TestUpdateFlushFailureStillSucceeds(t *testing.T) {
t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls)
}
}
// Deleting a remote flushes its cached state so a re-created remote of the
// same name starts cold; a flush error does not fail the delete.
func TestDeleteFlushesCache(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
for _, ff := range []*fakeFlusher{{}, {err: errors.New("redis down")}} {
const name = "rpm-flush-delete"
seedRemote(t, db, name, "https://example.com/repo")
h := NewRemotesHandler(db, ff, nil).Routes()
if c := do(t, h, "DELETE", "/"+name, ""); c != 204 {
t.Fatalf("delete = %d, want 204", c)
}
if len(ff.calls) != 1 || ff.calls[0] != name {
t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name)
}
}
}
internal/api/v2/remotes_ttl_test.go
+57
View File
@@ -0,0 +1,57 @@
package v2
import (
"context"
"testing"
"git.unkin.net/unkin/artifactapi/internal/database"
)
// An omitted mutable_ttl defaults to 3600 on create and keeps the stored value
// on update; an explicit 0 is stored as 0 in both.
func TestMutableTTLOmittedVsZero(t *testing.T) {
if testDSN == "" {
t.Skip("Docker unavailable")
}
db, err := database.New(testDSN)
if err != nil {
t.Fatal(err)
}
defer db.Close()
h := NewRemotesHandler(db, nil, nil).Routes()
ttl := func(name string) int {
t.Helper()
r, err := db.GetRemote(context.Background(), name)
if err != nil {
t.Fatalf("get %s: %v", name, err)
}
return r.MutableTTL
}
const body = `"package_type":"rpm","base_url":"https://example.com/repo"`
steps := []struct {
method, path, json string
want int
}{
{"POST", "/", `{"name":"ttl-omit",` + body + `}`, 3600},
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":120}`, 120},
{"PUT", "/ttl-omit", `{` + body + `}`, 120},
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":0}`, 0},
{"PUT", "/ttl-omit", `{` + body + `}`, 0},
}
for i, s := range steps {
if c := do(t, h, s.method, s.path, s.json); c/100 != 2 {
t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c)
}
if got := ttl("ttl-omit"); got != s.want {
t.Errorf("step %d: mutable_ttl = %d, want %d", i, got, s.want)
}
}
if c := do(t, h, "POST", "/", `{"name":"ttl-zero",`+body+`,"mutable_ttl":0}`); c != 201 {
t.Fatalf("create explicit 0 = %d", c)
}
if got := ttl("ttl-zero"); got != 0 {
t.Errorf("explicit 0 stored as %d", got)
}
}
internal/cache/cache_test.go
Vendored
+49 -7
View File
@@ -61,18 +61,54 @@ func TestTTL(t *testing.T) {
func TestLock(t *testing.T) {
requireRedis(t)
ctx := context.Background()
ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
if err != nil || !ok {
t.Fatalf("first acquire should succeed: %v %v", ok, err)
token, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
if err != nil || token == "" {
t.Fatalf("first acquire should succeed: %q %v", token, err)
}
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok {
if again, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); again != "" {
t.Error("second acquire should fail while held")
}
if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil {
if err := testRedis.ReleaseLock(ctx, "r", "lockpath", "not-the-owner"); err != nil {
t.Fatal(err)
}
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok {
t.Error("acquire should succeed after release")
if again, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); again != "" {
t.Error("release with a foreign token must not drop the lock")
}
if err := testRedis.ReleaseLock(ctx, "r", "lockpath", token); err != nil {
t.Fatal(err)
}
next, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
if next == "" {
t.Fatal("acquire should succeed after release")
}
if next == token {
t.Error("tokens must differ between holders")
}
}
func TestStaleReleaseKeepsNextHolder(t *testing.T) {
requireRedis(t)
ctx := context.Background()
stale, err := testRedis.AcquireLock(ctx, "r", "stale", 50*time.Millisecond)
if err != nil || stale == "" {
t.Fatalf("acquire: %q %v", stale, err)
}
time.Sleep(100 * time.Millisecond)
fresh, err := testRedis.AcquireLock(ctx, "r", "stale", time.Minute)
if err != nil || fresh == "" {
t.Fatalf("acquire after expiry: %q %v", fresh, err)
}
if err := testRedis.ReleaseLock(ctx, "r", "stale", stale); err != nil {
t.Fatal(err)
}
if again, _ := testRedis.AcquireLock(ctx, "r", "stale", time.Minute); again != "" {
t.Fatal("stale holder's release dropped the second holder's lock")
}
if err := testRedis.ReleaseLock(ctx, "r", "stale", fresh); err != nil {
t.Fatal(err)
}
if again, _ := testRedis.AcquireLock(ctx, "r", "stale", time.Minute); again == "" {
t.Error("owner release should free the lock")
}
}
@@ -86,6 +122,12 @@ func TestETagAndToken(t *testing.T) {
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
t.Errorf("etag = %q", v)
}
if err := testRedis.SetETag(ctx, "r", "p", "", 0); err != nil {
t.Fatal(err)
}
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != "" {
t.Errorf("empty etag should drop the stored one, got %q", v)
}
if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
t.Error("missing token should be empty")
internal/cache/redis.go
Vendored
+18 -5
View File
@@ -2,6 +2,7 @@ package cache
import (
"context"
"crypto/rand"
"fmt"
"strings"
"time"
@@ -46,19 +47,31 @@ func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error)
return exists > 0, nil
}
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
// AcquireLock returns the owner token on success and "" when the lock is held.
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (string, error) {
key := fmt.Sprintf("lock:%s:%s", remote, path)
ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
return ok, err
token := rand.Text()
ok, err := r.client.SetNX(ctx, key, token, ttl).Result()
if err != nil || !ok {
return "", err
}
return token, nil
}
func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
var releaseScript = redis.NewScript(`if redis.call('get',KEYS[1])==ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end`)
// ReleaseLock deletes the lock only while token still owns it.
func (r *Redis) ReleaseLock(ctx context.Context, remote, path, token string) error {
key := fmt.Sprintf("lock:%s:%s", remote, path)
return r.client.Del(ctx, key).Err()
return releaseScript.Run(ctx, r.client, []string{key}, token).Err()
}
// SetETag stores etag for path, or drops the stored one when etag is empty.
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
key := fmt.Sprintf("etag:%s:%s", remote, path)
if etag == "" {
return r.client.Del(ctx, key).Err()
}
return r.client.Set(ctx, key, etag, ttl).Err()
}
internal/database/github_sync_test.go
+69
View File
@@ -207,3 +207,72 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
t.Fatal("claimed inside freshness window after a successful sync")
}
}
// Deleting a remote drops its sync state, so a recreated remote of the same name
// starts clean instead of inheriting the old ETag, sync time and retry state.
func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
requireDB(t)
for _, tc := range []struct {
table string
pt models.PackageType
claim func(name string) (bool, string, error)
release func(name string, res provider.SyncResult) error
}{
{"github_rpm_sync_state", models.PackageGitHubRPM,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
}},
{"github_deb_sync_state", models.PackageGitHubDeb,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
}},
{"github_alpine_sync_state", models.PackageGitHubAlpine,
func(n string) (bool, string, error) {
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
},
func(n string, res provider.SyncResult) error {
return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
}},
} {
t.Run(tc.table, func(t *testing.T) {
name := "gh-recreate-" + time.Now().Format("150405.000000")
create := func() {
t.Helper()
if err := testDB.CreateRemote(ctx(), &models.Remote{
Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
}); err != nil {
t.Fatalf("create remote: %v", err)
}
}
create()
if ok, _, err := tc.claim(name); err != nil || !ok {
t.Fatalf("claim: ok=%v err=%v", ok, err)
}
if err := tc.release(name, provider.SyncResult{Etag: `"old"`}); err != nil {
t.Fatalf("release: %v", err)
}
if err := testDB.DeleteRemote(ctx(), name); err != nil {
t.Fatalf("delete remote: %v", err)
}
var rows int
if err := testDB.Pool.QueryRow(ctx(), `SELECT count(*) FROM `+tc.table+` WHERE remote_name = $1`, name).Scan(&rows); err != nil || rows != 0 {
t.Fatalf("sync state left after delete: rows=%d err=%v", rows, err)
}
create()
ok, etag, err := tc.claim(name)
if err != nil || !ok || etag != "" {
t.Fatalf("recreated remote claim ok=%v etag=%q err=%v, want a fresh claim with no etag", ok, etag, err)
}
})
}
}
internal/database/helm_metadata.go
+44
View File
@@ -0,0 +1,44 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml)
VALUES ($1,$2,$3,$4,$5,$6)
ON CONFLICT (repo_name, file_path) DO NOTHING
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML)
return err
}
func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at
FROM helm_metadata
WHERE repo_name = $1
ORDER BY name, version, file_path
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []provider.HelmMetadata
for rows.Next() {
var m provider.HelmMetadata
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil {
return nil, err
}
result = append(result, m)
}
return result, rows.Err()
}
internal/database/puppet_metadata.go
+46
View File
@@ -0,0 +1,46 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
ON CONFLICT (repo_name, file_path) DO NOTHING
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5)
return err
}
func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to
// one module slug (empty slug lists every module).
func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at
FROM puppet_metadata
WHERE repo_name = $1 AND ($2 = '' OR slug = $2)
ORDER BY slug, file_path
`, repoName, slug)
if err != nil {
return nil, err
}
defer rows.Close()
var result []provider.PuppetMetadata
for rows.Next() {
var m provider.PuppetMetadata
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil {
return nil, err
}
result = append(result, m)
}
return result, rows.Err()
}
internal/provider/all/alpine.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
internal/provider/all/ansible.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
internal/provider/all/cargo.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
internal/provider/all/deb.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb"
internal/provider/all/docker.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
internal/provider/all/gems.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/gems"
internal/provider/all/generic.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
internal/provider/all/goproxy.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
internal/provider/all/helm.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
internal/provider/all/npm.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
internal/provider/all/nuget.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
internal/provider/all/puppet.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
internal/provider/all/pypi.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
internal/provider/all/rpm.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
internal/provider/all/terraform.go
+3
View File
@@ -0,0 +1,3 @@
package all
import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
internal/provider/alpine/github.go
+14 -13
View File
@@ -372,6 +372,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
return newEtag, false, err
}
// A transiently failed asset fails the scan so the ETag is not advanced and
// the retry re-derives it; an invalid package is skipped as no retry fixes it.
var failed []error
seen := map[string]bool{}
for _, rel := range releases {
if rel.Draft {
@@ -400,10 +403,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
meta, err := p.deriveAsset(ctx, remote, asset, fp)
if err != nil {
slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
if !errors.Is(err, provider.ErrInvalidPackage) {
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
}
continue
}
if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
slog.Error("github_alpine: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
continue
}
slog.Info("github_alpine: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -415,7 +422,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
}
}
return newEtag, true, nil
return newEtag, true, errors.Join(failed...)
}
type ghRelease struct {
@@ -496,7 +503,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
return nil, err
}
if meta.Name == "" || meta.Arch == "" {
return nil, errors.New(".PKGINFO missing pkgname/arch")
return nil, fmt.Errorf("%w: .PKGINFO missing pkgname/arch", provider.ErrInvalidPackage)
}
meta.RepoName = remote.Name
@@ -526,13 +533,13 @@ func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote,
}
meta, complete, perr := pkginfoFromPrefix(body)
if perr != nil {
return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
return nil, fmt.Errorf("%w: parse apk .PKGINFO: %w", provider.ErrInvalidPackage, perr)
}
if complete {
return meta, nil
}
if full || n >= p.headerMax {
return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
return nil, fmt.Errorf("%w: .PKGINFO not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
}
n *= 2
if n > p.headerMax {
@@ -594,7 +601,7 @@ func pkginfoFromPrefix(prefix []byte) (meta *provider.AlpineMetadata, complete b
}
// rangeGet returns the first n bytes of downloadURL. full is true when the
// response body was shorter than n (i.e. we already have the whole object).
// response holds the whole object.
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
if err != nil {
@@ -616,15 +623,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
}
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
if err != nil {
return nil, false, err
}
full := int64(len(body)) < n
return body, full, nil
return provider.ReadPrefix(resp, n)
}
// assetPath is the package's location relative to github.com — the path the
internal/provider/alpine/github_test.go
+22 -3
View File
@@ -8,6 +8,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -88,15 +89,17 @@ type githubFixture struct {
notModHit int
releaseAuth string
assetAuth string
assetFail map[string]int // asset filename -> downloads left to answer 500
mu sync.Mutex
}
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
t.Helper()
f := &githubFixture{
apkBytes: map[string][]byte{},
rangeHit: map[string]int{},
fullHit: map[string]int{},
apkBytes: map[string][]byte{},
rangeHit: map[string]int{},
fullHit: map[string]int{},
assetFail: map[string]int{},
}
f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
@@ -146,6 +149,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
}
rng := r.Header.Get("Range")
f.mu.Lock()
if f.assetFail[name] > 0 {
f.assetFail[name]--
f.mu.Unlock()
http.Error(w, "boom", http.StatusInternalServerError)
return
}
f.assetAuth = r.Header.Get("Authorization")
if rng != "" {
f.rangeHit[name]++
@@ -495,3 +504,13 @@ func readAPKIndex(t *testing.T, gzBytes []byte) string {
}
}
}
func TestGitHubDeriveAssetMissingPkgnameIsInvalid(t *testing.T) {
fx := newGitHubFixture(t, true)
fx.apkBytes["nameless-1-r0.apk"] = testsupport.MinimalApk("", "1-r0", "x86_64")
asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2.3/nameless-1-r0.apk"}
_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
if !errors.Is(err, provider.ErrInvalidPackage) {
t.Fatalf("want ErrInvalidPackage, got %v", err)
}
}
internal/provider/alpine/syncer_test.go
+55
View File
@@ -308,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
}
}
// A transiently failed asset fails the scan: the ETag is not advanced, a retry
// is scheduled, and the retry against the unchanged release list derives it. An
// invalid package is skipped instead, since no retry can fix it.
func TestSyncerFailedAssetRetry(t *testing.T) {
for _, tc := range []struct {
name string
corrupt bool
wantRetry bool
}{
{name: "transient", wantRetry: true},
{name: "invalid package", corrupt: true},
} {
t.Run(tc.name, func(t *testing.T) {
fx := newGitHubFixture(t, true)
fx.etag = `"v1"`
if tc.corrupt {
fx.apkBytes["other-9-r0.apk"] = []byte("not a package")
} else {
fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
fx.assetFail["other-9-r0.apk"] = 1
}
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
remote := fx.remote()
bg := context.Background()
s.process(bg, syncJob{remote: remote, prime: true})
_, pending := store.retryAt[remote.Name]
if pending != tc.wantRetry {
t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
}
if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
t.Fatalf("etag saved=%v after first scan", saved)
}
if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 1 {
t.Fatalf("healthy asset not served: %d rows", len(rows))
}
if !tc.wantRetry {
return
}
store.mu.Lock()
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
store.mu.Unlock()
s.process(bg, syncJob{remote: remote})
if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
}
if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 2 {
t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
}
})
}
}
internal/provider/cap_file_store.go
+15
View File
@@ -0,0 +1,15 @@
package provider
import (
"context"
"io"
)
type FileStore interface {
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
ListPackages(ctx context.Context, repoName string) ([]string, error)
}
type BlobReader interface {
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
}
internal/provider/cap_index_merger.go
+10
View File
@@ -0,0 +1,10 @@
package provider
type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
}
type MemberIndex struct {
RemoteName string
Body []byte
}
internal/provider/cap_local_indexer.go
+11
View File
@@ -0,0 +1,11 @@
package provider
import (
"context"
"net/http"
)
type LocalIndexer interface {
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
}
internal/provider/cap_local_uploader.go
+6
View File
@@ -0,0 +1,6 @@
package provider
type LocalUploader interface {
ValidateUpload(filePath string) (storagePath, contentType string, err error)
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
}
internal/provider/cap_post_delete_hook.go
+11
View File
@@ -0,0 +1,11 @@
package provider
import (
"context"
)
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
// feeds generated repodata) after a local file is removed.
type PostDeleteHook interface {
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
}
internal/provider/cap_post_upload_hook.go
+9
View File
@@ -0,0 +1,9 @@
package provider
import (
"context"
)
type PostUploadHook interface {
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
}
internal/provider/cap_query_forwarder.go
+12
View File
@@ -0,0 +1,12 @@
package provider
// QueryForwarder is a remote provider whose upstream answers depend on the
// query string (e.g. paginated JSON APIs). Only the named params for path are
// forwarded upstream and kept in the cache key, each as the canonical value
// its validator returns; a value the validator rejects is dropped.
type QueryForwarder interface {
ForwardedQuery(path string) map[string]QueryValidator
}
// QueryValidator canonicalises a raw query value, reporting false to drop it.
type QueryValidator func(v string) (string, bool)
internal/provider/cap_remote_server.go
+25
View File
@@ -0,0 +1,25 @@
package provider
import (
"net/http"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// RemoteServer lets a remote provider fully answer a request itself instead of
// going through the byte-proxy engine. It is the remote-side analog of
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
// repodata from derived per-asset metadata and to redirect package downloads to
// a backend remote, without ever precaching the packages. Returning false lets
// the normal proxy path take over.
type RemoteServer interface {
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
}
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
type RemoteMetadataStore interface {
RPMMetadataReader
MetadataStore
MetadataDeleter
}
internal/provider/cap_response_rewriter.go
+7
View File
@@ -0,0 +1,7 @@
package provider
// ResponseRewriter marks providers whose RewriteResponse changes mutable
// bodies, so the served length is only known after a fetch.
type ResponseRewriter interface {
RewritesResponse()
}
internal/provider/cap_serve_rewriter.go
+12
View File
@@ -0,0 +1,12 @@
package provider
import (
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// ServeRewriter rewrites a cached mutable remote response per request, for
// indexes whose URLs must name the requesting host and so cannot be rewritten
// before caching.
type ServeRewriter interface {
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
}
internal/provider/deb/github.go
+15 -14
View File
@@ -333,6 +333,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
return newEtag, false, err
}
// A transiently failed asset fails the scan so the ETag is not advanced and
// the retry re-derives it; an invalid package is skipped as no retry fixes it.
var failed []error
seen := map[string]bool{}
for _, rel := range releases {
if rel.Draft {
@@ -361,10 +364,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
meta, err := p.deriveAsset(ctx, remote, asset, fp)
if err != nil {
slog.Warn("github_deb: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
if !errors.Is(err, provider.ErrInvalidPackage) {
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
}
continue
}
if err := store.InsertDebMetadata(ctx, meta); err != nil {
slog.Error("github_deb: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
continue
}
slog.Info("github_deb: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
@@ -376,7 +383,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
}
}
return newEtag, true, nil
return newEtag, true, errors.Join(failed...)
}
type ghRelease struct {
@@ -468,7 +475,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
Size: asset.Size,
}
if meta.Name == "" {
return nil, errors.New("control missing Package field")
return nil, fmt.Errorf("%w: control missing Package field", provider.ErrInvalidPackage)
}
// The Packages SHA256 must be the sha256 of the whole .deb. Prefer GitHub's
@@ -503,13 +510,13 @@ func (p *GitHubProvider) fetchControl(ctx context.Context, remote models.Remote,
}
control, complete, perr := controlFromPrefix(body)
if perr != nil {
return "", fmt.Errorf("parse deb control: %w", perr)
return "", fmt.Errorf("%w: parse deb control: %w", provider.ErrInvalidPackage, perr)
}
if complete {
return control, nil
}
if full || n >= p.headerMax {
return "", fmt.Errorf("control.tar not found within %d bytes of %s", n, downloadURL)
return "", fmt.Errorf("%w: control.tar not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
}
n *= 2
if n > p.headerMax {
@@ -567,7 +574,7 @@ func controlFromPrefix(prefix []byte) (control string, complete bool, err error)
}
// rangeGet returns the first n bytes of downloadURL. full is true when the
// response body was shorter than n (i.e. we already have the whole object).
// response holds the whole object.
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
if err != nil {
@@ -589,15 +596,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
}
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
if err != nil {
return nil, false, err
}
full := int64(len(body)) < n
return body, full, nil
return provider.ReadPrefix(resp, n)
}
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -620,7 +621,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
}
h := sha256.New()
internal/provider/deb/github_test.go
+22 -3
View File
@@ -7,6 +7,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -81,15 +82,17 @@ type githubFixture struct {
notModHit int
releaseAuth string
assetAuth string
assetFail map[string]int // asset filename -> downloads left to answer 500
mu sync.Mutex
}
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
t.Helper()
f := &githubFixture{
debBytes: map[string][]byte{},
rangeHit: map[string]int{},
fullHit: map[string]int{},
debBytes: map[string][]byte{},
rangeHit: map[string]int{},
fullHit: map[string]int{},
assetFail: map[string]int{},
}
f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
@@ -139,6 +142,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
}
rng := r.Header.Get("Range")
f.mu.Lock()
if f.assetFail[name] > 0 {
f.assetFail[name]--
f.mu.Unlock()
http.Error(w, "boom", http.StatusInternalServerError)
return
}
f.assetAuth = r.Header.Get("Authorization")
if rng != "" {
f.rangeHit[name]++
@@ -460,3 +469,13 @@ func TestGitHubAssetPatternFilter(t *testing.T) {
t.Fatalf("pattern filter failed, rows=%+v", rows)
}
}
func TestGitHubDeriveAssetMissingPackageIsInvalid(t *testing.T) {
fx := newGitHubFixture(t, true)
fx.debBytes["nameless_1_amd64.deb"] = testsupport.MinimalDeb("", "1", "amd64")
asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2-3/nameless_1_amd64.deb", Digest: "sha256:00"}
_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
if !errors.Is(err, provider.ErrInvalidPackage) {
t.Fatalf("want ErrInvalidPackage, got %v", err)
}
}
internal/provider/deb/syncer_test.go
+55
View File
@@ -308,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
}
}
// A transiently failed asset fails the scan: the ETag is not advanced, a retry
// is scheduled, and the retry against the unchanged release list derives it. An
// invalid package is skipped instead, since no retry can fix it.
func TestSyncerFailedAssetRetry(t *testing.T) {
for _, tc := range []struct {
name string
corrupt bool
wantRetry bool
}{
{name: "transient", wantRetry: true},
{name: "invalid package", corrupt: true},
} {
t.Run(tc.name, func(t *testing.T) {
fx := newGitHubFixture(t, true)
fx.etag = `"v1"`
if tc.corrupt {
fx.debBytes["other_9-9_arm64.deb"] = []byte("not a package")
} else {
fx.debBytes["other_9-9_arm64.deb"] = testsupport.MinimalDeb("other", "9-9", "arm64")
fx.assetFail["other_9-9_arm64.deb"] = 1
}
store := newFakeSyncStore()
s := newSyncer(store, newTestProvider(), testSyncConfig())
remote := fx.remote()
bg := context.Background()
s.process(bg, syncJob{remote: remote, prime: true})
_, pending := store.retryAt[remote.Name]
if pending != tc.wantRetry {
t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
}
if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
t.Fatalf("etag saved=%v after first scan", saved)
}
if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 1 {
t.Fatalf("healthy asset not served: %d rows", len(rows))
}
if !tc.wantRetry {
return
}
store.mu.Lock()
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
store.mu.Unlock()
s.process(bg, syncJob{remote: remote})
if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
}
if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 2 {
t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
}
})
}
}
internal/provider/gems/gems.go
+62
View File
@@ -0,0 +1,62 @@
// Package gems proxies a RubyGems source: mutable compact index (versions,
// info/*, names) and legacy specs indexes, plus immutable .gem and
// quick/Marshal.4.8 gemspec downloads.
package gems
import (
"context"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageGems }
func (p *Provider) Classify(path string) provider.Mutability {
if isGem(path) || isQuickSpec(path) {
return provider.Immutable
}
return provider.Mutable
}
func (p *Provider) ContentType(path string) string {
path = strings.TrimLeft(path, "/")
if path == "versions" || path == "names" || strings.HasPrefix(path, "info/") {
return "text/plain; charset=utf-8"
}
if strings.HasPrefix(path, "api/v1/") && strings.HasSuffix(path, ".json") {
return "application/json"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
func isGem(path string) bool {
path = strings.TrimLeft(path, "/")
return strings.HasPrefix(path, "gems/") && strings.HasSuffix(path, ".gem")
}
func isQuickSpec(path string) bool {
path = strings.TrimLeft(path, "/")
return strings.HasPrefix(path, "quick/Marshal.4.8/") && strings.HasSuffix(path, ".gemspec.rz")
}
internal/provider/gems/gems_test.go
+61
View File
@@ -0,0 +1,61 @@
package gems
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestClassify(t *testing.T) {
p := &Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"gems/rack-3.1.8.gem", provider.Immutable},
{"/gems/nokogiri-1.16.7-x86_64-linux.gem", provider.Immutable},
{"quick/Marshal.4.8/rack-3.1.8.gemspec.rz", provider.Immutable},
{"versions", provider.Mutable},
{"names", provider.Mutable},
{"info/rack", provider.Mutable},
{"info/gems", provider.Mutable},
{"specs.4.8.gz", provider.Mutable},
{"latest_specs.4.8.gz", provider.Mutable},
{"prerelease_specs.4.8.gz", provider.Mutable},
{"api/v1/dependencies", provider.Mutable},
{"quick/Marshal.4.8/", provider.Mutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestContentType(t *testing.T) {
p := &Provider{}
tests := map[string]string{
"versions": "text/plain; charset=utf-8",
"info/rack": "text/plain; charset=utf-8",
"names": "text/plain; charset=utf-8",
"gems/rack-3.1.8.gem": "application/octet-stream",
"specs.4.8.gz": "application/octet-stream",
"api/v1/gems/rack.json": "application/json",
"api/v1/dependencies.json": "application/json",
"api/v1/dependencies": "application/octet-stream",
}
for path, want := range tests {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
}
}
}
func TestUpstreamURL(t *testing.T) {
p := &Provider{}
r := models.Remote{BaseURL: "https://rubygems.org/"}
if got := p.UpstreamURL(r, "/info/rack"); got != "https://rubygems.org/info/rack" {
t.Errorf("UpstreamURL = %q", got)
}
}
internal/provider/gems/local.go
+468
View File
@@ -0,0 +1,468 @@
package gems
import (
"archive/tar"
"bytes"
"compress/gzip"
"compress/zlib"
"crypto/md5"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"maps"
"regexp"
"slices"
"strings"
"time"
"gopkg.in/yaml.v3"
)
var (
NameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
VersionRe = regexp.MustCompile(`^[0-9]+(\.[0-9A-Za-z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
PlatformRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
opRe = regexp.MustCompile(`^(=|!=|>|<|>=|<=|~>)$`)
)
const maxMetadataBytes = 8 << 20
type Requirement [][2]string
type Dependency struct {
Name string
Requirement Requirement
Type string
Prerelease bool
}
// Spec is the subset of a gemspec the compact index and quick spec need.
type Spec struct {
Name, Version, Platform string
Summary, RubyGemsVer string
SpecVersion int
Date time.Time
Authors, Licenses []string
Email, Description string
Homepage string
Metadata map[string]string
Dependencies []Dependency
RequiredRuby Requirement
RequiredRubyGems Requirement
}
// FullName is the gem's file stem: name-version, plus -platform when not ruby.
func (s *Spec) FullName() string {
if s.Platform == "" || s.Platform == "ruby" {
return s.Name + "-" + s.Version
}
return s.Name + "-" + s.Version + "-" + s.Platform
}
type yamlReq struct {
Requirements [][]yaml.Node `yaml:"requirements"`
}
func (r *yamlReq) decode() (Requirement, error) {
if r == nil {
return nil, nil
}
var out Requirement
for _, pair := range r.Requirements {
var op string
var ver struct {
Version string `yaml:"version"`
}
if len(pair) != 2 || pair[0].Decode(&op) != nil || pair[1].Decode(&ver) != nil {
return nil, errors.New("malformed requirement")
}
if !opRe.MatchString(op) || !VersionRe.MatchString(ver.Version) {
return nil, fmt.Errorf("invalid requirement %q %q", op, ver.Version)
}
out = append(out, [2]string{op, ver.Version})
}
return out, nil
}
// String renders the requirement as the compact index does: "op ver&op ver".
func (r Requirement) String() string {
parts := make([]string, len(r))
for i, p := range r {
parts[i] = p[0] + " " + p[1]
}
return strings.Join(parts, "&")
}
func (r Requirement) trivial() bool {
return len(r) == 0 || len(r) == 1 && r[0] == [2]string{">=", "0"}
}
// ParseGem reads the gemspec from a .gem archive (a tar holding metadata.gz).
func ParseGem(gem []byte) (*Spec, error) {
tr := tar.NewReader(bytes.NewReader(gem))
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil, errors.New("gem has no metadata.gz")
}
if err != nil {
return nil, fmt.Errorf("read gem: %w", err)
}
if hdr.Name != "metadata.gz" {
continue
}
zr, err := gzip.NewReader(tr)
if err != nil {
return nil, fmt.Errorf("metadata.gz: %w", err)
}
raw, err := io.ReadAll(io.LimitReader(zr, maxMetadataBytes+1))
if err != nil {
return nil, fmt.Errorf("metadata.gz: %w", err)
}
if len(raw) > maxMetadataBytes {
return nil, fmt.Errorf("metadata exceeds %d bytes", maxMetadataBytes)
}
return parseMetadata(raw)
}
}
func parseMetadata(raw []byte) (*Spec, error) {
var doc struct {
Name string `yaml:"name"`
Version struct {
Version string `yaml:"version"`
} `yaml:"version"`
Platform string `yaml:"platform"`
Summary string `yaml:"summary"`
Authors []string `yaml:"authors"`
Email yaml.Node `yaml:"email"`
Description string `yaml:"description"`
Homepage string `yaml:"homepage"`
Licenses []string `yaml:"licenses"`
Metadata map[string]string `yaml:"metadata"`
Date string `yaml:"date"`
RubyGemsVer string `yaml:"rubygems_version"`
SpecVersion int `yaml:"specification_version"`
Dependencies []struct {
Name string `yaml:"name"`
Requirement *yamlReq `yaml:"requirement"`
VersionReqs *yamlReq `yaml:"version_requirements"`
Type string `yaml:"type"`
Prerelease bool `yaml:"prerelease"`
} `yaml:"dependencies"`
RequiredRuby *yamlReq `yaml:"required_ruby_version"`
RequiredRubyGems *yamlReq `yaml:"required_rubygems_version"`
}
if err := yaml.Unmarshal(raw, &doc); err != nil {
return nil, fmt.Errorf("gemspec yaml: %w", err)
}
s := &Spec{
Name: doc.Name, Version: doc.Version.Version, Platform: doc.Platform,
Summary: doc.Summary, Authors: doc.Authors, Description: doc.Description,
Homepage: doc.Homepage, Licenses: doc.Licenses, Metadata: doc.Metadata,
RubyGemsVer: doc.RubyGemsVer, SpecVersion: doc.SpecVersion,
}
if s.Platform == "" {
s.Platform = "ruby"
}
if !NameRe.MatchString(s.Name) || !VersionRe.MatchString(s.Version) || !PlatformRe.MatchString(s.Platform) {
return nil, fmt.Errorf("invalid gem name/version/platform %q %q %q", s.Name, s.Version, s.Platform)
}
if doc.Email.Kind == yaml.ScalarNode {
s.Email = doc.Email.Value
}
if t, err := time.Parse("2006-01-02", strings.SplitN(doc.Date, " ", 2)[0]); err == nil {
s.Date = t
}
var err error
if s.RequiredRuby, err = doc.RequiredRuby.decode(); err != nil {
return nil, err
}
if s.RequiredRubyGems, err = doc.RequiredRubyGems.decode(); err != nil {
return nil, err
}
for _, d := range doc.Dependencies {
if !NameRe.MatchString(d.Name) {
return nil, fmt.Errorf("invalid dependency name %q", d.Name)
}
req := d.Requirement
if req == nil {
req = d.VersionReqs
}
r, err := req.decode()
if err != nil {
return nil, fmt.Errorf("dependency %s: %w", d.Name, err)
}
typ := strings.TrimPrefix(d.Type, ":")
if typ == "" {
typ = "runtime"
}
s.Dependencies = append(s.Dependencies, Dependency{Name: d.Name, Requirement: r, Type: typ, Prerelease: d.Prerelease})
}
return s, nil
}
// InfoLine renders the spec's compact index /info line; checksum is the
// .gem's sha256 hex.
func InfoLine(s *Spec, gem []byte) string {
sum := sha256.Sum256(gem)
var deps []string
for _, d := range s.Dependencies {
if d.Type == "runtime" {
r := d.Requirement
if len(r) == 0 {
r = Requirement{{">=", "0"}}
}
deps = append(deps, d.Name+":"+r.String())
}
}
key := s.Version
if s.Platform != "ruby" {
key += "-" + s.Platform
}
line := key + " " + strings.Join(deps, ",") + "|checksum:" + hex.EncodeToString(sum[:])
if !s.RequiredRuby.trivial() {
line += ",ruby:" + s.RequiredRuby.String()
}
if !s.RequiredRubyGems.trivial() {
line += ",rubygems:" + s.RequiredRubyGems.String()
}
return line
}
// InfoBody renders an /info document from its lines.
func InfoBody(lines []string) []byte {
var b strings.Builder
b.WriteString("---\n")
for _, l := range lines {
b.WriteString(l)
b.WriteByte('\n')
}
return []byte(b.String())
}
// InfoKey returns an /info line's version[-platform] token.
func InfoKey(line string) string {
k, _, _ := strings.Cut(line, " ")
return k
}
// InfoLines returns an /info body's version lines.
func InfoLines(body []byte) []string {
var out []string
for _, l := range strings.Split(string(body), "\n") {
if l != "" && l != "---" {
out = append(out, l)
}
}
return out
}
func MD5Hex(b []byte) string {
sum := md5.Sum(b)
return hex.EncodeToString(sum[:])
}
// VersionsLine renders one /versions entry from a gem's /info body.
func VersionsLine(name string, info []byte) string {
var keys []string
for _, l := range InfoLines(info) {
keys = append(keys, InfoKey(l))
}
return name + " " + strings.Join(keys, ",") + " " + MD5Hex(info)
}
// VersionsHeader starts a /versions document.
func VersionsHeader(at time.Time) string {
return "created_at: " + at.UTC().Format(time.RFC3339) + "\n---\n"
}
// QuickSpec returns the deflated Marshal.4.8 dump of the spec, as served at
// quick/Marshal.4.8/<full>.gemspec.rz.
func QuickSpec(s *Spec) ([]byte, error) {
var buf bytes.Buffer
zw := zlib.NewWriter(&buf)
if _, err := zw.Write(MarshalSpec(s)); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// MarshalSpec mirrors Gem::Specification#_dump wrapped by Marshal.dump.
func MarshalSpec(s *Spec) []byte {
var in rbMarshal
in.WriteString("\x04\x08")
in.array(19)
in.str(s.RubyGemsVer)
in.WriteByte('i')
in.fixnum(s.SpecVersion)
in.str(s.Name)
in.version(s.Version)
in.time(s.Date)
in.str(s.Summary)
in.requirement(s.RequiredRuby)
in.requirement(s.RequiredRubyGems)
in.str(s.Platform)
in.array(len(s.Dependencies))
for _, d := range s.Dependencies {
in.WriteByte('o')
in.sym("Gem::Dependency")
in.fixnum(4)
in.sym("@name")
in.str(d.Name)
in.sym("@requirement")
in.requirement(d.Requirement)
in.sym("@type")
in.sym(d.Type)
in.sym("@prerelease")
in.boolean(d.Prerelease)
}
in.str("")
in.optStr(s.Email)
in.strs(s.Authors)
in.optStr(s.Description)
in.optStr(s.Homepage)
in.boolean(true)
in.str(s.Platform)
in.strs(s.Licenses)
in.WriteByte('{')
in.fixnum(len(s.Metadata))
for _, k := range slices.Sorted(maps.Keys(s.Metadata)) {
in.str(k)
in.str(s.Metadata[k])
}
var out rbMarshal
out.WriteString("\x04\x08u")
out.sym("Gem::Specification")
out.fixnum(in.Len())
out.Write(in.Bytes())
return out.Bytes()
}
// rbMarshal writes the Ruby Marshal 4.8 subset gemspecs use. It never emits
// object or symbol links, so every symbol is written in full.
type rbMarshal struct{ bytes.Buffer }
func (m *rbMarshal) fixnum(n int) {
switch {
case n == 0:
m.WriteByte(0)
case n > 0 && n < 123:
m.WriteByte(byte(n + 5))
case n < 0 && n > -124:
m.WriteByte(byte(n - 5))
default:
var b []byte
for x := n; len(b) < 4; x >>= 8 {
b = append(b, byte(x))
if (n >= 0 && x>>8 == 0) || (n < 0 && x>>8 == -1) {
break
}
}
if n >= 0 {
m.WriteByte(byte(len(b)))
} else {
m.WriteByte(byte(-len(b)))
}
m.Write(b)
}
}
func (m *rbMarshal) raw(s string) {
m.fixnum(len(s))
m.WriteString(s)
}
func (m *rbMarshal) sym(s string) {
m.WriteByte(':')
m.raw(s)
}
// str writes a UTF-8 String (an ivar-wrapped string with E=true).
func (m *rbMarshal) str(s string) {
m.WriteString("I\"")
m.raw(s)
m.fixnum(1)
m.sym("E")
m.WriteByte('T')
}
func (m *rbMarshal) optStr(s string) {
if s == "" {
m.WriteByte('0')
return
}
m.str(s)
}
func (m *rbMarshal) strs(ss []string) {
m.array(len(ss))
for _, s := range ss {
m.str(s)
}
}
func (m *rbMarshal) array(n int) {
m.WriteByte('[')
m.fixnum(n)
}
func (m *rbMarshal) boolean(b bool) {
if b {
m.WriteByte('T')
} else {
m.WriteByte('F')
}
}
func (m *rbMarshal) version(v string) {
m.WriteByte('U')
m.sym("Gem::Version")
m.array(1)
m.str(v)
}
func (m *rbMarshal) requirement(r Requirement) {
if len(r) == 0 {
r = Requirement{{">=", "0"}}
}
m.WriteByte('U')
m.sym("Gem::Requirement")
m.array(1)
m.array(len(r))
for _, p := range r {
m.array(2)
m.str(p[0])
m.version(p[1])
}
}
// time writes a UTC Time with day resolution, as Time#_dump lays it out.
func (m *rbMarshal) time(t time.Time) {
t = t.UTC()
// Time#_dump holds year-1900 in 16 bits.
if t.IsZero() || t.Year() < 1900 || t.Year() > 1900+0xffff {
t = time.Unix(0, 0).UTC()
}
p := uint32(1)<<31 | uint32(1)<<30 | uint32(t.Year()-1900)<<14 | uint32(t.Month()-1)<<10 | uint32(t.Day())<<5 | uint32(t.Hour())
s := uint32(t.Minute())<<26 | uint32(t.Second())<<20
m.WriteString("Iu")
m.sym("Time")
m.fixnum(8)
for _, w := range []uint32{p, s} {
m.Write([]byte{byte(w), byte(w >> 8), byte(w >> 16), byte(w >> 24)})
}
m.fixnum(1)
m.sym("zone")
m.WriteString("I\"")
m.raw("UTC")
m.fixnum(1)
m.sym("E")
m.WriteByte('F')
}
internal/provider/gems/local_test.go
+223
View File
@@ -0,0 +1,223 @@
package gems
import (
"archive/tar"
"bytes"
"compress/gzip"
"compress/zlib"
"crypto/sha256"
"encoding/hex"
"io"
"strings"
"testing"
"time"
)
const testMetadata = `--- !ruby/object:Gem::Specification
name: unkin-e2e-hello
version: !ruby/object:Gem::Version
version: 0.1.0
platform: %PLATFORM%
authors:
- e2e
date: 2026-10-17 00:00:00.000000000 Z
dependencies:
- !ruby/object:Gem::Dependency
name: rack
requirement: &1 !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.0'
- - "<"
- !ruby/object:Gem::Version
version: '4'
type: :runtime
prerelease: false
version_requirements: *1
- !ruby/object:Gem::Dependency
name: json
version_requirements: !ruby/object:Gem::Requirement
requirements:
- - "~>"
- !ruby/object:Gem::Version
version: '2.1'
type: :runtime
prerelease: false
- !ruby/object:Gem::Dependency
name: rake
requirement: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
type: :development
prerelease: false
email:
licenses:
- MIT
metadata:
source_code_uri: https://example.com
required_ruby_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '2.7'
required_rubygems_version: !ruby/object:Gem::Requirement
requirements:
- - ">="
- !ruby/object:Gem::Version
version: '0'
rubygems_version: 3.5.22
specification_version: 4
summary: hi
`
func buildGem(t *testing.T, metadata []byte) []byte {
t.Helper()
var gz bytes.Buffer
zw := gzip.NewWriter(&gz)
_, _ = zw.Write(metadata)
_ = zw.Close()
var buf bytes.Buffer
tw := tar.NewWriter(&buf)
for name, body := range map[string][]byte{"data.tar.gz": []byte("x"), "metadata.gz": gz.Bytes()} {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o444, Size: int64(len(body))}); err != nil {
t.Fatal(err)
}
_, _ = tw.Write(body)
}
_ = tw.Close()
return buf.Bytes()
}
func testGem(t *testing.T, platform string) []byte {
return buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", platform, 1)))
}
func TestParseGemAndInfoLine(t *testing.T) {
gem := testGem(t, "ruby")
s, err := ParseGem(gem)
if err != nil {
t.Fatal(err)
}
if s.FullName() != "unkin-e2e-hello-0.1.0" || !s.Date.Equal(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC)) {
t.Fatalf("spec = %+v", s)
}
if len(s.Dependencies) != 3 || s.Dependencies[1].Requirement.String() != "~> 2.1" {
t.Fatalf("deps = %+v", s.Dependencies)
}
sum := sha256.Sum256(gem)
want := "0.1.0 rack:>= 2.0&< 4,json:~> 2.1|checksum:" + hex.EncodeToString(sum[:]) + ",ruby:>= 2.7"
if got := InfoLine(s, gem); got != want {
t.Fatalf("InfoLine =\n %s\nwant\n %s", got, want)
}
plat, err := ParseGem(testGem(t, "x86_64-linux"))
if err != nil {
t.Fatal(err)
}
if plat.FullName() != "unkin-e2e-hello-0.1.0-x86_64-linux" || InfoKey(InfoLine(plat, nil)) != "0.1.0-x86_64-linux" {
t.Fatalf("platform gem: %s / %s", plat.FullName(), InfoLine(plat, nil))
}
}
func TestParseGemRejects(t *testing.T) {
big := []byte(testMetadata + "description: " + strings.Repeat("a", maxMetadataBytes) + "\n")
cases := map[string][]byte{
"not a tar": []byte("garbage"),
"no metadata": func() []byte { var b bytes.Buffer; tw := tar.NewWriter(&b); _ = tw.Close(); return b.Bytes() }(),
"bad name": buildGem(t, []byte(strings.Replace(testMetadata, "name: unkin-e2e-hello", "name: ../x", 1))),
"bad version": buildGem(t, []byte(strings.Replace(testMetadata, "version: 0.1.0", "version: 'a b'", 1))),
"bad dep op": buildGem(t, []byte(strings.Replace(testMetadata, `- - "~>"`, `- - "=~"`, 1))),
"metadata bomb": buildGem(t, big),
"bad platform": buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", "x/y", 1))),
"bad dependency": buildGem(t, []byte(strings.Replace(testMetadata, "name: json", "name: 'a|b'", 1))),
}
for name, gem := range cases {
if _, err := ParseGem(gem); err == nil {
t.Errorf("%s: accepted", name)
}
}
}
func TestVersionsLineMD5(t *testing.T) {
info := InfoBody([]string{"0.1.0 |checksum:aa", "0.2.0-java |checksum:bb"})
if string(info) != "---\n0.1.0 |checksum:aa\n0.2.0-java |checksum:bb\n" {
t.Fatalf("InfoBody = %q", info)
}
if got := VersionsLine("g", info); got != "g 0.1.0,0.2.0-java "+MD5Hex(info) {
t.Fatalf("VersionsLine = %q", got)
}
}
// Byte layouts below are taken from Ruby 3.3's Marshal.dump.
func TestRubyMarshalEncoding(t *testing.T) {
for n, want := range map[int]string{0: "\x00", 4: "\x09", 122: "\x7f", 123: "\x01\x7b", 255: "\x01\xff", 256: "\x02\x00\x01", 65536: "\x03\x00\x00\x01", -1: "\xfa", -123: "\x80"} {
var m rbMarshal
m.fixnum(n)
if m.String() != want {
t.Errorf("fixnum(%d) = %x, want %x", n, m.String(), want)
}
}
var m rbMarshal
m.str("3.5.22")
if m.String() != "I\"\x0b3.5.22\x06:\x06ET" {
t.Errorf("str = %q", m.String())
}
m.Reset()
m.version("0.1.0")
if m.String() != "U:\x11Gem::Version[\x06I\"\x0a0.1.0\x06:\x06ET" {
t.Errorf("version = %q", m.String())
}
m.Reset()
m.time(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC))
if want := "Iu:\x09Time\x0d\x20\xa6\x1f\xc0\x00\x00\x00\x00\x06:\x09zoneI\"\x08UTC\x06:\x06EF"; m.String() != want {
t.Errorf("time = %q, want %q", m.String(), want)
}
var epoch rbMarshal
epoch.time(time.Time{})
for _, y := range []int{1, 1899, 1900 + 0x10000} {
m.Reset()
m.time(time.Date(y, 1, 1, 0, 0, 0, 0, time.UTC))
if m.String() != epoch.String() {
t.Errorf("time(year %d) = %q, want the epoch encoding %q", y, m.String(), epoch.String())
}
}
}
func TestQuickSpec(t *testing.T) {
s, err := ParseGem(testGem(t, "ruby"))
if err != nil {
t.Fatal(err)
}
q, err := QuickSpec(s)
if err != nil {
t.Fatal(err)
}
q2, _ := QuickSpec(s)
if !bytes.Equal(q, q2) {
t.Fatal("QuickSpec is not deterministic")
}
zr, err := zlib.NewReader(bytes.NewReader(q))
if err != nil {
t.Fatal(err)
}
raw, _ := io.ReadAll(zr)
head := "\x04\x08u:\x17Gem::Specification"
if !strings.HasPrefix(string(raw), head) {
t.Fatalf("quick spec prefix = %q", raw[:30])
}
inner := string(raw[len(head):])
at := strings.Index(inner, "\x04\x08[\x18")
var length rbMarshal
length.fixnum(len(inner) - at)
if at < 1 || inner[:at] != length.String() {
t.Fatalf("inner dump header = %q", inner[:8])
}
for _, want := range []string{"o:\x14Gem::Dependency", ":\x0cruntime", ":\x10development", "source_code_uri"} {
if !strings.Contains(string(raw), want) {
t.Errorf("quick spec missing %q", want)
}
}
}
internal/provider/helm/helm.go
+214
View File
@@ -1,12 +1,24 @@
package helm
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"path"
"strings"
"time"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -39,6 +51,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewritesResponse() {}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" {
return nil, nil
@@ -56,3 +70,203 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
// ValidateUpload stores charts flat by filename so index.yaml can reference
// them with relative urls, as `helm repo index` does.
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := path.Base(path.Clean("/" + filePath))
if !strings.HasSuffix(filename, ".tgz") || filename == ".tgz" {
return "", "", fmt.Errorf("file must be a .tgz chart package")
}
return filename, "application/gzip", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
return map[string]any{
"filename": storagePath,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
reader, _, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
if err != nil {
slog.Error("helm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer func() { _ = reader.Close() }()
meta, err := parseChart(reader)
if err != nil {
slog.Error("helm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
if want := meta.Name + "-" + meta.Version + ".tgz"; storagePath != want {
slog.Error("helm metadata: filename does not match chart", "repo", repoName, "path", storagePath, "expected", want)
return
}
meta.RepoName = repoName
meta.FilePath = storagePath
meta.ContentHash = contentHash
store, ok := db.(provider.HelmMetadataStore)
if !ok {
slog.Error("helm metadata: store does not support helm metadata", "repo", repoName)
return
}
if err := store.InsertHelmMetadata(ctx, meta); err != nil {
slog.Error("helm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("helm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version)
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
deleter, ok := db.(provider.HelmMetadataDeleter)
if !ok {
return nil
}
return deleter.DeleteHelmMetadata(ctx, repoName, storagePath)
}
func isIndexPath(p string) bool {
return strings.TrimPrefix(path.Clean("/"+p), "/") == "index.yaml"
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
if !isIndexPath(reqPath) {
return false
}
body, err := p.GenerateLocalIndex(r.Context(), files, repoName, reqPath)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
w.Header().Set("Content-Type", "text/yaml")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(body)
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, reqPath string) ([]byte, error) {
if !isIndexPath(reqPath) {
return nil, fmt.Errorf("unsupported index path: %q", reqPath)
}
reader, ok := files.(provider.HelmMetadataReader)
if !ok {
return nil, errors.New("helm metadata not available")
}
metas, err := reader.ListHelmMetadataEntries(ctx, repoName)
if err != nil {
return nil, err
}
return generateIndex(metas)
}
// generateIndex builds index.yaml from stored Chart.yaml documents; it is
// stamped from the newest upload so every replica serves identical bytes.
func generateIndex(metas []provider.HelmMetadata) ([]byte, error) {
entries := map[string][]map[string]yaml.Node{}
var newest time.Time
budget := provider.MaxAliasExpandedNodes
for _, m := range metas {
var raw map[string]yaml.Node
if err := yaml.Unmarshal([]byte(m.ChartYAML), &raw); err != nil {
return nil, fmt.Errorf("chart %s: %w", m.FilePath, err)
}
chart, err := provider.ExpandYAMLAliases(raw, &budget)
if err != nil {
slog.Warn("helm index: dropping chart", "chart", m.Name, "path", m.FilePath, "error", err)
continue
}
chart["name"] = node(m.Name)
chart["version"] = node(m.Version)
chart["urls"] = node([]string{url.PathEscape(m.FilePath)})
chart["digest"] = node(strings.TrimPrefix(m.ContentHash, "sha256:"))
chart["created"] = node(m.CreatedAt.UTC().Format(time.RFC3339Nano))
entries[m.Name] = append(entries[m.Name], chart)
if m.CreatedAt.After(newest) {
newest = m.CreatedAt
}
}
return yaml.Marshal(map[string]any{
"apiVersion": "v1",
"entries": entries,
"generated": newest.UTC().Format(time.RFC3339Nano),
})
}
func node(v any) yaml.Node {
var n yaml.Node
_ = n.Encode(v)
return n
}
const (
maxChartYAML = 1 << 20
maxChartBytes = 50 << 20
maxChartEntries = 10000
)
var errChartTooLarge = fmt.Errorf("chart archive exceeds %d decompressed bytes", maxChartBytes)
type capReader struct {
r io.Reader
n int64
}
func (c *capReader) Read(p []byte) (int, error) {
if c.n <= 0 {
return 0, errChartTooLarge
}
if int64(len(p)) > c.n {
p = p[:c.n]
}
n, err := c.r.Read(p)
c.n -= int64(n)
return n, err
}
// parseChart reads the top-level <chart>/Chart.yaml from a packaged chart.
func parseChart(r io.Reader) (*provider.HelmMetadata, error) {
zr, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
tr := tar.NewReader(&capReader{r: zr, n: maxChartBytes})
for entries := 0; ; entries++ {
if entries >= maxChartEntries {
return nil, fmt.Errorf("chart archive exceeds %d entries", maxChartEntries)
}
hdr, err := tr.Next()
if err == io.EOF {
return nil, errors.New("no Chart.yaml in chart archive")
}
if err != nil {
return nil, err
}
dir, file := path.Split(path.Clean(hdr.Name))
if file != "Chart.yaml" || strings.Count(dir, "/") != 1 {
continue
}
raw, err := io.ReadAll(io.LimitReader(tr, maxChartYAML+1))
if err != nil {
return nil, err
}
if len(raw) > maxChartYAML {
return nil, fmt.Errorf("parse Chart.yaml: exceeds %d bytes", maxChartYAML)
}
var chart struct {
Name string `yaml:"name"`
Version string `yaml:"version"`
}
if err := yaml.Unmarshal(raw, &chart); err != nil {
return nil, fmt.Errorf("parse Chart.yaml: %w", err)
}
if chart.Name == "" || chart.Version == "" {
return nil, errors.New("chart metadata missing name or version")
}
return &provider.HelmMetadata{Name: chart.Name, Version: chart.Version, ChartYAML: string(raw)}, nil
}
}
internal/provider/helm/helm_local_test.go
+236
View File
@@ -0,0 +1,236 @@
package helm
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"io"
"net/http/httptest"
"strconv"
"strings"
"testing"
"time"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func chartTgz(t *testing.T, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for name, body := range files {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
type fakeBlobs struct{ data []byte }
func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
}
type fakeStore struct {
inserted *provider.HelmMetadata
metas []provider.HelmMetadata
}
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
func (f *fakeStore) InsertHelmMetadata(_ context.Context, m *provider.HelmMetadata) error {
f.inserted = m
return nil
}
func (f *fakeStore) ListHelmMetadataEntries(context.Context, string) ([]provider.HelmMetadata, error) {
return f.metas, nil
}
func (f *fakeStore) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (f *fakeStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestValidateUpload(t *testing.T) {
p := &Provider{}
for in, want := range map[string]string{
"mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
"charts/mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
"../../mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
} {
sp, ct, err := p.ValidateUpload(in)
if err != nil || sp != want || ct != "application/gzip" {
t.Errorf("ValidateUpload(%q) = %q %q %v", in, sp, ct, err)
}
}
for _, bad := range []string{"index.yaml", "chart.tar.gz", ".tgz", ""} {
if _, _, err := p.ValidateUpload(bad); err == nil {
t.Errorf("ValidateUpload(%q) accepted", bad)
}
}
}
func TestAfterUploadParsesTopLevelChart(t *testing.T) {
data := chartTgz(t, map[string]string{
"mychart/charts/sub/Chart.yaml": "apiVersion: v2\nname: sub\nversion: 9.9.9\n",
"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0\ndescription: hi\n",
"mychart/values.yaml": "a: 1\n",
})
store := &fakeStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "mychart-1.0.tgz", "sha256:ab", fakeBlobs{data}, store)
m := store.inserted
if m == nil || m.Name != "mychart" || m.Version != "1.0" || m.FilePath != "mychart-1.0.tgz" || m.ContentHash != "sha256:ab" {
t.Fatalf("inserted %+v", m)
}
}
func TestAfterUploadRejectsChartWithoutMetadata(t *testing.T) {
store := &fakeStore{}
data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\n"})
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{data}, store)
if store.inserted != nil {
t.Fatal("chart without version was recorded")
}
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{[]byte("not gzip")}, store)
if store.inserted != nil {
t.Fatal("non-gzip upload was recorded")
}
}
func TestServeLocalIndex(t *testing.T) {
t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
t2 := t1.Add(time.Hour)
store := &fakeStore{metas: []provider.HelmMetadata{
{FilePath: "mychart-1.0.tgz", ContentHash: "sha256:aa", Name: "mychart", Version: "1.0", CreatedAt: t1,
ChartYAML: "apiVersion: v2\nname: mychart\nversion: 1.0\nappVersion: \"2\"\ndescription: hi\n"},
{FilePath: "other-0.2.0.tgz", ContentHash: "sha256:bb", Name: "other", Version: "0.2.0", CreatedAt: t2,
ChartYAML: "apiVersion: v2\nname: other\nversion: 0.2.0\n"},
}}
p := &Provider{}
if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), store, "r", "mychart-1.0.tgz") {
t.Fatal("chart path handled as index")
}
rec := httptest.NewRecorder()
if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), store, "r", "./index.yaml") {
t.Fatal("index.yaml not handled")
}
if rec.Code != 200 || rec.Header().Get("Content-Type") != "text/yaml" {
t.Fatalf("code=%d ct=%q", rec.Code, rec.Header().Get("Content-Type"))
}
var idx struct {
APIVersion string `yaml:"apiVersion"`
Generated string `yaml:"generated"`
Entries map[string][]map[string]any
}
if err := yaml.Unmarshal(rec.Body.Bytes(), &idx); err != nil {
t.Fatal(err)
}
if idx.APIVersion != "v1" || idx.Generated != t2.Format(time.RFC3339Nano) {
t.Errorf("header %+v", idx)
}
e := idx.Entries["mychart"][0]
if e["digest"] != "aa" || e["description"] != "hi" || e["appVersion"] != "2" || e["apiVersion"] != "v2" {
t.Errorf("entry %v", e)
}
if urls, _ := e["urls"].([]any); len(urls) != 1 || urls[0] != "mychart-1.0.tgz" {
t.Errorf("urls %v", e["urls"])
}
if e["version"] != "1.0" || e["name"] != "mychart" {
t.Errorf("version must stay a string: %#v", e["version"])
}
if len(idx.Entries["other"]) != 1 {
t.Errorf("other entries %v", idx.Entries["other"])
}
again, err := p.GenerateLocalIndex(context.Background(), store, "r", "index.yaml")
if err != nil || !bytes.Equal(again, rec.Body.Bytes()) {
t.Errorf("index not deterministic: %v", err)
}
}
func TestAfterUploadRejectsFilenameMismatch(t *testing.T) {
store := &fakeStore{}
data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0.0\n"})
(&Provider{}).AfterUpload(context.Background(), "r", "other-1.0.0.tgz", "sha256:ab", fakeBlobs{data}, store)
if store.inserted != nil {
t.Fatal("chart indexed under a filename that does not match name-version")
}
}
func TestParseChartLimits(t *testing.T) {
chart := "apiVersion: v2\nname: c\nversion: 1.0.0\n"
for name, tc := range map[string]struct {
files map[string]string
want string
}{
"decompressed": {map[string]string{"c/a.bin": strings.Repeat("\x00", maxChartBytes+1)}, "decompressed bytes"},
"chart yaml": {map[string]string{"c/Chart.yaml": chart + "#" + strings.Repeat("x", maxChartYAML)}, "Chart.yaml: exceeds"},
} {
_, err := parseChart(bytes.NewReader(chartTgz(t, tc.files)))
if err == nil || !strings.Contains(err.Error(), tc.want) {
t.Errorf("%s: err = %v, want %q", name, err, tc.want)
}
}
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for i := 0; i <= maxChartEntries; i++ {
_ = tw.WriteHeader(&tar.Header{Name: "c/templates/" + strconv.Itoa(i), Mode: 0o644})
}
_ = tw.WriteHeader(&tar.Header{Name: "c/Chart.yaml", Mode: 0o644, Size: int64(len(chart))})
_, _ = tw.Write([]byte(chart))
_ = tw.Close()
_ = zw.Close()
if _, err := parseChart(&buf); err == nil || !strings.Contains(err.Error(), "entries") {
t.Errorf("entries: err = %v", err)
}
if m, err := parseChart(bytes.NewReader(chartTgz(t, map[string]string{"c/Chart.yaml": chart}))); err != nil || m.Name != "c" {
t.Errorf("valid chart: %v %v", m, err)
}
}
func TestIndexEscapesURLs(t *testing.T) {
out, err := generateIndex([]provider.HelmMetadata{{FilePath: "my chart-1.0.0.tgz", Name: "my chart", Version: "1.0.0", ChartYAML: "name: x\n"}})
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(out), "my%20chart-1.0.0.tgz") {
t.Errorf("url not escaped:\n%s", out)
}
}
func TestIndexExpandsAliasesBeforeOverride(t *testing.T) {
body, err := generateIndex([]provider.HelmMetadata{{
FilePath: "app-1.0.0.tgz", ContentHash: "sha256:aa", Name: "app", Version: "1.0.0",
ChartYAML: "apiVersion: v2\nname: app\nversion: &v 1.0.0\nappVersion: *v\n",
}})
if err != nil {
t.Fatal(err)
}
var idx struct {
Entries map[string][]map[string]any
}
if err := yaml.Unmarshal(body, &idx); err != nil {
t.Fatalf("index does not parse: %v\n%s", err, body)
}
if e := idx.Entries["app"]; len(e) != 1 || e[0]["appVersion"] != "1.0.0" {
t.Errorf("entries %v", idx.Entries)
}
}
internal/provider/metadata_alpine.go
+49
View File
@@ -0,0 +1,49 @@
package provider
import (
"context"
)
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
// Alpine-specific persistence surfaces. They are kept separate from the shared
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
// satisfies all three.
type AlpineMetadataStore interface {
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
}
type AlpineMetadataDeleter interface {
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
}
type AlpineMetadataReader interface {
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
}
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
// sha1 of the control gzip stream) and the download/installed sizes.
type AlpineMetadata struct {
RepoName string
FilePath string
ContentHash string
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
Name string // P:
Version string // V:
Arch string // A:
DownloadSize int64 // S: on-disk .apk size
InstalledSize int64 // I: unpacked size from .PKGINFO
Description string // T:
URL string // U:
License string // L:
Origin string // o:
Maintainer string // m:
BuildTime int64 // t:
Commit string // c:
ProviderPriority string // k:
Depends []string // D:
Provides []string // p:
InstallIf []string // i:
}
internal/provider/metadata_deb.go
+34
View File
@@ -0,0 +1,34 @@
package provider
import (
"context"
"time"
)
// DebMetadataReader is the read surface the deb LocalIndexer needs to
// regenerate a flat apt repository (Packages/Release) from stored rows.
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
// mirroring how the rpm provider reaches its RPMMetadataReader.
type DebMetadataReader interface {
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
}
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
// the full raw control stanza so the Packages index can be regenerated
// faithfully alongside the computed size/md5/sha256 apt requires.
type DebMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
Architecture string
Control string
Size int64
MD5 string
SHA256 string
// CreatedAt is the persisted insert time; the Release Date: is derived from
// the newest value so the index is byte-identical across replicas and
// regenerations (issue #117) rather than stamped from wall clock.
CreatedAt time.Time
}
internal/provider/metadata_helm.go
+30
View File
@@ -0,0 +1,30 @@
package provider
import (
"context"
"time"
)
type HelmMetadataStore interface {
InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error
}
type HelmMetadataDeleter interface {
DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error
}
type HelmMetadataReader interface {
ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error)
}
// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml
// entries carry every field helm reads.
type HelmMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
ChartYAML string
CreatedAt time.Time
}
internal/provider/metadata_puppet.go
+32
View File
@@ -0,0 +1,32 @@
package provider
import (
"context"
"time"
)
type PuppetMetadataStore interface {
InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error
}
type PuppetMetadataDeleter interface {
DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error
}
type PuppetMetadataReader interface {
ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error)
}
// PuppetMetadata is a local module release; MetadataJSON is the tarball's
// metadata.json verbatim so Forge release objects carry every field.
type PuppetMetadata struct {
RepoName string
FilePath string
ContentHash string
Slug string
Version string
MetadataJSON string
Size int64
MD5 string
CreatedAt time.Time
}
internal/provider/metadata_rpm.go
+60
View File
@@ -0,0 +1,60 @@
package provider
import (
"context"
"time"
)
type RPMMetadataReader interface {
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
}
type RPMMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires []RPMDep
Provides []RPMDep
Conflicts []RPMDep
Obsoletes []RPMDep
Files []RPMFile
Changelogs []RPMChangelog
// CreatedAt is the persisted upload timestamp; used as a stable, replica-independent
// value for the repodata <time>/<revision> fields so generated indexes are deterministic.
CreatedAt time.Time
}
type RPMDep struct {
Name string `json:"name"`
Flags string `json:"flags,omitempty"`
Epoch string `json:"epoch,omitempty"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
}
type RPMFile struct {
Path string `json:"path"`
Type string `json:"type,omitempty"`
}
type RPMChangelog struct {
Author string `json:"author"`
Date int64 `json:"date"`
Text string `json:"text"`
}
internal/provider/metadata_store.go
+15
View File
@@ -0,0 +1,15 @@
package provider
import (
"context"
)
type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
}
type MetadataDeleter interface {
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
}
internal/provider/npm/npm.go
+2
View File
@@ -37,6 +37,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewritesResponse() {}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" || !json.Valid(body) {
return nil, nil
internal/provider/nuget/nuget.go
+17 -5
View File
@@ -1,5 +1,6 @@
// Package nuget proxies a NuGet v3 feed: the service index, registration pages
// and flat container indexes are mutable, .nupkg/.nuspec downloads immutable.
// and flat container indexes are mutable; package, symbol, nuspec, readme and
// icon blobs are immutable.
package nuget
import (
@@ -7,6 +8,7 @@ import (
"encoding/json"
"net/http"
"net/url"
pathpkg "path"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
@@ -34,10 +36,12 @@ func (p *Provider) Classify(path string) provider.Mutability {
func (p *Provider) ContentType(path string) string {
switch {
case strings.HasSuffix(path, ".nupkg"):
case strings.HasSuffix(path, ".nupkg"), strings.HasSuffix(path, ".snupkg"), pathpkg.Base(path) == "icon":
return "application/octet-stream"
case strings.HasSuffix(path, ".nuspec"):
return "application/xml"
case pathpkg.Base(path) == "readme":
return "text/markdown"
}
return "application/json"
}
@@ -64,8 +68,8 @@ func (p *Provider) RewriteServed(body []byte, remote models.Remote, proxyBaseURL
if from == "" || !json.Valid(body) {
return body
}
to := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
return []byte(strings.ReplaceAll(string(body), from, to))
to, _ := json.Marshal(strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name)
return []byte(strings.ReplaceAll(string(body), from, string(to[1:len(to)-1])))
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
@@ -80,6 +84,14 @@ func origin(baseURL string) string {
return u.Scheme + "://" + u.Host
}
// isPackage matches flat container blobs: {id}.{ver}.(s)nupkg, {id}.nuspec and
// the {id}/{ver}/readme and {id}/{ver}/icon files.
func isPackage(path string) bool {
return strings.HasSuffix(path, ".nupkg") || strings.HasSuffix(path, ".nuspec")
for _, ext := range []string{".nupkg", ".snupkg", ".nuspec"} {
if strings.HasSuffix(path, ext) {
return true
}
}
base := pathpkg.Base(path)
return (base == "readme" || base == "icon") && strings.Count(strings.Trim(path, "/"), "/") >= 2
}
internal/provider/nuget/nuget_test.go
+26
View File
@@ -19,7 +19,11 @@ func TestClassify(t *testing.T) {
}{
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", provider.Immutable},
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec", provider.Immutable},
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.snupkg", provider.Immutable},
{"v3-flatcontainer/newtonsoft.json/13.0.3/readme", provider.Immutable},
{"v3-flatcontainer/newtonsoft.json/13.0.3/icon", provider.Immutable},
{"v3-flatcontainer/newtonsoft.json/index.json", provider.Mutable},
{"readme", provider.Mutable},
{"v3/registration5-gz-semver2/newtonsoft.json/index.json", provider.Mutable},
{"index.json", provider.Mutable},
}
@@ -30,6 +34,28 @@ func TestClassify(t *testing.T) {
}
}
func TestContentType(t *testing.T) {
p := &Provider{}
for path, want := range map[string]string{
"v3-flatcontainer/a/1.0.0/a.1.0.0.snupkg": "application/octet-stream",
"v3-flatcontainer/a/1.0.0/icon": "application/octet-stream",
"v3-flatcontainer/a/1.0.0/readme": "text/markdown",
"v3-flatcontainer/a/index.json": "application/json",
} {
if got := p.ContentType(path); got != want {
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
}
}
}
func TestRewriteServedEscapesJSON(t *testing.T) {
body := `{"@id":"https://api.nuget.org/v3-flatcontainer/"}`
got := (&Provider{}).RewriteServed([]byte(body), nugetOrg, `http://evil"<x>`)
if want := `{"@id":"http://evil\"\u003cx\u003e/api/v1/remote/nuget-org/v3-flatcontainer/"}`; string(got) != want {
t.Errorf("RewriteServed = %s, want %s", got, want)
}
}
func TestUpstreamURL(t *testing.T) {
p := &Provider{}
tests := []struct {
internal/provider/provider.go
-212
View File
@@ -3,9 +3,7 @@ package provider
import (
"context"
"fmt"
"io"
"net/http"
"time"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -34,216 +32,6 @@ type FileEntry struct {
ContentHash string
}
type FileStore interface {
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
ListPackages(ctx context.Context, repoName string) ([]string, error)
}
type LocalUploader interface {
ValidateUpload(filePath string) (storagePath, contentType string, err error)
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
}
type LocalIndexer interface {
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
}
type BlobReader interface {
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
}
type PostUploadHook interface {
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
}
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
// feeds generated repodata) after a local file is removed.
type PostDeleteHook interface {
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
}
type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
}
// QueryForwarder is a remote provider whose upstream answers depend on the
// query string (e.g. paginated JSON APIs). Only the named params for path are
// forwarded upstream and kept in the cache key, each as the canonical value
// its validator returns; a value the validator rejects is dropped.
type QueryForwarder interface {
ForwardedQuery(path string) map[string]QueryValidator
}
// QueryValidator canonicalises a raw query value, reporting false to drop it.
type QueryValidator func(v string) (string, bool)
// RemoteServer lets a remote provider fully answer a request itself instead of
// going through the byte-proxy engine. It is the remote-side analog of
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
// repodata from derived per-asset metadata and to redirect package downloads to
// a backend remote, without ever precaching the packages. Returning false lets
// the normal proxy path take over.
type RemoteServer interface {
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
}
// ServeRewriter rewrites a cached mutable remote response per request, for
// indexes whose URLs must name the requesting host and so cannot be rewritten
// before caching.
type ServeRewriter interface {
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
}
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
type RemoteMetadataStore interface {
RPMMetadataReader
MetadataStore
MetadataDeleter
}
type MetadataDeleter interface {
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
}
type RPMMetadataReader interface {
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
}
// DebMetadataReader is the read surface the deb LocalIndexer needs to
// regenerate a flat apt repository (Packages/Release) from stored rows.
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
// mirroring how the rpm provider reaches its RPMMetadataReader.
type DebMetadataReader interface {
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
}
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
// the full raw control stanza so the Packages index can be regenerated
// faithfully alongside the computed size/md5/sha256 apt requires.
type DebMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
Architecture string
Control string
Size int64
MD5 string
SHA256 string
// CreatedAt is the persisted insert time; the Release Date: is derived from
// the newest value so the index is byte-identical across replicas and
// regenerations (issue #117) rather than stamped from wall clock.
CreatedAt time.Time
}
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
// Alpine-specific persistence surfaces. They are kept separate from the shared
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
// satisfies all three.
type AlpineMetadataStore interface {
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
}
type AlpineMetadataDeleter interface {
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
}
type AlpineMetadataReader interface {
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
}
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
// sha1 of the control gzip stream) and the download/installed sizes.
type AlpineMetadata struct {
RepoName string
FilePath string
ContentHash string
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
Name string // P:
Version string // V:
Arch string // A:
DownloadSize int64 // S: on-disk .apk size
InstalledSize int64 // I: unpacked size from .PKGINFO
Description string // T:
URL string // U:
License string // L:
Origin string // o:
Maintainer string // m:
BuildTime int64 // t:
Commit string // c:
ProviderPriority string // k:
Depends []string // D:
Provides []string // p:
InstallIf []string // i:
}
type RPMMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires []RPMDep
Provides []RPMDep
Conflicts []RPMDep
Obsoletes []RPMDep
Files []RPMFile
Changelogs []RPMChangelog
// CreatedAt is the persisted upload timestamp; used as a stable, replica-independent
// value for the repodata <time>/<revision> fields so generated indexes are deterministic.
CreatedAt time.Time
}
type RPMDep struct {
Name string `json:"name"`
Flags string `json:"flags,omitempty"`
Epoch string `json:"epoch,omitempty"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
}
type RPMFile struct {
Path string `json:"path"`
Type string `json:"type,omitempty"`
}
type RPMChangelog struct {
Author string `json:"author"`
Date int64 `json:"date"`
Text string `json:"text"`
}
type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
}
type MemberIndex struct {
RemoteName string
Body []byte
}
var registry = map[models.PackageType]Provider{}
func Register(p Provider) {
internal/provider/puppet/local.go
+393
View File
@@ -0,0 +1,393 @@
package puppet
import (
"archive/tar"
"compress/gzip"
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"path"
"regexp"
"slices"
"strconv"
"strings"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
)
const filesPrefix = "v3/files/"
// releaseRe matches <owner>-<name>-<semver>, the Forge release slug and the
// tarball basename that `puppet module build` produces.
var releaseRe = regexp.MustCompile(`^([a-z0-9]+)-([a-z][a-z0-9_]*)-(\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?)$`)
func parseRelease(s string) (slug, version string, ok bool) {
m := releaseRe.FindStringSubmatch(strings.ToLower(s))
if m == nil {
return "", "", false
}
return m[1] + "-" + m[2], m[3], true
}
func normalizeSlug(s string) string {
return strings.ToLower(strings.ReplaceAll(s, "/", "-"))
}
// ValidateUpload stores tarballs under v3/files/ so Forge file_uri downloads
// resolve straight to the stored file.
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := strings.ToLower(path.Base(path.Clean("/" + filePath)))
if _, _, ok := parseRelease(strings.TrimSuffix(filename, ".tar.gz")); !ok || !strings.HasSuffix(filename, ".tar.gz") {
return "", "", fmt.Errorf("file must be named <owner>-<module>-<version>.tar.gz")
}
return filesPrefix + filename, "application/gzip", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
return map[string]any{
"filename": path.Base(storagePath),
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
reader, size, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
if err != nil {
slog.Error("puppet metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer func() { _ = reader.Close() }()
if size > maxModuleTarball {
slog.Error("puppet metadata: tarball too large", "repo", repoName, "path", storagePath, "size", size)
return
}
hash := md5.New()
src := io.TeeReader(io.LimitReader(reader, maxModuleTarball), hash)
meta, err := parseModule(src)
if err != nil {
slog.Error("puppet metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
if _, err := io.Copy(io.Discard, src); err != nil {
slog.Error("puppet metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slug, version, _ := parseRelease(strings.TrimSuffix(path.Base(storagePath), ".tar.gz"))
if meta.Slug != slug || meta.Version != version {
slog.Error("puppet metadata: metadata.json does not match filename", "repo", repoName, "path", storagePath, "name", meta.Slug, "version", meta.Version)
return
}
meta.RepoName = repoName
meta.FilePath = storagePath
meta.ContentHash = contentHash
meta.Size = size
meta.MD5 = hex.EncodeToString(hash.Sum(nil))
store, ok := db.(provider.PuppetMetadataStore)
if !ok {
slog.Error("puppet metadata: store does not support puppet metadata", "repo", repoName)
return
}
if err := store.InsertPuppetMetadata(ctx, meta); err != nil {
slog.Error("puppet metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("puppet metadata: parsed", "repo", repoName, "module", meta.Slug, "version", meta.Version)
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
deleter, ok := db.(provider.PuppetMetadataDeleter)
if !ok {
return nil
}
return deleter.DeletePuppetMetadata(ctx, repoName, storagePath)
}
const maxMetadataJSON = 1 << 20
var (
maxModuleTarball int64 = 100 << 20
maxModuleScan int64 = 200 << 20
)
// parseModule reads the top-level <dir>/metadata.json from a module tarball,
// scanning at most maxModuleScan decompressed bytes.
func parseModule(r io.Reader) (*provider.PuppetMetadata, error) {
zr, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
scan := &io.LimitedReader{R: zr, N: maxModuleScan}
tr := tar.NewReader(scan)
for {
hdr, err := tr.Next()
if scan.N <= 0 {
return nil, errors.New("module tarball exceeds decompressed scan limit before metadata.json")
}
if err == io.EOF {
return nil, errors.New("no metadata.json in module tarball")
}
if err != nil {
return nil, err
}
dir, file := path.Split(strings.TrimPrefix(path.Clean(hdr.Name), "./"))
if hdr.Typeflag != tar.TypeReg || file != "metadata.json" || strings.Count(dir, "/") != 1 {
continue
}
body, err := io.ReadAll(io.LimitReader(tr, maxMetadataJSON))
if err != nil {
return nil, err
}
var md struct {
Name string `json:"name"`
Version string `json:"version"`
}
if err := json.Unmarshal(body, &md); err != nil {
return nil, fmt.Errorf("parse metadata.json: %w", err)
}
if md.Name == "" || md.Version == "" {
return nil, errors.New("metadata.json missing name or version")
}
return &provider.PuppetMetadata{Slug: normalizeSlug(md.Name), Version: md.Version, MetadataJSON: string(body)}, nil
}
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
clean := strings.TrimPrefix(path.Clean("/"+reqPath), "/")
var list func(slug string) ([]provider.PuppetMetadata, error)
if reader, ok := files.(provider.PuppetMetadataReader); ok {
list = func(slug string) ([]provider.PuppetMetadata, error) {
metas, err := reader.ListPuppetMetadataEntries(r.Context(), repoName, slug)
sortReleases(metas)
return metas, err
}
} else {
list = func(string) ([]provider.PuppetMetadata, error) {
return nil, errors.New("puppet metadata not available")
}
}
switch {
case clean == "v3/releases":
serveReleaseList(w, r, list)
case strings.HasPrefix(clean, "v3/releases/"):
slug, version, ok := parseRelease(strings.TrimPrefix(clean, "v3/releases/"))
if !ok {
notFound(w)
return true
}
metas, err := list(slug)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
for _, m := range metas {
if m.Version == version {
writeJSON(w, releaseObject(m))
return true
}
}
notFound(w)
case strings.HasPrefix(clean, "v3/modules/"):
slug := normalizeSlug(strings.TrimPrefix(clean, "v3/modules/"))
metas, err := list(slug)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
if len(metas) == 0 {
notFound(w)
return true
}
writeJSON(w, moduleObject(metas))
default:
return false
}
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("puppet local index generation for virtual repos not supported")
}
func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) ([]provider.PuppetMetadata, error)) {
q := r.URL.Query()
slug := normalizeSlug(q.Get("module"))
limit, err := strconv.Atoi(q.Get("limit"))
if err != nil || limit <= 0 {
limit = 20
}
limit = min(limit, 100)
offset, err := strconv.Atoi(q.Get("offset"))
if err != nil || offset < 0 {
offset = 0
}
metas, err := list(slug)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
offset = min(offset, len(metas))
results := []map[string]any{}
for _, m := range metas[offset : offset+min(limit, len(metas)-offset)] {
results = append(results, releaseObject(m))
}
page := func(off int) any {
if off < 0 || off >= len(metas) && off != 0 {
return nil
}
v := url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}
if slug != "" {
v.Set("module", slug)
}
return "/v3/releases?" + v.Encode()
}
var previous any
if offset > 0 {
previous = page(max(offset-limit, 0))
}
writeJSON(w, map[string]any{
"pagination": map[string]any{
"limit": limit,
"offset": offset,
"first": page(0),
"previous": previous,
"current": page(offset),
"next": page(offset + limit),
"total": len(metas),
},
"results": results,
})
}
func owner(slug string) map[string]any {
name, _, _ := strings.Cut(slug, "-")
return map[string]any{"uri": "/v3/users/" + name, "slug": name, "username": name}
}
func moduleRef(slug string) map[string]any {
_, name, _ := strings.Cut(slug, "-")
return map[string]any{"uri": "/v3/modules/" + slug, "slug": slug, "name": name, "deprecated_at": nil, "owner": owner(slug)}
}
const forgeTime = "2006-01-02 15:04:05 -0700"
func releaseObject(m provider.PuppetMetadata) map[string]any {
slug := m.Slug + "-" + m.Version
return map[string]any{
"uri": "/v3/releases/" + slug,
"slug": slug,
"module": moduleRef(m.Slug),
"version": m.Version,
"metadata": json.RawMessage(m.MetadataJSON),
"tags": []string{},
"supported": false,
"pdk": false,
"file_uri": "/" + m.FilePath,
"file_size": m.Size,
"file_md5": m.MD5,
"file_sha256": strings.TrimPrefix(m.ContentHash, "sha256:"),
"downloads": 0,
"readme": nil,
"changelog": nil,
"license": nil,
"created_at": m.CreatedAt.Format(forgeTime),
"updated_at": m.CreatedAt.Format(forgeTime),
"deleted_at": nil,
}
}
// moduleObject expects metas sorted newest version first.
func moduleObject(metas []provider.PuppetMetadata) map[string]any {
obj := moduleRef(metas[0].Slug)
releases := make([]map[string]any, 0, len(metas))
for _, m := range metas {
slug := m.Slug + "-" + m.Version
releases = append(releases, map[string]any{
"uri": "/v3/releases/" + slug, "slug": slug, "version": m.Version,
"file_uri": "/" + m.FilePath, "file_size": m.Size, "supported": false,
"created_at": m.CreatedAt.Format(forgeTime), "deleted_at": nil,
})
}
obj["current_release"] = releaseObject(metas[0])
obj["releases"] = releases
obj["downloads"] = 0
obj["created_at"] = metas[len(metas)-1].CreatedAt.Format(forgeTime)
obj["updated_at"] = metas[0].CreatedAt.Format(forgeTime)
return obj
}
func writeJSON(w http.ResponseWriter, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
enc := json.NewEncoder(w)
enc.SetEscapeHTML(false)
_ = enc.Encode(v)
}
func notFound(w http.ResponseWriter) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusNotFound)
_, _ = io.WriteString(w, `{"message":"404 Not Found","errors":["The requested resource could not be found"]}`)
}
// sortReleases orders by module slug, then newest semver first.
func sortReleases(metas []provider.PuppetMetadata) {
slices.SortStableFunc(metas, func(a, b provider.PuppetMetadata) int {
if c := strings.Compare(a.Slug, b.Slug); c != 0 {
return c
}
return compareSemver(b.Version, a.Version)
})
}
func compareSemver(a, b string) int {
a, _, _ = strings.Cut(a, "+")
b, _, _ = strings.Cut(b, "+")
aCore, aPre, aHasPre := strings.Cut(a, "-")
bCore, bPre, bHasPre := strings.Cut(b, "-")
if c := compareIdents(strings.Split(aCore, "."), strings.Split(bCore, ".")); c != 0 {
return c
}
switch {
case aHasPre && !bHasPre:
return -1
case !aHasPre && bHasPre:
return 1
}
return compareIdents(strings.Split(aPre, "."), strings.Split(bPre, "."))
}
func compareIdents(a, b []string) int {
for i := 0; i < len(a) && i < len(b); i++ {
an, aErr := strconv.Atoi(a[i])
bn, bErr := strconv.Atoi(b[i])
var c int
switch {
case aErr == nil && bErr == nil:
c = an - bn
case aErr == nil:
c = -1
case bErr == nil:
c = 1
default:
c = strings.Compare(a[i], b[i])
}
if c != 0 {
return c
}
}
return len(a) - len(b)
}
internal/provider/puppet/local_test.go
+301
View File
@@ -0,0 +1,301 @@
package puppet
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func moduleTarball(t *testing.T, dir, metadata string) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for name, body := range map[string]string{dir + "/metadata.json": metadata, dir + "/manifests/init.pp": "class hello {}\n"} {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
t.Fatal(err)
}
_, _ = tw.Write([]byte(body))
}
_ = tw.Close()
_ = zw.Close()
return buf.Bytes()
}
const helloMetadata = `{"name":"unkin-hello","version":"0.1.0","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0"}]}`
type fakeBlobs struct{ data []byte }
func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
}
type fakeStore struct{ inserted *provider.PuppetMetadata }
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
func (f *fakeStore) InsertPuppetMetadata(_ context.Context, m *provider.PuppetMetadata) error {
f.inserted = m
return nil
}
type fakeReader struct{ metas []provider.PuppetMetadata }
func (f fakeReader) ListPuppetMetadataEntries(_ context.Context, _, slug string) ([]provider.PuppetMetadata, error) {
var out []provider.PuppetMetadata
for _, m := range f.metas {
if slug == "" || m.Slug == slug {
out = append(out, m)
}
}
return out, nil
}
func (fakeReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (fakeReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestValidateUpload(t *testing.T) {
p := &Provider{}
sp, ct, err := p.ValidateUpload("some/dir/Unkin-hello-0.1.0.tar.gz")
if err != nil || sp != "v3/files/unkin-hello-0.1.0.tar.gz" || ct != "application/gzip" {
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
}
for _, bad := range []string{"hello-0.1.0.tar.gz", "unkin-hello-0.1.tar.gz", "unkin-hello-0.1.0.zip", "unkin-Hello_x-0.1.0.tgz"} {
if _, _, err := p.ValidateUpload(bad); err == nil {
t.Errorf("%q accepted", bad)
}
}
}
func TestAfterUploadRecordsMetadata(t *testing.T) {
raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata)
store := &fakeStore{}
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
m := store.inserted
if m == nil {
t.Fatal("nothing inserted")
}
sum := md5.Sum(raw)
if m.Slug != "unkin-hello" || m.Version != "0.1.0" || m.MD5 != hex.EncodeToString(sum[:]) || m.Size != int64(len(raw)) || m.MetadataJSON != helloMetadata {
t.Errorf("got %+v", m)
}
}
func TestAfterUploadRejectsMismatchedFilename(t *testing.T) {
raw := moduleTarball(t, "unkin-hello-0.2.0", `{"name":"unkin-hello","version":"0.2.0"}`)
store := &fakeStore{}
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
if store.inserted != nil {
t.Error("mismatched version indexed")
}
}
func TestParseModuleRequiresTopLevelMetadata(t *testing.T) {
if _, err := parseModule(bytes.NewReader(moduleTarball(t, "a/b", helloMetadata))); err == nil {
t.Error("nested metadata.json accepted")
}
if _, err := parseModule(bytes.NewReader(moduleTarball(t, "x", `{"name":"unkin-hello"}`))); err == nil {
t.Error("missing version accepted")
}
}
func tarballOf(t *testing.T, entries ...*tar.Header) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for _, h := range entries {
if err := tw.WriteHeader(h); err != nil {
t.Fatal(err)
}
if h.Typeflag == tar.TypeReg {
body := helloMetadata
if !strings.HasSuffix(h.Name, "metadata.json") {
body = strings.Repeat("x", int(h.Size))
}
_, _ = tw.Write([]byte(body))
}
}
_ = tw.Close()
_ = zw.Close()
return buf.Bytes()
}
func TestParseModuleSkipsNonRegularMetadata(t *testing.T) {
raw := tarballOf(t,
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"},
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json/", Typeflag: tar.TypeDir, Mode: 0o755},
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))},
)
meta, err := parseModule(bytes.NewReader(raw))
if err != nil || meta.Slug != "unkin-hello" || meta.Version != "0.1.0" {
t.Fatalf("got %v, %v", meta, err)
}
}
func TestParseModuleCapsDecompressedScan(t *testing.T) {
defer func(n int64) { maxModuleScan = n }(maxModuleScan)
maxModuleScan = 64 << 10
raw := tarballOf(t,
&tar.Header{Name: "unkin-hello-0.1.0/big", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1 << 20},
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))},
)
if _, err := parseModule(bytes.NewReader(raw)); err == nil || !strings.Contains(err.Error(), "scan limit") {
t.Fatalf("expected scan limit error, got %v", err)
}
maxModuleScan = 2 << 20
if _, err := parseModule(bytes.NewReader(raw)); err != nil {
t.Fatalf("under cap: %v", err)
}
}
func TestAfterUploadCapsTarballSize(t *testing.T) {
defer func(n int64) { maxModuleTarball = n }(maxModuleTarball)
raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata)
maxModuleTarball = int64(len(raw)) - 1
store := &fakeStore{}
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
if store.inserted != nil {
t.Error("oversized tarball indexed")
}
maxModuleTarball = int64(len(raw))
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
if store.inserted == nil {
t.Error("tarball at cap not indexed")
}
}
func TestReleaseListClampsPagination(t *testing.T) {
files := releases("0.1.0", "0.2.0", "0.3.0")
for _, q := range []string{"offset=9223372036854775807", "offset=9223372036854775807&limit=9223372036854775807", "offset=-5&limit=-5", "limit=9223372036854775807"} {
code, body := get(t, files, "v3/releases?module=unkin-hello&"+q)
if code != http.StatusOK {
t.Fatalf("%s: code %d", q, code)
}
pg := body["pagination"].(map[string]any)
if l := pg["limit"].(float64); l < 1 || l > 100 {
t.Errorf("%s: limit %v", q, l)
}
if o := pg["offset"].(float64); o < 0 || o > 3 {
t.Errorf("%s: offset %v", q, o)
}
}
_, body := get(t, files, "v3/releases?module=unkin-hello&offset=9223372036854775807")
if len(body["results"].([]any)) != 0 || body["pagination"].(map[string]any)["next"] != nil {
t.Errorf("past-end page: %v", body)
}
}
func get(t *testing.T, files provider.FileStore, target string) (int, map[string]any) {
t.Helper()
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/x/"+target, nil)
path, _, _ := strings.Cut(target, "?")
if !(&Provider{}).ServeLocalIndex(rec, req, files, "forge", path) {
return 0, nil
}
var body map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &body)
return rec.Code, body
}
func releases(versions ...string) fakeReader {
var f fakeReader
for i, v := range versions {
f.metas = append(f.metas, provider.PuppetMetadata{
FilePath: "v3/files/unkin-hello-" + v + ".tar.gz", ContentHash: "sha256:feed", Slug: "unkin-hello",
Version: v, MetadataJSON: `{"name":"unkin-hello","version":"` + v + `"}`, MD5: "m" + v,
CreatedAt: time.Unix(int64(i), 0),
})
}
f.metas = append(f.metas, provider.PuppetMetadata{Slug: "other-mod", Version: "9.9.9", MetadataJSON: `{}`})
return f
}
func TestReleaseListPaginatesNewestFirst(t *testing.T) {
files := releases("0.1.0", "0.10.0", "0.2.0", "1.0.0-rc.1", "1.0.0")
code, body := get(t, files, "v3/releases?module=unkin/hello&limit=2")
if code != http.StatusOK {
t.Fatalf("code %d", code)
}
results := body["results"].([]any)
pg := body["pagination"].(map[string]any)
if len(results) != 2 || results[0].(map[string]any)["version"] != "1.0.0" || results[1].(map[string]any)["version"] != "1.0.0-rc.1" {
t.Fatalf("page 1: %v", results)
}
if pg["total"] != float64(5) || pg["next"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" || pg["previous"] != nil {
t.Errorf("pagination: %v", pg)
}
_, body = get(t, files, "v3/releases?module=unkin-hello&limit=2&offset=4")
results = body["results"].([]any)
pg = body["pagination"].(map[string]any)
if len(results) != 1 || results[0].(map[string]any)["version"] != "0.1.0" || pg["next"] != nil || pg["previous"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" {
t.Errorf("last page: %v %v", results, pg)
}
_, body = get(t, files, "v3/releases?module=nobody-none")
if len(body["results"].([]any)) != 0 {
t.Error("unknown module returned results")
}
}
func TestReleaseObject(t *testing.T) {
code, body := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.1.0")
if code != http.StatusOK {
t.Fatalf("code %d", code)
}
if body["file_uri"] != "/v3/files/unkin-hello-0.1.0.tar.gz" || body["file_md5"] != "m0.1.0" || body["file_sha256"] != "feed" {
t.Errorf("release: %v", body)
}
if body["metadata"].(map[string]any)["name"] != "unkin-hello" || body["module"].(map[string]any)["owner"].(map[string]any)["username"] != "unkin" {
t.Errorf("release refs: %v", body)
}
if code, _ := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.2.0"); code != http.StatusNotFound {
t.Errorf("missing version: %d", code)
}
}
func TestModuleObject(t *testing.T) {
code, body := get(t, releases("0.9.0", "0.10.0"), "v3/modules/unkin-hello")
if code != http.StatusOK {
t.Fatalf("code %d", code)
}
if body["slug"] != "unkin-hello" || body["name"] != "hello" || body["current_release"].(map[string]any)["version"] != "0.10.0" || len(body["releases"].([]any)) != 2 {
t.Errorf("module: %v", body)
}
if code, _ := get(t, releases(), "v3/modules/unkin-missing"); code != http.StatusNotFound {
t.Errorf("missing module: %d", code)
}
}
func TestFilesFallThrough(t *testing.T) {
if code, _ := get(t, releases(), "v3/files/unkin-hello-0.1.0.tar.gz"); code != 0 {
t.Error("file downloads must fall through to the stored file")
}
}
func TestCompareSemver(t *testing.T) {
ordered := []string{"0.1.0", "0.2.0", "0.10.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0", "1.0.1+build"}
for i := 1; i < len(ordered); i++ {
if compareSemver(ordered[i-1], ordered[i]) >= 0 || compareSemver(ordered[i], ordered[i-1]) <= 0 {
t.Errorf("%s should sort before %s", ordered[i-1], ordered[i])
}
}
if compareSemver("1.0.0+a", "1.0.0+b") != 0 {
t.Error("build metadata must not affect precedence")
}
}
internal/provider/puppet/puppet.go
+2
View File
@@ -39,6 +39,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewritesResponse() {}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" {
return nil, nil
internal/provider/pypi/pypi.go
+2
View File
@@ -52,6 +52,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewritesResponse() {}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" {
return nil, nil
internal/provider/rpm/github.go
+13 -12
View File
@@ -343,6 +343,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
return newEtag, false, err
}
// A transiently failed asset fails the scan so the ETag is not advanced and
// the retry re-derives it; an invalid package is skipped as no retry fixes it.
var failed []error
seen := map[string]bool{}
for _, rel := range releases {
if rel.Draft {
@@ -373,10 +376,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
meta, err := p.deriveAsset(ctx, remote, asset, fp)
if err != nil {
slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
if !errors.Is(err, provider.ErrInvalidPackage) {
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
}
continue
}
if err := store.InsertRPMMetadata(ctx, meta); err != nil {
slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
continue
}
slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -388,7 +395,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
}
}
return newEtag, true, nil
return newEtag, true, errors.Join(failed...)
}
type ghRelease struct {
@@ -569,12 +576,12 @@ func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote,
}
continue
}
return nil, fmt.Errorf("parse rpm header: %w", perr)
return nil, fmt.Errorf("%w: parse rpm header: %w", provider.ErrInvalidPackage, perr)
}
}
// rangeGet returns the first n bytes of downloadURL. full is true when the
// response body was shorter than n (i.e. we already have the whole object).
// response holds the whole object.
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
if err != nil {
@@ -596,15 +603,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
}
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
if err != nil {
return nil, false, err
}
full := int64(len(body)) < n
return body, full, nil
return provider.ReadPrefix(resp, n)
}
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -627,7 +628,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
}
h := sha256.New()
internal/provider/rpm/github_test.go
+34 -6
View File
@@ -1,11 +1,13 @@
package rpm
import (
"cmp"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@@ -24,15 +26,22 @@ import (
// fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
// mirroring the (repo_name, file_path) uniqueness of the real table.
type fakeStore struct {
mu sync.Mutex
rows map[string]provider.RPMMetadata
mu sync.Mutex
rows map[string]provider.RPMMetadata
insertFail map[string]int // package name -> inserts left to fail
}
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
func newFakeStore() *fakeStore {
return &fakeStore{rows: map[string]provider.RPMMetadata{}, insertFail: map[string]int{}}
}
func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.insertFail[m.Name] > 0 {
f.insertFail[m.Name]--
return errors.New("insert failed")
}
if _, ok := f.rows[m.FilePath]; ok {
return nil // ON CONFLICT DO NOTHING
}
@@ -79,15 +88,20 @@ type githubFixture struct {
releaseAuth string // Authorization header seen on the last releases request
assetAuth string // Authorization header seen on the last asset request
failStatus int // when set, the releases list answers this status (rate-limit style)
assetFail map[string]int // asset filename -> downloads left to answer failCode
failCode int // status for assetFail downloads; 0 = 500
shortRead map[string]int // asset filename -> ranged GETs left to close early
mu sync.Mutex
}
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
t.Helper()
f := &githubFixture{
rpmBytes: map[string][]byte{},
rangeHit: map[string]int{},
fullHit: map[string]int{},
rpmBytes: map[string][]byte{},
rangeHit: map[string]int{},
fullHit: map[string]int{},
assetFail: map[string]int{},
shortRead: map[string]int{},
}
f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
@@ -145,6 +159,17 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
}
rng := r.Header.Get("Range")
f.mu.Lock()
if f.assetFail[name] > 0 {
f.assetFail[name]--
code := cmp.Or(f.failCode, http.StatusInternalServerError)
f.mu.Unlock()
http.Error(w, "boom", code)
return
}
short := rng != "" && f.shortRead[name] > 0
if short {
f.shortRead[name]--
}
f.assetAuth = r.Header.Get("Authorization")
if rng != "" {
f.rangeHit[name]++
@@ -165,6 +190,9 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
end = len(body) - 1
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
if short {
end /= 2
}
w.Header().Set("Content-Length", strconv.Itoa(end+1))
w.WriteHeader(http.StatusPartialContent)
w.Write(body[:end+1])
internal/provider/rpm/syncer_test.go
+71
View File
@@ -399,3 +399,74 @@ func TestSyncerFailedScanRetriesAfterBackoff(t *testing.T) {
t.Fatalf("recovery did not refresh metadata: %d rows", len(rows))
}
}
// A transiently failed asset fails the scan: the ETag is not advanced, a retry
// is scheduled, and the retry against the unchanged release list derives it. An
// invalid package is skipped instead, since no retry can fix it.
func TestSyncerFailedAssetRetry(t *testing.T) {
for _, tc := range []struct {
name string
corrupt bool
failCode int
short bool
insert bool
wantRetry bool
}{
{name: "transient", wantRetry: true},
{name: "rate limited", failCode: http.StatusForbidden, wantRetry: true},
{name: "short read", short: true, wantRetry: true},
{name: "insert failure", insert: true, wantRetry: true},
{name: "invalid package", corrupt: true},
{name: "asset not found", failCode: http.StatusNotFound},
{name: "asset gone", failCode: http.StatusGone},
} {
t.Run(tc.name, func(t *testing.T) {
const other = "other-9-9.aarch64.rpm"
fx := newGitHubFixture(t, true)
fx.etag = `"v1"`
fx.rpmBytes[other] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
store := newFakeSyncStore()
switch {
case tc.corrupt:
fx.rpmBytes[other] = []byte("not a package")
case tc.short:
fx.shortRead[other] = 1
case tc.insert:
store.insertFail["other"] = 1
case tc.failCode != 0 && !tc.wantRetry:
fx.failCode, fx.assetFail[other] = tc.failCode, 1<<30
default:
fx.failCode, fx.assetFail[other] = tc.failCode, 1
}
s := newSyncer(store, newTestProvider(), testSyncConfig())
remote := fx.remote()
bg := context.Background()
s.process(bg, syncJob{remote: remote, prime: true})
_, pending := store.retryAt[remote.Name]
if pending != tc.wantRetry {
t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
}
if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
t.Fatalf("etag saved=%v after first scan", saved)
}
if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
t.Fatalf("healthy asset not served: %d rows", len(rows))
}
if !tc.wantRetry {
return
}
store.mu.Lock()
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
store.mu.Unlock()
s.process(bg, syncJob{remote: remote})
if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
}
if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 2 {
t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
}
})
}
}
internal/provider/syncretry.go
+45
View File
@@ -3,8 +3,10 @@ package provider
import (
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
)
@@ -43,6 +45,49 @@ func NewUpstreamStatusError(url string, resp *http.Response) *UpstreamStatusErro
return e
}
// ErrInvalidPackage marks an asset whose bytes are not a valid package. It is
// permanent, so a scan skips the asset instead of failing and retrying.
var ErrInvalidPackage = errors.New("invalid package")
// AssetStatusError classifies a non-success asset download. A missing or
// unsatisfiable asset is permanent; anything else (403/429 rate limits, 5xx) is
// transient.
func AssetStatusError(url string, resp *http.Response) error {
switch resp.StatusCode {
case http.StatusNotFound, http.StatusGone, http.StatusRequestedRangeNotSatisfiable:
return fmt.Errorf("%w: %s: status %d", ErrInvalidPackage, url, resp.StatusCode)
}
return NewUpstreamStatusError(url, resp)
}
// ReadPrefix reads up to n bytes of a 200 or 206 asset response. full reports
// that the body is the whole object (per Content-Length or the Content-Range
// total, or a clean EOF before n when the total is unknown); a body ending before n bytes that is not the whole object is a
// transient short read.
func ReadPrefix(resp *http.Response, n int64) (body []byte, full bool, err error) {
body, err = io.ReadAll(io.LimitReader(resp.Body, n))
if err != nil {
return nil, false, err
}
total := resp.ContentLength
unknown := resp.StatusCode == http.StatusOK && total == -1
if resp.StatusCode == http.StatusPartialContent {
total = -1
if _, t, ok := strings.Cut(resp.Header.Get("Content-Range"), "/"); ok {
if v, perr := strconv.ParseInt(t, 10, 64); perr == nil {
total = v
}
unknown = t == "*"
}
}
// net/http errors on a premature close, so a clean EOF before n with an unknown total is the whole object.
full = total == int64(len(body)) || (unknown && int64(len(body)) < n)
if !full && int64(len(body)) < n {
return nil, false, fmt.Errorf("short read: got %d of %d bytes", len(body), n)
}
return body, full, nil
}
// SyncResult is a background scan's outcome, recorded when its sync lease is
// released. A failed scan keeps the prior sync time and ETag and schedules a
// retry after Backoff, doubled per consecutive failure up to MaxBackoff, and
internal/provider/syncretry_test.go
+75
View File
@@ -3,7 +3,9 @@ package provider
import (
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
@@ -61,3 +63,76 @@ func TestNewSyncResult(t *testing.T) {
t.Fatalf("wrapped hint not clamped to ttl: retry in %v", until)
}
}
func TestAssetStatusErrorClassification(t *testing.T) {
for status, permanent := range map[int]bool{
http.StatusNotFound: true, http.StatusGone: true, http.StatusRequestedRangeNotSatisfiable: true,
http.StatusForbidden: false, http.StatusTooManyRequests: false, http.StatusInternalServerError: false, http.StatusBadGateway: false,
} {
err := AssetStatusError("u", &http.Response{StatusCode: status, Header: http.Header{}})
if errors.Is(err, ErrInvalidPackage) != permanent {
t.Errorf("status %d: permanent=%v, want %v", status, !permanent, permanent)
}
}
}
func TestReadPrefix(t *testing.T) {
cases := []struct {
name string
status int
cl int64
crange string
body string
n int64
wantFull bool
wantErr bool
}{
{name: "200 whole object", status: 200, cl: 5, body: "hello", n: 32, wantFull: true},
{name: "200 larger than n", status: 200, cl: 5, body: "hello", n: 4},
{name: "200 chunked shorter than n", status: 200, cl: -1, body: "hello", n: 32, wantFull: true},
{name: "200 chunked larger than n", status: 200, cl: -1, body: "hello", n: 4},
{name: "206 whole object", status: 206, crange: "bytes 0-4/5", body: "hello", n: 32, wantFull: true},
{name: "206 prefix", status: 206, crange: "bytes 0-3/100", body: "hell", n: 4},
{name: "206 short read", status: 206, crange: "bytes 0-31/100", body: "hello", n: 32, wantErr: true},
{name: "206 no content-range short", status: 206, body: "hello", n: 32, wantErr: true},
{name: "206 unknown total shorter than n", status: 206, cl: -1, crange: "bytes 0-4/*", body: "hello", n: 32, wantFull: true},
{name: "206 unknown total prefix", status: 206, cl: -1, crange: "bytes 0-3/*", body: "hell", n: 4},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
if tc.cl > 0 {
w.Header().Set("Content-Length", strconv.FormatInt(tc.cl, 10))
}
if tc.crange != "" {
w.Header().Set("Content-Range", tc.crange)
}
w.WriteHeader(tc.status)
_, _ = io.WriteString(w, tc.body)
w.(http.Flusher).Flush()
}))
defer srv.Close()
resp, err := http.Get(srv.URL)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if tc.cl == -1 && resp.ContentLength != -1 {
t.Fatalf("response not chunked: ContentLength=%d", resp.ContentLength)
}
body, full, err := ReadPrefix(resp, tc.n)
if (err != nil) != tc.wantErr {
t.Fatalf("err=%v, wantErr=%v", err, tc.wantErr)
}
if err != nil {
if errors.Is(err, ErrInvalidPackage) {
t.Fatalf("short read must be transient: %v", err)
}
return
}
if full != tc.wantFull || int64(len(body)) > tc.n {
t.Fatalf("full=%v len=%d, want full=%v", full, len(body), tc.wantFull)
}
})
}
}
internal/provider/terraform/terraform.go
+2
View File
@@ -73,6 +73,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewritesResponse() {}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if remote.ReleasesRemote == "" {
return nil, nil
internal/provider/yamlalias.go
+46
View File
@@ -0,0 +1,46 @@
package provider
import (
"errors"
"gopkg.in/yaml.v3"
)
const MaxAliasExpandedNodes = 100000
// ExpandYAMLAliases inlines alias targets and drops anchors so a node map
// stays valid when copied away from the document that held its anchors;
// budget caps nodes reached through aliases against alias bombs.
func ExpandYAMLAliases(m map[string]yaml.Node, budget *int) (map[string]yaml.Node, error) {
var expand func(n *yaml.Node, viaAlias bool) (*yaml.Node, error)
expand = func(n *yaml.Node, viaAlias bool) (*yaml.Node, error) {
if viaAlias {
*budget--
if *budget < 0 {
return nil, errors.New("yaml aliases expand to too many nodes")
}
}
if n.Kind == yaml.AliasNode {
return expand(n.Alias, true)
}
c := *n
c.Anchor = ""
c.Content = make([]*yaml.Node, len(n.Content))
for i, ch := range n.Content {
var err error
if c.Content[i], err = expand(ch, viaAlias); err != nil {
return nil, err
}
}
return &c, nil
}
out := make(map[string]yaml.Node, len(m))
for k, v := range m {
e, err := expand(&v, false)
if err != nil {
return nil, err
}
out[k] = *e
}
return out, nil
}
internal/proxy/engine.go
+37 -23
View File
@@ -134,10 +134,11 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path)
}
locked, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
token, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
if err != nil {
slog.Warn("lock acquire failed", "error", err)
}
locked := token != ""
if !locked {
// Another request holds the fetch lock. Poll the store until the leader
@@ -151,7 +152,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
}
if locked {
defer e.cache.ReleaseLock(ctx, remote.Name, path)
defer func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remote.Name, path, token) }()
}
if class == ClassMutable && remote.CheckMutable {
@@ -160,7 +161,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
notModified, err := e.checkUpstream(ctx, remote, path, etag, prov)
if err == nil && notModified {
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
_ = e.cache.SetETag(ctx, remote.Name, path, etag, etagTTL(ttl))
result, err := e.serveFromStore(ctx, remote, path)
if err == nil {
result.Source = "cache"
@@ -229,10 +230,11 @@ func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
return fmt.Errorf("get remote: %w", err)
}
if !wildcard {
if err := e.waitForLock(ctx, remoteName, path); err != nil {
release, err := e.Lock(ctx, remoteName, path, e.evictLockWait, fetchLockTTL)
if err != nil {
return err
}
defer func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remoteName, path) }()
defer release()
if err := e.db.DeleteArtifact(ctx, remoteName, path); err != nil {
return fmt.Errorf("delete artifact: %w", err)
}
@@ -265,28 +267,36 @@ func (e *Engine) evictPrefix(ctx context.Context, remoteName, prefix string) err
return e.cache.ForgetPrefix(ctx, remoteName, prefix)
}
// waitForLock takes the per-path fetch lock so an in-flight Fetch cannot
// re-set TTL/ETag keys after an evict. It fails with a 503 when the lock
// cannot be taken within evictLockWait or Redis errors.
func (e *Engine) waitForLock(ctx context.Context, remoteName, path string) error {
deadline := time.Now().Add(e.evictLockWait)
// Lock takes the per-path Redis lock (the one Fetch holds) and returns its
// release. It fails with a 503 when the lock cannot be taken within wait or
// Redis errors.
func (e *Engine) Lock(ctx context.Context, remoteName, path string, wait, ttl time.Duration) (func(), error) {
token, err := e.waitForLock(ctx, remoteName, path, wait, ttl)
if err != nil {
return nil, err
}
return func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remoteName, path, token) }, nil
}
func (e *Engine) waitForLock(ctx context.Context, remoteName, path string, wait, ttl time.Duration) (string, error) {
deadline := time.Now().Add(wait)
for {
ok, err := e.cache.AcquireLock(ctx, remoteName, path, fetchLockTTL)
if ok {
return nil
token, err := e.cache.AcquireLock(ctx, remoteName, path, ttl)
if token != "" {
return token, nil
}
if ctx.Err() != nil {
return ctx.Err()
return "", ctx.Err()
}
if err != nil {
return &ProxyError{Status: http.StatusServiceUnavailable, Message: fmt.Sprintf("fetch lock: %v", err)}
return "", &ProxyError{Status: http.StatusServiceUnavailable, Message: fmt.Sprintf("lock: %v", err)}
}
if time.Now().After(deadline) {
return &ProxyError{Status: http.StatusServiceUnavailable, Message: "fetch in progress, retry evict"}
return "", &ProxyError{Status: http.StatusServiceUnavailable, Message: "resource busy, retry later"}
}
select {
case <-ctx.Done():
return ctx.Err()
return "", ctx.Err()
case <-time.After(50 * time.Millisecond):
}
}
@@ -366,7 +376,7 @@ func (e *Engine) headUpstreamOnce(ctx context.Context, remote models.Remote, pat
req.Header.Set(k, v)
}
}
return http.DefaultClient.Do(req)
return clientForRemote(remote).Do(req)
}
resp, err := doHead(nil)
@@ -504,8 +514,9 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
}
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
if etag := resp.Header.Get("ETag"); etag != "" {
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
// A zero TTL never expires, so it never needs revalidating.
if remote.CheckMutable && ttl > 0 {
_ = e.cache.SetETag(ctx, remote.Name, path, resp.Header.Get("ETag"), etagTTL(ttl))
}
return &FetchResult{
@@ -533,9 +544,6 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
}
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
if etag := resp.Header.Get("ETag"); etag != "" {
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
}
reader, info, err := e.store.Download(ctx, casResult.S3Key)
if err != nil {
@@ -655,6 +663,12 @@ func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, pa
return resp.StatusCode == http.StatusNotModified, nil
}
// etagTTL keeps a mutable path's ETag well past its freshness key so an expired
// path revalidates with a conditional HEAD, while idle paths still age out.
func etagTTL(ttl time.Duration) time.Duration {
return max(10*ttl, 7*24*time.Hour)
}
func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration {
switch class {
case ClassImmutable:
internal/proxy/engine_test.go
+173 -17
View File
@@ -17,12 +17,14 @@ import (
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
goredis "github.com/redis/go-redis/v9"
)
var (
testEngine *Engine
testCache *cache.Redis
testDB *database.DB
testRedis string
upstream *httptest.Server
)
@@ -64,6 +66,7 @@ func TestMain(m *testing.M) {
}
testCache = redis
testRedis = redisURL
testDB = db
testEngine = NewEngine(db, redis, s3)
upstream = httptest.NewServer(http.HandlerFunc(mockUpstream))
@@ -216,6 +219,20 @@ func TestHead(t *testing.T) {
}
}
func TestHeadUpstreamUsesRemoteTimeout(t *testing.T) {
slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(3 * time.Second)
}))
defer slow.Close()
r := models.Remote{Name: "slow-head", PackageType: models.PackageGeneric, BaseURL: slow.URL, UpstreamResponseHeaderTimeout: 1}
start := time.Now()
_, err := (&Engine{}).headUpstreamOnce(context.Background(), r, "blob.bin", prov(t, models.PackageGeneric))
if err == nil || time.Since(start) > 2*time.Second {
t.Fatalf("HEAD ignored the remote's response header timeout: err=%v after %s", err, time.Since(start))
}
}
func TestStaleOnError(t *testing.T) {
requireStack(t)
ctx := context.Background()
@@ -260,30 +277,169 @@ func TestCircuitOpenServesStale(t *testing.T) {
res.Reader.Close()
}
// expire drops path's freshness key so the next fetch revalidates.
func expire(t *testing.T, remote, path string) {
t.Helper()
if err := testCache.SetTTL(context.Background(), remote, path, time.Millisecond); err != nil {
t.Fatal(err)
}
time.Sleep(5 * time.Millisecond)
}
func mutableFetch(t *testing.T, r models.Remote, path string) (string, string) {
t.Helper()
res, err := testEngine.Fetch(context.Background(), r, path, prov(t, models.PackageRPM))
if err != nil {
t.Fatalf("fetch: %v", err)
}
return readAll(t, res), res.Source
}
const repomd = "8/Everything/x86_64/repodata/repomd.xml"
func TestMutableRevalidation(t *testing.T) {
requireStack(t)
srv, rev, conditional := changingUpstream(t)
r := seed(t, models.Remote{Name: "eng-reval", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
if body, _ := mutableFetch(t, r, repomd); body != "rev1:/"+repomd {
t.Fatalf("initial fetch = %q", body)
}
expire(t, r.Name, repomd)
if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" {
t.Errorf("after expiry = %q from %s, want cached rev1", body, src)
}
if n := conditional.Load(); n != 1 {
t.Errorf("conditional requests = %d, want 1 revalidation", n)
}
if fresh, _ := testCache.CheckTTL(context.Background(), r.Name, repomd); !fresh {
t.Error("304 should re-arm the freshness key")
}
rev.Store("rev2")
expire(t, r.Name, repomd)
if body, src := mutableFetch(t, r, repomd); body != "rev2:/"+repomd || src != "remote" {
t.Errorf("after upstream change = %q from %s, want remote rev2", body, src)
}
}
func etagKeyTTL(t *testing.T, remote, path string) time.Duration {
t.Helper()
opts, err := goredis.ParseURL(testRedis)
if err != nil {
t.Fatal(err)
}
c := goredis.NewClient(opts)
defer func() { _ = c.Close() }()
d, err := c.PTTL(context.Background(), "etag:"+remote+":"+path).Result()
if err != nil {
t.Fatal(err)
}
return d
}
// The ETag is only kept when it can be used, and never without an expiry.
func TestMutableETagStorage(t *testing.T) {
requireStack(t)
srv, _, _ := changingUpstream(t)
for _, tc := range []struct {
name string
check bool
ttl int
want time.Duration
}{
{"eng-etag-off", false, 60, -2},
{"eng-etag-never", true, 0, -2},
{"eng-etag-short", true, 60, 7 * 24 * time.Hour},
{"eng-etag-long", true, 86400, 10 * 24 * time.Hour},
} {
r := seed(t, models.Remote{Name: tc.name, PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: tc.check, MutableTTL: tc.ttl})
mutableFetch(t, r, repomd)
got := etagKeyTTL(t, tc.name, repomd)
if tc.want < 0 {
if got != tc.want {
t.Errorf("%s: etag key ttl = %v, want no key", tc.name, got)
}
continue
}
if got <= tc.want-time.Minute || got > tc.want {
t.Errorf("%s: etag key ttl = %v, want ~%v", tc.name, got, tc.want)
}
}
}
// Immutable paths are never revalidated, so their upstream ETag is not kept.
func TestImmutableFetchStoresNoETag(t *testing.T) {
requireStack(t)
srv, _, _ := changingUpstream(t)
r := seed(t, models.Remote{Name: "eng-etag-imm", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
const rpm = "Packages/e2e-1.0-1.noarch.rpm"
if body, src := mutableFetch(t, r, rpm); body != "rev1:/"+rpm || src != "remote" {
t.Fatalf("fetch = %q from %s", body, src)
}
if got := etagKeyTTL(t, r.Name, rpm); got != -2 {
t.Errorf("etag key ttl = %v, want no key", got)
}
}
// A 304 for a path whose S3 body is gone falls back to a full refetch.
func TestNotModifiedWithEvictedBodyRefetches(t *testing.T) {
requireStack(t)
srv, _, conditional := changingUpstream(t)
r := seed(t, models.Remote{Name: "eng-304-gone", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
mutableFetch(t, r, repomd)
if err := testEngine.store.Delete(context.Background(), storage.IndexKey(r.Name, repomd)); err != nil {
t.Fatal(err)
}
expire(t, r.Name, repomd)
if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "remote" {
t.Errorf("got %q from %s, want refetched rev1 from remote", body, src)
}
if n := conditional.Load(); n != 1 {
t.Errorf("conditional requests = %d, want the 304 HEAD first", n)
}
}
// A failing revalidation HEAD still serves the stored copy with stale_on_error.
func TestStaleOnErrorWhenRevalidationFails(t *testing.T) {
requireStack(t)
srv, _, _ := changingUpstream(t)
r := seed(t, models.Remote{Name: "eng-stale-head", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60, StaleOnError: true})
mutableFetch(t, r, repomd)
srv.Close()
expire(t, r.Name, repomd)
if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" {
t.Errorf("got %q from %s, want stale rev1 from cache", body, src)
}
}
// After a remote is deleted and flushed, a re-created remote of the same name
// fetches from its own upstream instead of revalidating the old ETag.
func TestRecreatedRemoteHasNoStaleState(t *testing.T) {
requireStack(t)
ctx := context.Background()
r := seed(t, models.Remote{Name: "eng-npm", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true})
p := prov(t, models.PackageNPM)
oldSrv, _, _ := changingUpstream(t)
r := seed(t, models.Remote{Name: "eng-recreate", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: oldSrv.URL, CheckMutable: true, MutableTTL: 60})
mutableFetch(t, r, repomd)
res, err := testEngine.Fetch(ctx, r, "pkg", p)
if err != nil {
t.Fatalf("initial mutable fetch: %v", err)
if err := testDB.DeleteRemote(ctx, r.Name); err != nil {
t.Fatal(err)
}
res.Reader.Close()
// Expire only the freshness marker; the ETag persists, forcing a
// conditional revalidation that the upstream answers with 304.
testCache.SetTTL(ctx, "eng-npm", "pkg", time.Millisecond)
time.Sleep(10 * time.Millisecond)
res, err = testEngine.Fetch(ctx, r, "pkg", p)
if err != nil {
t.Fatalf("revalidation fetch: %v", err)
if err := testCache.FlushRemote(ctx, r.Name); err != nil {
t.Fatal(err)
}
if res.Source != "cache" {
t.Errorf("revalidated response should come from cache, got %s", res.Source)
newSrv, rev, conditional := changingUpstream(t)
rev.Store("rev9")
r.BaseURL = newSrv.URL
seed(t, r)
if body, src := mutableFetch(t, r, repomd); body != "rev9:/"+repomd || src != "remote" {
t.Errorf("got %q from %s, want rev9 from the new upstream", body, src)
}
if n := conditional.Load(); n != 0 {
t.Errorf("conditional requests = %d, want none after flush", n)
}
res.Reader.Close()
}
func TestBearerTokenFlow(t *testing.T) {
internal/proxy/evict_test.go
+12 -10
View File
@@ -59,7 +59,7 @@ type cached struct{ row, index, ttl, etag bool }
// Mutable indexes live in the S3 index; immutable blobs get an artifact row.
var (
indexCached = cached{index: true, ttl: true, etag: true}
blobCached = cached{row: true, ttl: true, etag: true}
blobCached = cached{row: true, ttl: true}
)
func layers(t *testing.T, remote, path string) cached {
@@ -152,13 +152,15 @@ func TestEvictUnknownRemote(t *testing.T) {
}
}
func holdLock(t *testing.T, remote, path string) {
func holdLock(t *testing.T, remote, path string) string {
t.Helper()
ctx := context.Background()
if ok, err := testCache.AcquireLock(ctx, remote, path, time.Minute); !ok || err != nil {
t.Fatalf("acquire: %v %v", ok, err)
token, err := testCache.AcquireLock(ctx, remote, path, time.Minute)
if token == "" || err != nil {
t.Fatalf("acquire: %q %v", token, err)
}
t.Cleanup(func() { _ = testCache.ReleaseLock(ctx, remote, path) })
t.Cleanup(func() { _ = testCache.ReleaseLock(ctx, remote, path, token) })
return token
}
func TestEvictWaitsForFetchLock(t *testing.T) {
@@ -167,7 +169,7 @@ func TestEvictWaitsForFetchLock(t *testing.T) {
r := rpmRemote(t, "evict-lock", srv.URL)
ctx := context.Background()
const path = "8/Everything/x86_64/repodata/repomd.xml"
holdLock(t, r.Name, path)
token := holdLock(t, r.Name, path)
done := make(chan error, 1)
go func() { done <- testEngine.Evict(ctx, r.Name, path) }()
select {
@@ -175,13 +177,13 @@ func TestEvictWaitsForFetchLock(t *testing.T) {
t.Fatalf("evict returned while fetch lock held: %v", err)
case <-time.After(200 * time.Millisecond):
}
_ = testCache.ReleaseLock(ctx, r.Name, path)
_ = testCache.ReleaseLock(ctx, r.Name, path, token)
if err := <-done; err != nil {
t.Fatalf("evict: %v", err)
}
ok, err := testCache.AcquireLock(ctx, r.Name, path, time.Second)
if !ok || err != nil {
t.Fatalf("lock still held after evict: %v %v", ok, err)
token, err := testCache.AcquireLock(ctx, r.Name, path, time.Second)
if token == "" || err != nil {
t.Fatalf("lock still held after evict: %q %v", token, err)
}
}
internal/server/server.go
+3 -11
View File
@@ -21,20 +21,10 @@ import (
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/gc"
"git.unkin.net/unkin/artifactapi/internal/githubauth"
_ "git.unkin.net/unkin/artifactapi/internal/provider/all"
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
_ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/tfsign"
@@ -176,6 +166,8 @@ func (s *Server) routes() chi.Router {
r.Mount(tfregistry.MountPath, s.tfRegistry.Routes())
proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
s.virtEngine.NPMLocal = proxyHandler.LocalNPMPackument
s.virtEngine.GemsLocal = proxyHandler.LocalGemsIndex
r.Mount("/api/v1", proxyHandler.Routes())
r.Mount("/v2", proxyHandler.DockerV2Routes())
internal/virtual/engine.go
+83 -39
View File
@@ -26,18 +26,38 @@ type Engine struct {
proxyEngine *proxy.Engine
getRemote func(context.Context, string) (*models.Remote, error)
rpmMember func(context.Context, string) (*RPMMember, error)
npmMember func(ctx context.Context, name, pkg string) (*NPMMember, error)
// NPMLocal returns a local npm repo's packument with tarball paths relative
// to the repo, or ErrNotFound.
NPMLocal func(ctx context.Context, repo, pkg string) ([]byte, error)
gemsMember func(ctx context.Context, name, path string) (*GemsMember, error)
// GemsLocal returns a local gems repo's versions, names or info/<gem>, or
// ErrNotFound.
GemsLocal func(ctx context.Context, repo, path string) ([]byte, error)
mergeTTL time.Duration
sf singleflight.Group
mu sync.Mutex
rpm map[string]*rpmGen
sf singleflight.Group
mu sync.Mutex
gens map[string]*repoGen
}
// rpmGen holds a virtual's current merge plus the previous one, so a client
// holding the previous repomd can still fetch its content-hashed data files.
type rpmGen struct {
cur, prev *RPMRepo
// mergedRepo is a merged index document plus the files it references.
type mergedRepo interface {
index() []byte
files() map[string][]byte
}
func (r *RPMRepo) index() []byte { return r.Repomd }
func (r *RPMRepo) files() map[string][]byte { return r.Files }
// repoGen holds a virtual's current merge plus the previous one, so a client
// holding the previous index can still fetch the files it references.
type repoGen struct {
cur, prev mergedRepo
at time.Time
}
@@ -46,6 +66,8 @@ const rpmMergeTTL = 60 * time.Second
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
e := &Engine{db: db, proxyEngine: proxyEngine, getRemote: db.GetRemote, mergeTTL: rpmMergeTTL}
e.rpmMember = e.fetchRPMMember
e.npmMember = e.fetchNPMMember
e.gemsMember = e.fetchGemsMember
return e
}
@@ -170,17 +192,28 @@ var (
ErrBadPath = errors.New("invalid path")
)
// MemberRedirect maps an rpm virtual package path (<member>/<href>, as written
// by MergeRPM) to the owning member's route. ok is false for any other path;
// ErrBadPath means the path is absolute or escapes the member.
// MemberRedirect maps an rpm or npm virtual package path (<member>/<href>, as
// written by MergeRPM/MergeNPM) to the owning member's route. ok is false for
// any other path; ErrBadPath means the path is absolute or escapes the member.
func (e *Engine) MemberRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, bool, error) {
if virt.PackageType != models.PackageRPM || strings.HasPrefix(path, "repodata/") {
switch virt.PackageType {
case models.PackageRPM:
if strings.HasPrefix(path, "repodata/") {
return "", false, nil
}
case models.PackageNPM:
default:
return "", false, nil
}
name, rest, found := strings.Cut(path, "/")
if !found || !slices.Contains(virt.Members, name) {
return "", false, nil
}
return e.memberLocation(ctx, name, rest, rawQuery, proxyBaseURL)
}
// memberLocation returns the URL of rest on member name's own route.
func (e *Engine) memberLocation(ctx context.Context, name, rest, rawQuery, proxyBaseURL string) (string, bool, error) {
segs, err := memberSegments(rest)
if err != nil {
return "", false, err
@@ -217,54 +250,57 @@ func (e *Engine) fetchRPM(ctx context.Context, virt models.Virtual, path string)
if !strings.HasPrefix(path, "repodata/") {
return nil, "", ErrNotFound
}
key := "rpm\x00" + virt.Name
if path != "repodata/repomd.xml" {
if body, ok := e.cachedRPMFile(virt.Name, path); ok {
if body, ok := e.cachedFile(key, path); ok {
return body, "application/gzip", nil
}
}
repo, err := e.mergedRPM(ctx, virt)
repo, err := e.merged(ctx, key, func(ctx context.Context) (mergedRepo, error) {
return e.mergeRPM(ctx, virt)
})
if err != nil {
return nil, "", err
}
if path == "repodata/repomd.xml" {
return repo.Repomd, "application/xml", nil
return repo.index(), "application/xml", nil
}
if body, ok := repo.Files[path]; ok {
if body, ok := repo.files()[path]; ok {
return body, "application/gzip", nil
}
return nil, "", ErrNotFound
}
// mergedRPM returns the virtual's merge, reusing it for mergeTTL and
// collapsing concurrent merges of the same virtual into one.
// merged returns the merge cached under key, reusing it for mergeTTL and
// collapsing concurrent merges of the same key into one.
// ponytail: per-replica cache; behind a non-sticky LB a data request landing on
// another replica re-merges and 404s if a member changed in between. Move to the
// shared redis cache if that shows up.
func (e *Engine) mergedRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, error) {
func (e *Engine) merged(ctx context.Context, key string, merge func(context.Context) (mergedRepo, error)) (mergedRepo, error) {
e.mu.Lock()
g := e.rpm[virt.Name]
g := e.gens[key]
if g != nil && time.Since(g.at) < e.mergeTTL {
e.mu.Unlock()
return g.cur, nil
}
e.mu.Unlock()
v, err, _ := e.sf.Do(virt.Name, func() (any, error) {
repo, err := e.mergeRPM(context.WithoutCancel(ctx), virt)
v, err, _ := e.sf.Do(key, func() (any, error) {
repo, err := merge(context.WithoutCancel(ctx))
if err != nil {
return nil, err
}
e.mu.Lock()
defer e.mu.Unlock()
if e.rpm == nil {
e.rpm = map[string]*rpmGen{}
if e.gens == nil {
e.gens = map[string]*repoGen{}
}
g := e.rpm[virt.Name]
g := e.gens[key]
switch {
case g == nil:
e.rpm[virt.Name] = &rpmGen{cur: repo, at: time.Now()}
case string(g.cur.Repomd) == string(repo.Repomd):
e.gens[key] = &repoGen{cur: repo, at: time.Now()}
case string(g.cur.index()) == string(repo.index()):
g.at = time.Now()
repo = g.cur
default:
@@ -275,18 +311,31 @@ func (e *Engine) mergedRPM(ctx context.Context, virt models.Virtual) (*RPMRepo,
if err != nil {
return nil, err
}
return v.(*RPMRepo), nil
return v.(mergedRepo), nil
}
func (e *Engine) mergeRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, error) {
members := make([]RPMMember, len(virt.Members))
members, err := fetchMembers(virt, func(name string) (*RPMMember, error) { return e.rpmMember(ctx, name) })
if err != nil {
return nil, err
}
repo, err := MergeRPM(members)
if err != nil {
return nil, fmt.Errorf("merge rpm repodata: %w", err)
}
return repo, nil
}
// fetchMembers reads every member in parallel, failing if any member fails.
func fetchMembers[T any](virt models.Virtual, fetch func(name string) (*T, error)) ([]T, error) {
members := make([]T, len(virt.Members))
errs := make([]error, len(virt.Members))
var wg sync.WaitGroup
for i, name := range virt.Members {
wg.Add(1)
go func() {
defer wg.Done()
m, err := e.rpmMember(ctx, name)
m, err := fetch(name)
if err != nil {
errs[i] = fmt.Errorf("member %q: %w", name, err)
return
@@ -299,26 +348,21 @@ func (e *Engine) mergeRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, e
if err := errors.Join(errs...); err != nil {
return nil, fmt.Errorf("virtual %q: %v", virt.Name, err)
}
repo, err := MergeRPM(members)
if err != nil {
return nil, fmt.Errorf("merge rpm repodata: %w", err)
}
return repo, nil
return members, nil
}
func (e *Engine) cachedRPMFile(virt, path string) ([]byte, bool) {
func (e *Engine) cachedFile(key, path string) ([]byte, bool) {
e.mu.Lock()
defer e.mu.Unlock()
g := e.rpm[virt]
g := e.gens[key]
if g == nil {
return nil, false
}
for _, r := range []*RPMRepo{g.cur, g.prev} {
for _, r := range []mergedRepo{g.cur, g.prev} {
if r == nil {
continue
}
if body, ok := r.Files[path]; ok {
if body, ok := r.files()[path]; ok {
return body, true
}
}
internal/virtual/engine_test.go
+50
View File
@@ -156,3 +156,53 @@ func TestMemberRedirect(t *testing.T) {
}
}
}
type fakeRepo struct {
idx string
fs map[string][]byte
}
func (r *fakeRepo) index() []byte { return []byte(r.idx) }
func (r *fakeRepo) files() map[string][]byte { return r.fs }
func genRepo(gen string) func(context.Context) (mergedRepo, error) {
return func(context.Context) (mergedRepo, error) {
return &fakeRepo{idx: gen, fs: map[string][]byte{gen + ".gz": []byte(gen), "fixed": []byte(gen)}}, nil
}
}
func TestMergedRetainsPreviousGeneration(t *testing.T) {
e := fakeEngine(nil)
for _, gen := range []string{"g1", "g2", "g3"} {
if _, err := e.merged(context.Background(), "k", genRepo(gen)); err != nil {
t.Fatal(err)
}
}
for path, want := range map[string]string{"g3.gz": "g3", "g2.gz": "g2", "fixed": "g3"} {
if body, ok := e.cachedFile("k", path); !ok || string(body) != want {
t.Errorf("%s: got (%q, %v), want %q", path, body, ok, want)
}
}
if _, ok := e.cachedFile("k", "g1.gz"); ok {
t.Error("generation two merges back must be dropped")
}
}
func TestMergedReusesWithinTTLAndIsolatesKeys(t *testing.T) {
e := fakeEngine(nil)
e.mergeTTL = time.Minute
for _, step := range []struct{ key, gen, want string }{
{"a", "a1", "a1"}, {"b", "b1", "b1"}, {"a", "a2", "a1"},
} {
repo, err := e.merged(context.Background(), step.key, genRepo(step.gen))
if err != nil || string(repo.index()) != step.want {
t.Fatalf("%s: got %v (err %v), want %s", step.key, repo, err, step.want)
}
}
if _, ok := e.cachedFile("b", "a1.gz"); ok {
t.Error("key b served key a's file")
}
if _, ok := e.cachedFile("c", "a1.gz"); ok {
t.Error("unknown key served a file")
}
}
internal/virtual/gems_merger.go
+294
View File
@@ -0,0 +1,294 @@
package virtual
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"slices"
"strings"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider/gems"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const (
maxGemsInfos = 1024
maxGemsCandidates = 8
)
// GemsMember is one member's compact index document; Body is nil when the
// member does not have it.
type GemsMember struct {
RemoteName string
Body []byte
}
// GemsDoc is a merged compact index document. Owners maps each /info
// version[-platform] to the member serving it.
type GemsDoc struct {
Body []byte
Owners map[string]string
}
func (d *GemsDoc) index() []byte { return d.Body }
func (d *GemsDoc) files() map[string][]byte { return nil }
// MergeGemsInfo unions /info lines across members; the first member wins a
// duplicate version-platform. A gem only one member has is passed through
// verbatim so its md5 matches that member's /versions. ErrNotFound means no
// member has the gem.
func MergeGemsInfo(members []GemsMember) (*GemsDoc, error) {
var have []GemsMember
for _, m := range members {
if m.Body != nil {
have = append(have, m)
}
}
if len(have) == 0 {
return nil, ErrNotFound
}
d := &GemsDoc{Owners: map[string]string{}}
var lines []string
for _, m := range have {
for _, l := range gems.InfoLines(m.Body) {
k := gems.InfoKey(l)
if _, dup := d.Owners[k]; dup {
continue
}
d.Owners[k] = m.RemoteName
lines = append(lines, l)
}
}
if len(have) == 1 {
d.Body = have[0].Body
} else {
d.Body = gems.InfoBody(lines)
}
return d, nil
}
// versionsLines returns the entry lines of a /versions document.
func versionsLines(m GemsMember) ([]byte, error) {
if m.Body == nil {
return nil, fmt.Errorf("member %q has no versions", m.RemoteName)
}
if bytes.HasPrefix(m.Body, []byte("---\n")) {
return m.Body[4:], nil
}
_, rest, ok := bytes.Cut(m.Body, []byte("\n---\n"))
if !ok {
return nil, fmt.Errorf("member %q versions has no --- separator", m.RemoteName)
}
return rest, nil
}
func gemName(line []byte) string {
n, _, _ := bytes.Cut(line, []byte(" "))
return string(bytes.TrimSpace(n))
}
// MergeGemsVersions passes through the lines of gems only one member lists
// and writes one regenerated line, from the merged /info, per gem that
// several members list.
func MergeGemsVersions(members []GemsMember, info func(name string) (*GemsDoc, error), at time.Time) ([]byte, error) {
bodies := make([][]byte, len(members))
owners := map[string]int{}
for i, m := range members {
rest, err := versionsLines(m)
if err != nil {
return nil, err
}
bodies[i] = rest
seen := map[string]bool{}
for line := range bytes.Lines(rest) {
if n := gemName(line); n != "" && !seen[n] {
seen[n] = true
owners[n]++
}
}
}
var b bytes.Buffer
b.WriteString(gems.VersionsHeader(at))
var shared []string
for _, rest := range bodies {
for line := range bytes.Lines(rest) {
n := gemName(line)
if n == "" {
continue
}
if owners[n] > 1 {
if !slices.Contains(shared, n) {
shared = append(shared, n)
}
continue
}
b.Write(bytes.TrimRight(line, "\n"))
b.WriteByte('\n')
}
}
for _, n := range shared {
d, err := info(n)
if errors.Is(err, ErrNotFound) {
continue
}
if err != nil {
return nil, err
}
b.WriteString(gems.VersionsLine(n, d.Body) + "\n")
}
return b.Bytes(), nil
}
// MergeGemsNames unions member /names documents, sorted.
func MergeGemsNames(members []GemsMember) ([]byte, error) {
set := map[string]bool{}
for _, m := range members {
rest, err := versionsLines(m)
if err != nil {
return nil, err
}
for line := range bytes.Lines(rest) {
if n := gemName(line); n != "" {
set[n] = true
}
}
}
names := make([]string, 0, len(set))
for n := range set {
names = append(names, n)
}
slices.Sort(names)
return []byte("---\n" + strings.Join(names, "\n") + "\n"), nil
}
// FetchGems serves a gems virtual's merged versions, names or info/<gem>.
func (e *Engine) FetchGems(ctx context.Context, virt models.Virtual, path string) ([]byte, string, error) {
var d *GemsDoc
var err error
switch {
case strings.HasPrefix(path, "info/"):
name := strings.TrimPrefix(path, "info/")
if !gems.NameRe.MatchString(name) {
return nil, "", ErrNotFound
}
d, err = e.gemsInfo(ctx, virt, name)
case path == "versions":
d, err = e.gemsMerged(ctx, virt, path, func(ctx context.Context, ms []GemsMember) ([]byte, error) {
return MergeGemsVersions(ms, func(name string) (*GemsDoc, error) { return e.gemsInfo(ctx, virt, name) }, time.Now())
})
case path == "names":
d, err = e.gemsMerged(ctx, virt, path, func(_ context.Context, ms []GemsMember) ([]byte, error) {
return MergeGemsNames(ms)
})
default:
return nil, "", ErrNotFound
}
if err != nil {
return nil, "", err
}
return d.Body, "text/plain; charset=utf-8", nil
}
func (e *Engine) gemsMerged(ctx context.Context, virt models.Virtual, path string, merge func(context.Context, []GemsMember) ([]byte, error)) (*GemsDoc, error) {
repo, err := e.merged(ctx, "gems\x00"+virt.Name+"\x00"+path, func(ctx context.Context) (mergedRepo, error) {
members, err := fetchMembers(virt, func(name string) (*GemsMember, error) { return e.gemsMember(ctx, name, path) })
if err != nil {
return nil, err
}
body, err := merge(ctx, members)
if err != nil {
return nil, err
}
return &GemsDoc{Body: body}, nil
})
if err != nil {
return nil, err
}
return repo.(*GemsDoc), nil
}
func (e *Engine) gemsInfo(ctx context.Context, virt models.Virtual, name string) (*GemsDoc, error) {
prefix := "gems\x00" + virt.Name + "\x00info/"
inserted := false
repo, err := e.merged(ctx, prefix+name, func(ctx context.Context) (mergedRepo, error) {
inserted = true
members, err := fetchMembers(virt, func(m string) (*GemsMember, error) { return e.gemsMember(ctx, m, "info/"+name) })
if err != nil {
return nil, err
}
return MergeGemsInfo(members)
})
if err != nil {
return nil, err
}
if inserted {
e.evictOldest(prefix, maxGemsInfos)
}
return repo.(*GemsDoc), nil
}
// GemsRedirect maps a gems virtual download (gems/<full>.gem or
// quick/Marshal.4.8/<full>.gemspec.rz) to the member whose /info line won the
// merge for that version-platform.
func (e *Engine) GemsRedirect(ctx context.Context, virt models.Virtual, path, rawQuery, proxyBaseURL string) (string, error) {
full, ok := strings.CutPrefix(path, "gems/")
if ok {
full, ok = strings.CutSuffix(full, ".gem")
} else if full, ok = strings.CutPrefix(path, "quick/Marshal.4.8/"); ok {
full, ok = strings.CutSuffix(full, ".gemspec.rz")
}
if !ok || strings.Contains(full, "/") {
return "", ErrNotFound
}
tried := 0
for i := 1; i < len(full)-1 && tried < maxGemsCandidates; i++ {
if full[i] != '-' || full[i+1] < '0' || full[i+1] > '9' || !gems.NameRe.MatchString(full[:i]) {
continue
}
tried++
d, err := e.gemsInfo(ctx, virt, full[:i])
if errors.Is(err, ErrNotFound) {
continue
}
if err != nil {
return "", err
}
if owner, ok := d.Owners[full[i+1:]]; ok {
loc, ok, err := e.memberLocation(ctx, owner, path, rawQuery, proxyBaseURL)
if err == nil && !ok {
err = ErrNotFound
}
return loc, err
}
}
return "", ErrNotFound
}
// fetchGemsMember returns a nil Body when the member lacks the document.
func (e *Engine) fetchGemsMember(ctx context.Context, name, path string) (*GemsMember, error) {
remote, err := e.getRemote(ctx, name)
if err != nil {
return nil, fmt.Errorf("remote %q: %w", name, err)
}
m := &GemsMember{RemoteName: name}
if remote.RepoType == models.RepoTypeLocal {
if e.GemsLocal == nil {
return nil, errors.New("local gems indexes are not wired")
}
m.Body, err = e.GemsLocal(ctx, name, path)
} else {
m.Body, err = e.fetchMemberPath(ctx, *remote, path)
}
var pe *proxy.ProxyError
if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
return &GemsMember{RemoteName: name}, nil
}
if err != nil {
return nil, err
}
return m, nil
}
internal/virtual/gems_merger_test.go
+180
View File
@@ -0,0 +1,180 @@
package virtual
import (
"context"
"errors"
"strings"
"sync"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider/gems"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestMergeGemsInfoMemberOrderWins(t *testing.T) {
local := GemsMember{RemoteName: "internal", Body: []byte("---\n1.0.0 |checksum:aa\n1.0.0-java |checksum:ab\n")}
remote := GemsMember{RemoteName: "rubygems", Body: []byte("---\n0.9.0 |checksum:99\n1.0.0 |checksum:bb\n2.0.0 rack:>= 0|checksum:cc,ruby:>= 3.0\n")}
d, err := MergeGemsInfo([]GemsMember{local, {RemoteName: "empty"}, remote})
if err != nil {
t.Fatal(err)
}
want := "---\n1.0.0 |checksum:aa\n1.0.0-java |checksum:ab\n0.9.0 |checksum:99\n2.0.0 rack:>= 0|checksum:cc,ruby:>= 3.0\n"
if string(d.Body) != want {
t.Fatalf("merged info =\n%q\nwant\n%q", d.Body, want)
}
for k, owner := range map[string]string{"1.0.0": "internal", "1.0.0-java": "internal", "0.9.0": "rubygems", "2.0.0": "rubygems"} {
if d.Owners[k] != owner {
t.Errorf("owner of %s = %q, want %q", k, d.Owners[k], owner)
}
}
odd := GemsMember{RemoteName: "rubygems", Body: []byte("---\n1.0.0 |checksum:bb\n\n")}
if d, _ := MergeGemsInfo([]GemsMember{{RemoteName: "internal"}, odd}); string(d.Body) != string(odd.Body) {
t.Fatalf("a gem only one member has must pass through verbatim, got %q", d.Body)
}
if _, err := MergeGemsInfo([]GemsMember{{RemoteName: "a"}}); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestMergeGemsVersions(t *testing.T) {
local := GemsMember{RemoteName: "internal", Body: []byte("created_at: 2026-10-18T00:00:00Z\n---\nhello 0.1.0 11\nrack 9.0.0 22\n")}
remote := GemsMember{RemoteName: "rubygems", Body: []byte("created_at: 2024-04-01T00:00:05Z\n---\n" +
"rack 1.0.0,2.0.0 33\nrails 7.0.0 44\nrack -1.0.0,3.0.0 55\n")}
merged := &GemsDoc{Body: []byte("---\n9.0.0 |checksum:x\n2.0.0 |checksum:y\n3.0.0 |checksum:z\n")}
var asked []string
at := time.Date(2026, 10, 18, 1, 2, 3, 0, time.UTC)
body, err := MergeGemsVersions([]GemsMember{local, remote}, func(name string) (*GemsDoc, error) {
asked = append(asked, name)
return merged, nil
}, at)
if err != nil {
t.Fatal(err)
}
want := "created_at: 2026-10-18T01:02:03Z\n---\nhello 0.1.0 11\nrails 7.0.0 44\nrack 9.0.0,2.0.0,3.0.0 " + gems.MD5Hex(merged.Body) + "\n"
if string(body) != want {
t.Fatalf("versions =\n%q\nwant\n%q", body, want)
}
if len(asked) != 1 || asked[0] != "rack" {
t.Fatalf("merged info fetched for %v, want only the shared gem", asked)
}
if _, err := MergeGemsVersions([]GemsMember{{RemoteName: "x", Body: []byte("no separator")}}, nil, at); err == nil {
t.Fatal("malformed member versions must fail")
}
if _, err := MergeGemsVersions([]GemsMember{{RemoteName: "x"}}, nil, at); err == nil {
t.Fatal("a member without versions must fail")
}
}
func TestMergeGemsNames(t *testing.T) {
body, err := MergeGemsNames([]GemsMember{{RemoteName: "a", Body: []byte("---\nzeta\nalpha\n")}, {RemoteName: "b", Body: []byte("---\nalpha\nmid\n")}})
if err != nil || string(body) != "---\nalpha\nmid\nzeta\n" {
t.Fatalf("names = %q %v", body, err)
}
}
func gemsEngine(fetch func(name, path string) (*GemsMember, error)) *Engine {
return &Engine{
mergeTTL: time.Minute,
getRemote: func(_ context.Context, name string) (*models.Remote, error) {
rt := models.RepoTypeRemote
if name == "internal" {
rt = models.RepoTypeLocal
}
return &models.Remote{Name: name, RepoType: rt}, nil
},
gemsMember: func(_ context.Context, name, path string) (*GemsMember, error) { return fetch(name, path) },
}
}
func gemsVirt(members ...string) models.Virtual {
return models.Virtual{Name: "gems", PackageType: models.PackageGems, Members: members}
}
func TestGemsRedirectToOwner(t *testing.T) {
var mu sync.Mutex
asked := map[string]int{}
e := gemsEngine(func(name, path string) (*GemsMember, error) {
mu.Lock()
asked[name+" "+path]++
mu.Unlock()
switch name + " " + path {
case "internal info/foo-2fa":
return &GemsMember{RemoteName: name, Body: []byte("---\n1.0.0 |checksum:a\n")}, nil
case "rubygems info/foo-2fa":
return &GemsMember{RemoteName: name, Body: []byte("---\n1.0.0 |checksum:b\n1.1.0-x86_64-linux |checksum:c\n")}, nil
case "rubygems info/foo":
return &GemsMember{RemoteName: name, Body: []byte("---\n2.0.0 |checksum:d\n")}, nil
}
return &GemsMember{RemoteName: name}, nil
})
virt := gemsVirt("internal", "rubygems")
for path, want := range map[string]string{
"gems/foo-2fa-1.0.0.gem": "http://host/api/v1/local/internal/gems/foo-2fa-1.0.0.gem",
"gems/foo-2fa-1.1.0-x86_64-linux.gem": "http://host/api/v1/remote/rubygems/gems/foo-2fa-1.1.0-x86_64-linux.gem",
"quick/Marshal.4.8/foo-2fa-1.0.0.gemspec.rz": "http://host/api/v1/local/internal/quick/Marshal.4.8/foo-2fa-1.0.0.gemspec.rz",
"quick/Marshal.4.8/foo-2.0.0.gemspec.rz": "http://host/api/v1/remote/rubygems/quick/Marshal.4.8/foo-2.0.0.gemspec.rz",
} {
loc, err := e.GemsRedirect(context.Background(), virt, path, "", "http://host")
if err != nil || loc != want {
t.Errorf("%s -> %q %v, want %q", path, loc, err, want)
}
}
for _, path := range []string{"gems/foo-2fa-9.9.9.gem", "gems/nodash.gem", "gems/x/../foo-2.0.0.gem", "other/foo-2.0.0.gem", "specs.4.8.gz"} {
if _, err := e.GemsRedirect(context.Background(), virt, path, "", "http://host"); !errors.Is(err, ErrNotFound) {
t.Errorf("%s: want ErrNotFound, got %v", path, err)
}
}
if asked["internal info/foo-2fa"] != 1 {
t.Fatalf("merged info must be cached across redirects, internal asked %d times", asked["internal info/foo-2fa"])
}
}
func TestFetchGemsFailsClosed(t *testing.T) {
e := gemsEngine(func(name, path string) (*GemsMember, error) {
if name == "rubygems" {
return nil, errors.New("upstream down")
}
return &GemsMember{RemoteName: name, Body: []byte("created_at: x\n---\nhello 0.1.0 11\n")}, nil
})
for _, path := range []string{"versions", "names", "info/hello"} {
if _, _, err := e.FetchGems(context.Background(), gemsVirt("internal", "rubygems"), path); err == nil || errors.Is(err, ErrNotFound) {
t.Errorf("%s: want member error, got %v", path, err)
}
}
if _, err := e.GemsRedirect(context.Background(), gemsVirt("internal", "rubygems"), "gems/hello-0.1.0.gem", "", "http://h"); err == nil || errors.Is(err, ErrNotFound) {
t.Errorf("redirect: want member error, got %v", err)
}
}
func TestFetchGemsVersionsUsesMergedInfo(t *testing.T) {
e := gemsEngine(func(name, path string) (*GemsMember, error) {
docs := map[string]string{
"internal versions": "created_at: x\n---\nrack 9.0.0 11\n",
"rubygems versions": "created_at: y\n---\nrack 3.0.0 22\nrails 7.0.0 33\n",
"internal info/rack": "---\n9.0.0 |checksum:a\n",
"rubygems info/rack": "---\n3.0.0 |checksum:b\n",
}
if d, ok := docs[name+" "+path]; ok {
return &GemsMember{RemoteName: name, Body: []byte(d)}, nil
}
return &GemsMember{RemoteName: name}, nil
})
virt := gemsVirt("internal", "rubygems")
info, ct, err := e.FetchGems(context.Background(), virt, "info/rack")
if err != nil || !strings.HasPrefix(ct, "text/plain") {
t.Fatalf("info: %v %q", err, ct)
}
versions, _, err := e.FetchGems(context.Background(), virt, "versions")
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(versions), "\nrails 7.0.0 33\nrack 9.0.0,3.0.0 "+gems.MD5Hex(info)+"\n") {
t.Fatalf("versions = %q", versions)
}
if _, _, err := e.FetchGems(context.Background(), virt, "info/../x"); !errors.Is(err, ErrNotFound) {
t.Fatalf("bad name: %v", err)
}
}
internal/virtual/helm_merger.go
+59 -41
View File
@@ -2,10 +2,14 @@ package virtual
import (
"fmt"
"log/slog"
"maps"
"slices"
"strings"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -16,22 +20,17 @@ func init() {
type HelmMerger struct{}
type helmIndex struct {
APIVersion string `yaml:"apiVersion"`
Entries map[string][]helmChartVersion `yaml:"entries"`
Generated string `yaml:"generated,omitempty"`
}
type helmChartVersion struct {
Name string `yaml:"name"`
Version string `yaml:"version"`
URLs []string `yaml:"urls"`
rest map[string]any
APIVersion string `yaml:"apiVersion"`
Entries map[string][]map[string]yaml.Node `yaml:"entries"`
Generated string `yaml:"generated,omitempty"`
}
// MergeIndexes keeps each chart version's fields as raw nodes so digest,
// apiVersion and the rest survive the merge untouched; only urls are rewritten.
func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
merged := &helmIndex{
APIVersion: "v1",
Entries: make(map[string][]helmChartVersion),
Entries: make(map[string][]map[string]yaml.Node),
}
seen := map[string]map[string]bool{}
@@ -42,47 +41,38 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
continue
}
for chart, versions := range idx.Entries {
budget := provider.MaxAliasExpandedNodes
for _, chart := range slices.Sorted(maps.Keys(idx.Entries)) {
versions := idx.Entries[chart]
if seen[chart] == nil {
seen[chart] = map[string]bool{}
}
for _, ver := range versions {
key := chart + ":" + ver.Version
if seen[chart][ver.Version] {
for _, raw := range versions {
ver, err := provider.ExpandYAMLAliases(raw, &budget)
if err != nil {
slog.Warn("helm merge: dropping chart entry", "member", member.RemoteName, "chart", chart, "error", err)
continue
}
seen[chart][ver.Version] = true
version := ver["version"].Value
if seen[chart][version] {
continue
}
seen[chart][version] = true
if proxyBaseURL != "" {
routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
if urlsNode, ok := ver["urls"]; ok && proxyBaseURL != "" {
var urls []string
if err := urlsNode.Decode(&urls); err != nil {
continue
}
baseHost := extractHost(member.BaseURL)
for i, u := range ver.URLs {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
if baseHost != "" && extractHost(u) != baseHost {
continue
}
relPath := extractPathRelativeToBase(u, member.BaseURL)
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
relPath)
} else {
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
u)
}
rewriteHelmURLs(urls, member, proxyBaseURL)
var n yaml.Node
if err := n.Encode(urls); err != nil {
return nil, err
}
ver["urls"] = n
}
merged.Entries[chart] = append(merged.Entries[chart], ver)
_ = key
}
}
}
@@ -90,6 +80,34 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
return yaml.Marshal(merged)
}
func rewriteHelmURLs(urls []string, member MemberIndex, proxyBaseURL string) {
routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
}
baseHost := extractHost(member.BaseURL)
for i, u := range urls {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
if baseHost != "" && extractHost(u) != baseHost {
continue
}
relPath := extractPathRelativeToBase(u, member.BaseURL)
urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
relPath)
} else {
urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
u)
}
}
}
func extractHost(rawURL string) string {
idx := strings.Index(rawURL, "://")
if idx == -1 {
internal/virtual/helm_merger_test.go
+129
View File
@@ -1,9 +1,12 @@
package virtual_test
import (
"fmt"
"strings"
"testing"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
@@ -122,3 +125,129 @@ entries:
t.Error("expected fallback repo to be excluded for duplicate")
}
}
func TestHelmMerger_PreservesChartFields(t *testing.T) {
m := &virtual.HelmMerger{}
body := []byte(`apiVersion: v1
entries:
app:
- apiVersion: v2
name: app
version: "1.0"
appVersion: "3.1"
description: in-house
digest: deadbeef
urls:
- app-1.0.tgz
`)
result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "charts", RepoType: "local", Body: body}}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
out := string(result)
for _, want := range []string{
"digest: deadbeef",
"apiVersion: v2",
`appVersion: "3.1"`,
"description: in-house",
`version: "1.0"`,
"https://proxy.example.com/api/v1/local/charts/app-1.0.tgz",
} {
if !strings.Contains(out, want) {
t.Errorf("merged index missing %q:\n%s", want, out)
}
}
}
func TestHelmMerger_ExpandsAliasesAcrossDedupe(t *testing.T) {
m := &virtual.HelmMerger{}
first := []byte(`apiVersion: v1
entries:
app:
- version: "1.0"
urls: [app-1.0.tgz]
`)
second := []byte(`apiVersion: v1
entries:
app:
- version: "1.0"
maintainers: &team
- name: ops
urls: [app-1.0.tgz]
- version: "2.0"
maintainers: *team
urls: [app-2.0.tgz]
`)
result, err := m.MergeIndexes([]virtual.MemberIndex{
{RemoteName: "a", RepoType: "local", Body: first},
{RemoteName: "b", RepoType: "local", Body: second},
}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
var idx struct {
Entries map[string][]struct {
Version string `yaml:"version"`
Maintainers []struct {
Name string `yaml:"name"`
} `yaml:"maintainers"`
} `yaml:"entries"`
}
if err := yaml.Unmarshal(result, &idx); err != nil {
t.Fatalf("merged index does not parse: %v\n%s", err, result)
}
app := idx.Entries["app"]
if len(app) != 2 || app[1].Version != "2.0" || len(app[1].Maintainers) != 1 || app[1].Maintainers[0].Name != "ops" {
t.Errorf("alias not expanded: %+v\n%s", app, result)
}
}
func TestHelmMerger_DropsAliasBomb(t *testing.T) {
m := &virtual.HelmMerger{}
body := []byte(`apiVersion: v1
entries:
app:
- version: "1.0"
a: &a [x, x, x, x, x, x, x, x, x, x]
b: &b [*a, *a, *a, *a, *a, *a, *a, *a, *a, *a]
c: &c [*b, *b, *b, *b, *b, *b, *b, *b, *b, *b]
d: &d [*c, *c, *c, *c, *c, *c, *c, *c, *c, *c]
e: &e [*d, *d, *d, *d, *d, *d, *d, *d, *d, *d]
urls: [app-1.0.tgz]
`)
result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "a", RepoType: "local", Body: body}}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
if strings.Contains(string(result), "app-1.0.tgz") {
t.Errorf("alias bomb entry was kept")
}
}
func TestHelmMerger_LargeIndexWithoutAliasesIsComplete(t *testing.T) {
var b strings.Builder
b.WriteString("apiVersion: v1\nentries:\n")
for c := 0; c < 100; c++ {
fmt.Fprintf(&b, " chart%d:\n", c)
for v := 0; v < 100; v++ {
fmt.Fprintf(&b, " - name: chart%d\n version: \"1.0.%d\"\n description: d\n maintainers: [{name: ops, email: ops@example.com}]\n urls: [chart%d-1.0.%d.tgz]\n", c, v, c, v)
}
}
result, err := (&virtual.HelmMerger{}).MergeIndexes([]virtual.MemberIndex{{RemoteName: "a", RepoType: "local", Body: []byte(b.String())}}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
var idx struct {
Entries map[string][]map[string]any `yaml:"entries"`
}
if err := yaml.Unmarshal(result, &idx); err != nil {
t.Fatal(err)
}
total := 0
for _, vs := range idx.Entries {
total += len(vs)
}
if len(idx.Entries) != 100 || total != 10000 {
t.Errorf("got %d charts, %d versions; want 100, 10000", len(idx.Entries), total)
}
}
internal/virtual/npm_merger.go
+271
View File
@@ -0,0 +1,271 @@
package virtual
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"regexp"
"slices"
"strings"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const (
NPMAbbreviatedType = "application/vnd.npm.install-v1+json"
npmAbbreviatedFile = "install-v1"
maxNPMPackuments = 1024
)
var npmVirtualNameRe = regexp.MustCompile(`^(@[A-Za-z0-9][A-Za-z0-9._~-]*/)?[A-Za-z0-9][A-Za-z0-9._~-]*$`)
// npmAbbreviatedFields are the version fields kept in an install-v1 packument.
var npmAbbreviatedFields = []string{
"name", "version", "deprecated", "dependencies", "optionalDependencies",
"devDependencies", "bundleDependencies", "peerDependencies", "peerDependenciesMeta",
"bin", "directories", "dist", "engines", "_hasShrinkwrap", "hasInstallScript",
"cpu", "os", "libc",
}
// NPMMember is one member's full packument for a package; Doc is nil when the
// member does not have the package. Tarball URLs under one of Bases are
// rewritten to go via the virtual; with no Bases (a local) they are relative.
type NPMMember struct {
RemoteName string
Bases []string
Doc []byte
}
// NPMPackument is a merged packument in full and install-v1 form.
type NPMPackument struct {
Full, Abbreviated []byte
}
func (p *NPMPackument) index() []byte { return p.Full }
func (p *NPMPackument) files() map[string][]byte {
return map[string][]byte{npmAbbreviatedFile: p.Abbreviated}
}
// MergeNPM unions versions across members (first member wins a duplicate),
// keeps the first member's value for each dist-tag, drops tags naming versions
// outside the merged set and points every owned tarball at
// tarballBase + <member>/<path>. ErrNotFound means no member has the package.
func MergeNPM(members []NPMMember, tarballBase string) (*NPMPackument, error) {
var base map[string]json.RawMessage
versions := map[string]map[string]json.RawMessage{}
tags := map[string]string{}
times := map[string]json.RawMessage{}
var modified string
for _, m := range members {
if m.Doc == nil {
continue
}
var doc map[string]json.RawMessage
var vs map[string]map[string]json.RawMessage
var dt map[string]string
var tm map[string]json.RawMessage
if err := json.Unmarshal(m.Doc, &doc); err != nil {
return nil, fmt.Errorf("member %q packument: %w", m.RemoteName, err)
}
for field, dst := range map[string]any{"versions": &vs, "dist-tags": &dt, "time": &tm} {
if raw, ok := doc[field]; ok {
if err := json.Unmarshal(raw, dst); err != nil {
return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, field, err)
}
}
}
if base == nil {
base = doc
}
for v, man := range vs {
if _, dup := versions[v]; dup || man == nil {
continue
}
if err := rewriteNPMTarball(man, m, tarballBase); err != nil {
return nil, fmt.Errorf("member %q %s: %w", m.RemoteName, v, err)
}
versions[v] = man
}
for tag, v := range dt {
if _, ok := tags[tag]; !ok {
tags[tag] = v
}
}
for k, v := range tm {
if k == "modified" {
var s string
if json.Unmarshal(v, &s) == nil && s > modified {
modified = s
}
continue
}
if _, ok := times[k]; !ok {
times[k] = v
}
}
}
if base == nil {
return nil, ErrNotFound
}
for tag, v := range tags {
if _, ok := versions[v]; !ok {
delete(tags, tag)
}
}
if modified != "" {
times["modified"], _ = json.Marshal(modified)
}
full := map[string]any{}
for k, v := range base {
full[k] = v
}
full["versions"], full["dist-tags"] = versions, tags
delete(full, "time")
if len(times) > 0 {
full["time"] = times
}
abbrevVersions := make(map[string]map[string]json.RawMessage, len(versions))
for v, man := range versions {
a := map[string]json.RawMessage{}
for _, f := range npmAbbreviatedFields {
if raw, ok := man[f]; ok {
a[f] = raw
}
}
abbrevVersions[v] = a
}
abbrev := map[string]any{"name": base["name"], "dist-tags": tags, "versions": abbrevVersions}
if modified != "" {
abbrev["modified"] = modified
}
fullJSON, err := json.Marshal(full)
if err != nil {
return nil, err
}
abbrevJSON, err := json.Marshal(abbrev)
if err != nil {
return nil, err
}
return &NPMPackument{Full: fullJSON, Abbreviated: abbrevJSON}, nil
}
// rewriteNPMTarball leaves a tarball outside the member's bases untouched.
func rewriteNPMTarball(man map[string]json.RawMessage, m NPMMember, tarballBase string) error {
raw, ok := man["dist"]
if !ok {
return nil
}
var dist map[string]json.RawMessage
if err := json.Unmarshal(raw, &dist); err != nil {
return fmt.Errorf("dist: %w", err)
}
var tarball string
if err := json.Unmarshal(dist["tarball"], &tarball); err != nil {
return nil
}
rel, ok := npmMemberPath(tarball, m.Bases)
if !ok {
return nil
}
dist["tarball"], _ = json.Marshal(tarballBase + url.PathEscape(m.RemoteName) + "/" + rel)
man["dist"], _ = json.Marshal(dist)
return nil
}
func npmMemberPath(tarball string, bases []string) (string, bool) {
if len(bases) == 0 {
return tarball, tarball != "" && !strings.Contains(tarball, "://") && !strings.HasPrefix(tarball, "/")
}
for _, b := range bases {
if rel, ok := strings.CutPrefix(tarball, strings.TrimRight(b, "/")+"/"); ok && rel != "" {
return rel, true
}
}
return "", false
}
// FetchNPM serves the merged packument for the package named by path, full or
// abbreviated (install-v1).
func (e *Engine) FetchNPM(ctx context.Context, virt models.Virtual, path, proxyBaseURL string, abbreviated bool) ([]byte, string, error) {
pkg, err := url.PathUnescape(strings.TrimSuffix(path, "/"))
if err != nil || !npmVirtualNameRe.MatchString(pkg) {
return nil, "", ErrNotFound
}
prefix := "npm\x00" + virt.Name + "\x00"
tarballBase := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/virtual/" + url.PathEscape(virt.Name) + "/"
inserted := false
repo, err := e.merged(ctx, prefix+pkg+"\x00"+proxyBaseURL, func(ctx context.Context) (mergedRepo, error) {
inserted = true
members, err := fetchMembers(virt, func(name string) (*NPMMember, error) { return e.npmMember(ctx, name, pkg) })
if err != nil {
return nil, err
}
p, err := MergeNPM(members, tarballBase)
if err != nil {
return nil, err
}
return p, nil
})
if err != nil {
return nil, "", err
}
if inserted {
e.evictOldest(prefix, maxNPMPackuments)
}
if abbreviated {
return repo.files()[npmAbbreviatedFile], NPMAbbreviatedType, nil
}
return repo.index(), "application/json", nil
}
// fetchNPMMember returns a member with a nil Doc when it lacks the package.
func (e *Engine) fetchNPMMember(ctx context.Context, name, pkg string) (*NPMMember, error) {
remote, err := e.getRemote(ctx, name)
if err != nil {
return nil, fmt.Errorf("remote %q: %w", name, err)
}
m := &NPMMember{RemoteName: name}
if remote.RepoType == models.RepoTypeLocal {
if e.NPMLocal == nil {
return nil, errors.New("local npm packuments are not wired")
}
m.Doc, err = e.NPMLocal(ctx, name, pkg)
} else {
m.Bases = remote.UpstreamPool()
m.Doc, err = e.fetchMemberPath(ctx, *remote, strings.Replace(pkg, "/", "%2f", 1))
}
var pe *proxy.ProxyError
if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound {
return &NPMMember{RemoteName: name}, nil
}
if err != nil {
return nil, err
}
return m, nil
}
// evictOldest drops the oldest merges under prefix beyond keep.
func (e *Engine) evictOldest(prefix string, keep int) {
e.mu.Lock()
defer e.mu.Unlock()
var keys []string
for k := range e.gens {
if strings.HasPrefix(k, prefix) {
keys = append(keys, k)
}
}
if len(keys) <= keep {
return
}
slices.SortFunc(keys, func(a, b string) int { return e.gens[b].at.Compare(e.gens[a].at) })
for _, k := range keys[keep:] {
delete(e.gens, k)
}
}
internal/virtual/npm_merger_test.go
+289
View File
@@ -0,0 +1,289 @@
package virtual
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type npmDoc struct {
Name string `json:"name"`
Readme string `json:"readme,omitempty"`
Modified string `json:"modified,omitempty"`
DistTags map[string]string `json:"dist-tags"`
Versions map[string]map[string]any `json:"versions"`
Time map[string]string `json:"time,omitempty"`
}
func npmVersion(name, v, tarball string) map[string]any {
return map[string]any{"name": name, "version": v, "readme": "long", "scripts": map[string]string{"test": "x"},
"dependencies": map[string]string{"dep": "^1"}, "dist": map[string]any{"tarball": tarball, "shasum": "abc"}}
}
func mustJSON(t *testing.T, v any) []byte {
t.Helper()
b, err := json.Marshal(v)
if err != nil {
t.Fatal(err)
}
return b
}
func decodeNPM(t *testing.T, b []byte) npmDoc {
t.Helper()
var d npmDoc
if err := json.Unmarshal(b, &d); err != nil {
t.Fatalf("decode %s: %v", b, err)
}
return d
}
func tarballOf(d npmDoc, v string) string {
dist, _ := d.Versions[v]["dist"].(map[string]any)
s, _ := dist["tarball"].(string)
return s
}
func npmMembersFixture(t *testing.T) []NPMMember {
local := npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "beta": "9.9.9"},
Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}}
remote := npmDoc{Name: "pkg", Readme: "remote readme",
DistTags: map[string]string{"latest": "2.0.0", "next": "2.0.0", "beta": "1.0.0"},
Versions: map[string]map[string]any{
"1.0.0": npmVersion("pkg", "1.0.0", "https://registry.npmjs.org/pkg/-/pkg-1.0.0.tgz"),
"2.0.0": npmVersion("pkg", "2.0.0", "https://registry.npmjs.org/pkg/-/pkg-2.0.0.tgz"),
"3.0.0": npmVersion("pkg", "3.0.0", "https://cdn.example/pkg-3.0.0.tgz"),
},
Time: map[string]string{"modified": "2024-01-01T00:00:00.000Z", "2.0.0": "2023-01-01T00:00:00.000Z"}}
return []NPMMember{
{RemoteName: "internal", Doc: mustJSON(t, local)},
{RemoteName: "absent"},
{RemoteName: "npmjs", Bases: []string{"https://registry.npmjs.org/"}, Doc: mustJSON(t, remote)},
}
}
func TestMergeNPMUnionsVersionsMemberOrderWins(t *testing.T) {
p, err := MergeNPM(npmMembersFixture(t), "http://aa/api/v1/virtual/npm/")
if err != nil {
t.Fatal(err)
}
d := decodeNPM(t, p.Full)
if len(d.Versions) != 3 {
t.Fatalf("want 3 merged versions, got %v", d.Versions)
}
for v, want := range map[string]string{
"1.0.0": "http://aa/api/v1/virtual/npm/internal/pkg/-/pkg-1.0.0.tgz",
"2.0.0": "http://aa/api/v1/virtual/npm/npmjs/pkg/-/pkg-2.0.0.tgz",
"3.0.0": "https://cdn.example/pkg-3.0.0.tgz",
} {
if got := tarballOf(d, v); got != want {
t.Errorf("%s tarball = %q, want %q", v, got, want)
}
}
if d.Readme != "" {
t.Errorf("top-level fields come from the first member with the package, got readme %q", d.Readme)
}
if d.Time["modified"] != "2024-01-01T00:00:00.000Z" || d.Time["2.0.0"] == "" {
t.Errorf("time not merged: %v", d.Time)
}
}
func TestMergeNPMDistTags(t *testing.T) {
members := npmMembersFixture(t)
members[0].Doc = mustJSON(t, npmDoc{Name: "pkg", DistTags: map[string]string{"latest": "1.0.0", "gone": "0.0.1"},
Versions: map[string]map[string]any{"1.0.0": npmVersion("pkg", "1.0.0", "pkg/-/pkg-1.0.0.tgz")}})
p, err := MergeNPM(members, "http://aa/v/")
if err != nil {
t.Fatal(err)
}
got := decodeNPM(t, p.Full).DistTags
want := map[string]string{"latest": "1.0.0", "next": "2.0.0", "beta": "1.0.0"}
if fmt.Sprint(got) != fmt.Sprint(want) {
t.Fatalf("dist-tags = %v, want %v", got, want)
}
}
func TestMergeNPMDropsFirstTagNamingMissingVersion(t *testing.T) {
// internal claims beta=9.9.9 first; it wins over npmjs's beta, and is then
// dropped because no member has 9.9.9.
p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
if err != nil {
t.Fatal(err)
}
if tag, ok := decodeNPM(t, p.Full).DistTags["beta"]; ok {
t.Fatalf("beta should be dropped, got %q", tag)
}
}
func TestMergeNPMAbbreviated(t *testing.T) {
p, err := MergeNPM(npmMembersFixture(t), "http://aa/v/")
if err != nil {
t.Fatal(err)
}
var a map[string]any
if err := json.Unmarshal(p.Abbreviated, &a); err != nil {
t.Fatal(err)
}
if a["modified"] != "2024-01-01T00:00:00.000Z" || a["name"] != "pkg" || a["readme"] != nil || a["time"] != nil {
t.Fatalf("bad abbreviated top level: %v", a)
}
v := a["versions"].(map[string]any)["2.0.0"].(map[string]any)
if v["readme"] != nil || v["scripts"] != nil {
t.Errorf("abbreviated version keeps non-install fields: %v", v)
}
if v["dependencies"] == nil || v["dist"].(map[string]any)["tarball"] != "http://aa/v/npmjs/pkg/-/pkg-2.0.0.tgz" {
t.Errorf("abbreviated version lost install fields: %v", v)
}
}
func TestMergeNPMNoMemberHasPackage(t *testing.T) {
if _, err := MergeNPM([]NPMMember{{RemoteName: "a"}, {RemoteName: "b"}}, "http://aa/v/"); !errors.Is(err, ErrNotFound) {
t.Fatalf("want ErrNotFound, got %v", err)
}
}
func TestNPMMemberPath(t *testing.T) {
for _, c := range []struct {
tarball string
bases []string
want string
ok bool
}{
{"@s/n/-/n-1.0.0.tgz", nil, "@s/n/-/n-1.0.0.tgz", true},
{"https://evil.example/x.tgz", nil, "", false},
{"/abs/x.tgz", nil, "", false},
{"https://r.example/@s/n/-/n-1.0.0.tgz", []string{"https://m.example", "https://r.example"}, "@s/n/-/n-1.0.0.tgz", true},
{"https://r.example.evil/x.tgz", []string{"https://r.example"}, "", false},
} {
got, ok := npmMemberPath(c.tarball, c.bases)
if ok != c.ok || ok && got != c.want {
t.Errorf("npmMemberPath(%q, %v) = %q,%v want %q,%v", c.tarball, c.bases, got, ok, c.want, c.ok)
}
}
}
func npmEngine(fetch func(name, pkg string) (*NPMMember, error)) *Engine {
return &Engine{
mergeTTL: time.Minute,
getRemote: func(_ context.Context, name string) (*models.Remote, error) {
return &models.Remote{Name: name, RepoType: models.RepoTypeRemote}, nil
},
npmMember: func(_ context.Context, name, pkg string) (*NPMMember, error) { return fetch(name, pkg) },
}
}
func npmVirt(members ...string) models.Virtual {
return models.Virtual{Name: "npm", PackageType: models.PackageNPM, Members: members}
}
func TestFetchNPMScopedAndAbbreviated(t *testing.T) {
var mu sync.Mutex
var asked []string
e := npmEngine(func(name, pkg string) (*NPMMember, error) {
mu.Lock()
asked = append(asked, pkg)
mu.Unlock()
if name == "b" {
return &NPMMember{RemoteName: name}, nil
}
return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, DistTags: map[string]string{"latest": "1.0.0"},
Versions: map[string]map[string]any{"1.0.0": npmVersion(pkg, "1.0.0", pkg+"/-/n-1.0.0.tgz")}})}, nil
})
body, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", true)
if err != nil {
t.Fatal(err)
}
if ct != NPMAbbreviatedType || !strings.Contains(string(body), `"tarball":"http://host/api/v1/virtual/npm/a/@s/n/-/n-1.0.0.tgz"`) {
t.Fatalf("ct %q body %s", ct, body)
}
if asked[0] != "@s/n" {
t.Fatalf("member asked for %q, want decoded scoped name", asked[0])
}
full, ct, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "@s%2fn", "http://host", false)
if err != nil || ct != "application/json" || string(full) == string(body) {
t.Fatalf("full packument: ct %q err %v", ct, err)
}
if len(asked) != 2 {
t.Fatalf("second fetch should be served from the merge cache, members asked %d times", len(asked))
}
}
func TestFetchNPMFailsClosed(t *testing.T) {
e := npmEngine(func(name, pkg string) (*NPMMember, error) {
if name == "b" {
return nil, errors.New("upstream down")
}
return &NPMMember{RemoteName: name, Doc: []byte(`{"name":"x","versions":{}}`)}, nil
})
_, _, err := e.FetchNPM(context.Background(), npmVirt("a", "b"), "x", "http://host", false)
if err == nil || errors.Is(err, ErrNotFound) {
t.Fatalf("want upstream error with a member down, got %v", err)
}
}
func TestFetchNPMNotFound(t *testing.T) {
e := npmEngine(func(name, _ string) (*NPMMember, error) { return &NPMMember{RemoteName: name}, nil })
for _, path := range []string{"missing", "-/npm/v1/security/advisories/bulk", "a/../b", "@s"} {
if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), path, "http://host", false); !errors.Is(err, ErrNotFound) {
t.Errorf("%q: want ErrNotFound, got %v", path, err)
}
}
}
func TestFetchNPMMemberLocalNotFound(t *testing.T) {
e := &Engine{getRemote: func(_ context.Context, name string) (*models.Remote, error) {
return &models.Remote{Name: name, RepoType: models.RepoTypeLocal}, nil
}}
e.NPMLocal = func(_ context.Context, _, pkg string) ([]byte, error) {
if pkg == "boom" {
return nil, errors.New("db down")
}
return nil, fmt.Errorf("wrapped: %w", ErrNotFound)
}
m, err := e.fetchNPMMember(context.Background(), "internal", "nope")
if err != nil || m.Doc != nil {
t.Fatalf("missing package must be an empty member, got %v %v", m, err)
}
if _, err := e.fetchNPMMember(context.Background(), "internal", "boom"); err == nil {
t.Fatal("local read errors must surface")
}
}
func TestMemberRedirectNPM(t *testing.T) {
e := npmEngine(nil)
virt := npmVirt("internal", "npmjs")
loc, ok, err := e.MemberRedirect(context.Background(), virt, "npmjs/@s/n/-/n-1.0.0.tgz", "", "http://host")
if err != nil || !ok || loc != "http://host/api/v1/remote/npmjs/@s/n/-/n-1.0.0.tgz" {
t.Fatalf("got %q %v %v", loc, ok, err)
}
if _, ok, _ := e.MemberRedirect(context.Background(), virt, "@s/n", "", "http://host"); ok {
t.Fatal("scoped packument path must not redirect")
}
if _, _, err := e.MemberRedirect(context.Background(), virt, "npmjs/%2e%2e/x.tgz", "", "http://host"); !errors.Is(err, ErrBadPath) {
t.Fatalf("want ErrBadPath, got %v", err)
}
}
func TestEvictOldestBoundsNPMMerges(t *testing.T) {
e := npmEngine(func(name, pkg string) (*NPMMember, error) {
return &NPMMember{RemoteName: name, Doc: mustJSON(t, npmDoc{Name: pkg, Versions: map[string]map[string]any{}})}, nil
})
for i := range maxNPMPackuments + 5 {
if _, _, err := e.FetchNPM(context.Background(), npmVirt("a"), fmt.Sprintf("p%d", i), "http://host", false); err != nil {
t.Fatal(err)
}
}
if len(e.gens) != maxNPMPackuments {
t.Fatalf("cached merges = %d, want %d", len(e.gens), maxNPMPackuments)
}
if _, ok := e.gens["npm\x00npm\x00p0\x00http://host"]; ok {
t.Fatal("oldest merge should be evicted")
}
}