Merge origin/master into benvin/virtual-deb
This commit is contained in:
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
tfprov "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tfsign"
|
||||
@@ -290,12 +291,5 @@ func writeJSON(w http.ResponseWriter, v any) {
|
||||
}
|
||||
|
||||
func baseURL(r *http.Request) string {
|
||||
scheme := "http"
|
||||
if r.TLS != nil {
|
||||
scheme = "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
scheme = fwd
|
||||
}
|
||||
return scheme + "://" + r.Host
|
||||
return v1.Scheme(r) + "://" + r.Host
|
||||
}
|
||||
|
||||
@@ -0,0 +1,289 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var ansibleMaxPublishBytes int64 = 256 << 20
|
||||
|
||||
// ansibleError writes a Galaxy v3 error document, which ansible-galaxy prints.
|
||||
func ansibleError(w http.ResponseWriter, status int, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"errors": []map[string]string{{
|
||||
"status": strconv.Itoa(status), "code": strings.ToLower(strings.ReplaceAll(http.StatusText(status), " ", "_")),
|
||||
"title": http.StatusText(status), "detail": msg,
|
||||
}}})
|
||||
}
|
||||
|
||||
func writeAnsibleJSON(w http.ResponseWriter, status int, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
|
||||
w.WriteHeader(status)
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetEscapeHTML(false)
|
||||
_ = enc.Encode(v)
|
||||
}
|
||||
|
||||
func isAnsiblePublish(path string) bool {
|
||||
return strings.TrimSuffix(path, "/")+"/" == ansible.PublishPath
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleAnsiblePublish(w http.ResponseWriter, r *http.Request) {
|
||||
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
|
||||
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageAnsible {
|
||||
ansibleError(w, http.StatusMethodNotAllowed, "publish is only supported on local ansible repositories")
|
||||
return
|
||||
}
|
||||
base, ok := requestBaseURL(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, ansibleMaxPublishBytes)
|
||||
file, sum, err := readAnsibleUpload(r)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
ansibleError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("upload exceeds %d bytes", ansibleMaxPublishBytes))
|
||||
return
|
||||
}
|
||||
ansibleError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
digest := sha256.Sum256(file)
|
||||
if got := hex.EncodeToString(digest[:]); sum != "" && !strings.EqualFold(sum, got) {
|
||||
ansibleError(w, http.StatusBadRequest, "sha256 does not match the uploaded file")
|
||||
return
|
||||
}
|
||||
e, err := ansible.ParseCollection(bytes.NewReader(file))
|
||||
if err != nil {
|
||||
ansibleError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
e.SHA256, e.Size, e.CreatedAt = hex.EncodeToString(digest[:]), int64(len(file)), time.Now().UTC()
|
||||
|
||||
id := e.Namespace + "." + e.Name + ":" + e.Version
|
||||
unlock, err := h.engine.Lock(r.Context(), remote.Name, "_ansiblelock/"+e.TaskID(), gemsLockWait, 5*time.Minute)
|
||||
if err != nil {
|
||||
ansibleError(w, http.StatusServiceUnavailable, id+" is being published, retry")
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
conflict := func() {
|
||||
ansibleError(w, http.StatusConflict, id+" already exists; overwrites are not allowed")
|
||||
}
|
||||
if f, err := h.db.GetLocalFile(r.Context(), remote.Name, e.IndexPath()); err != nil {
|
||||
ansibleError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
} else if f != nil {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
// The version is not live, so a tarball left by a failed publish is safe to drop.
|
||||
artifact := ansible.ArtifactPrefix + e.Filename()
|
||||
fail := func(status int, msg string) {
|
||||
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, artifact); err != nil {
|
||||
slog.Error("ansible publish cleanup failed", "repo", remote.Name, "path", artifact, "error", err)
|
||||
}
|
||||
ansibleError(w, status, msg)
|
||||
}
|
||||
if ok, err := h.npmStoreImmutable(r.Context(), remote.Name, artifact, file, "application/gzip"); err != nil {
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
} else if !ok {
|
||||
fail(http.StatusConflict, id+" already exists; overwrites are not allowed")
|
||||
return
|
||||
}
|
||||
entry, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
// The index entry gates visibility, so it is written last.
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, e.IndexPath(), entry, "application/json", false); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
slog.Info("local ansible collection published", "repo", remote.Name, "collection", id)
|
||||
repoBase := base + "/api/v1/local/" + url.PathEscape(remote.Name)
|
||||
writeAnsibleJSON(w, http.StatusAccepted, map[string]string{"task": repoBase + "/api/v3/imports/collections/" + e.TaskID() + "/"})
|
||||
}
|
||||
|
||||
// readAnsibleUpload returns the multipart "file" part, which ansible-galaxy
|
||||
// base64-encodes, and the optional "sha256" field sent with it.
|
||||
func readAnsibleUpload(r *http.Request) ([]byte, string, error) {
|
||||
mr, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
return nil, "", errors.New("expected a multipart/form-data upload")
|
||||
}
|
||||
var file []byte
|
||||
var sum string
|
||||
for {
|
||||
part, err := mr.NextPart()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
switch part.FormName() {
|
||||
case "file":
|
||||
if file != nil {
|
||||
return nil, "", errors.New("multiple file parts")
|
||||
}
|
||||
var src io.Reader = part
|
||||
switch enc := strings.ToLower(part.Header.Get("Content-Transfer-Encoding")); enc {
|
||||
case "base64":
|
||||
src = base64.NewDecoder(base64.StdEncoding, part)
|
||||
case "", "binary", "7bit", "8bit":
|
||||
default:
|
||||
return nil, "", fmt.Errorf("unsupported Content-Transfer-Encoding %q", enc)
|
||||
}
|
||||
if file, err = io.ReadAll(src); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
case "sha256":
|
||||
b, err := io.ReadAll(io.LimitReader(part, 128))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sum = strings.TrimSpace(string(b))
|
||||
}
|
||||
}
|
||||
if file == nil {
|
||||
return nil, "", errors.New("missing file part")
|
||||
}
|
||||
return file, sum, nil
|
||||
}
|
||||
|
||||
// serveLocalAnsible answers the Galaxy v3 API of a local ansible repo.
|
||||
func (h *ProxyHandler) serveLocalAnsible(w http.ResponseWriter, r *http.Request, remote *models.Remote, ap ansible.APIPath) {
|
||||
base, ok := requestBaseURL(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
body, err := h.localAnsibleDoc(r.Context(), remote.Name, ap, r.URL.Query(), base+"/api/v1/local/"+url.PathEscape(remote.Name))
|
||||
if errors.Is(err, virtual.ErrNotFound) {
|
||||
ansibleError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
ansibleError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
writeAnsibleJSON(w, http.StatusOK, body)
|
||||
}
|
||||
|
||||
// LocalAnsible serves a local ansible repo's collection, full versions list
|
||||
// or version detail to ansible virtuals, with root-relative URLs.
|
||||
func (h *ProxyHandler) LocalAnsible(ctx context.Context, repo, path string) ([]byte, error) {
|
||||
doc, err := h.localAnsibleDoc(ctx, repo, ansible.ParseAPIPath(path), nil, "/api/v1/local/"+url.PathEscape(repo))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return json.Marshal(doc)
|
||||
}
|
||||
|
||||
// localAnsibleDoc builds the document for ap; a nil q lists every version.
|
||||
func (h *ProxyHandler) localAnsibleDoc(ctx context.Context, repo string, ap ansible.APIPath, q url.Values, base string) (any, error) {
|
||||
switch ap.Kind {
|
||||
case ansible.APIRoot:
|
||||
return ansible.Root(), nil
|
||||
case ansible.APIImport:
|
||||
ns, rest, _ := strings.Cut(ap.TaskID, "-")
|
||||
name, version, _ := strings.Cut(rest, "-")
|
||||
entries, err := h.localAnsibleEntries(ctx, repo, ns, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Version == version {
|
||||
return ansible.ImportTask(e), nil
|
||||
}
|
||||
}
|
||||
case ansible.APICollection, ansible.APIVersions, ansible.APIVersion:
|
||||
entries, err := h.localAnsibleEntries(ctx, repo, ap.Namespace, ap.Name)
|
||||
if err != nil || len(entries) == 0 {
|
||||
return nil, cmp.Or(err, virtual.ErrNotFound)
|
||||
}
|
||||
refs := make([]ansible.VersionRef, len(entries))
|
||||
for i, e := range entries {
|
||||
refs[i] = e.Ref()
|
||||
}
|
||||
switch ap.Kind {
|
||||
case ansible.APICollection:
|
||||
return ansible.CollectionDoc(base, ap.Namespace, ap.Name, refs), nil
|
||||
case ansible.APIVersions:
|
||||
limit, offset := len(refs), 0
|
||||
if q != nil {
|
||||
limit, offset = ansible.Page(q)
|
||||
}
|
||||
return ansible.VersionsPage(base, ap.Namespace, ap.Name, refs, limit, offset), nil
|
||||
}
|
||||
for _, e := range entries {
|
||||
if e.Version == ap.Version {
|
||||
return ansible.VersionDoc(base, e), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, virtual.ErrNotFound
|
||||
}
|
||||
|
||||
// localAnsibleEntries returns a collection's versions, newest publish first.
|
||||
func (h *ProxyHandler) localAnsibleEntries(ctx context.Context, repo, ns, name string) ([]ansible.Entry, error) {
|
||||
if !ansible.NameRe.MatchString(ns) || !ansible.NameRe.MatchString(name) {
|
||||
return nil, nil
|
||||
}
|
||||
prefix := ansible.IndexPrefix + ns + "/" + name + "/"
|
||||
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(b.ID, a.ID) })
|
||||
var out []ansible.Entry
|
||||
for _, f := range files {
|
||||
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
|
||||
if !strings.HasPrefix(f.FilePath, prefix) {
|
||||
continue
|
||||
}
|
||||
raw, err := h.readLocalBlob(ctx, f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var e ansible.Entry
|
||||
if err := json.Unmarshal(raw, &e); err != nil {
|
||||
return nil, fmt.Errorf("decode %s: %w", f.FilePath, err)
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,275 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func testCollection(t *testing.T, ns, name, version string) []byte {
|
||||
t.Helper()
|
||||
manifest := `{"collection_info":{"namespace":"` + ns + `","name":"` + name + `","version":"` + version + `","dependencies":{"ansible.posix":">=1.0.0"}}}`
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "MANIFEST.json", Mode: 0o644, Size: int64(len(manifest))})
|
||||
_, _ = tw.Write([]byte(manifest))
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// ansiblePublish uploads like ansible-galaxy: the file part base64-encoded
|
||||
// whenever a sha256 field is sent, raw otherwise.
|
||||
func ansiblePublish(t *testing.T, url string, tarball []byte, sum string) (int, string) {
|
||||
t.Helper()
|
||||
var body bytes.Buffer
|
||||
mw := multipart.NewWriter(&body)
|
||||
hdr := textproto.MIMEHeader{"Content-Disposition": {`form-data; name="file"; filename="c.tar.gz"`}}
|
||||
data := tarball
|
||||
if sum != "" {
|
||||
_ = mw.WriteField("sha256", sum)
|
||||
hdr.Set("Content-Transfer-Encoding", "base64")
|
||||
data = []byte(base64.StdEncoding.EncodeToString(tarball))
|
||||
}
|
||||
fw, _ := mw.CreatePart(hdr)
|
||||
_, _ = fw.Write(data)
|
||||
_ = mw.Close()
|
||||
req, _ := http.NewRequest(http.MethodPost, url, &body)
|
||||
req.Header.Set("Content-Type", mw.FormDataContentType())
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
var b bytes.Buffer
|
||||
_, _ = b.ReadFrom(resp.Body)
|
||||
return resp.StatusCode, b.String()
|
||||
}
|
||||
|
||||
func getJSON(t *testing.T, url string, v any) {
|
||||
t.Helper()
|
||||
code, body := npmDo(t, http.MethodGet, url, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d: %s", url, code, body)
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), v); err != nil {
|
||||
t.Fatalf("GET %s: %v: %s", url, err, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnsiblePublishAndServe(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageAnsible)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
publish := base + "api/v3/artifacts/collections/"
|
||||
|
||||
v1 := testCollection(t, "unkin", "e2e_hello", "1.0.0")
|
||||
code, body := ansiblePublish(t, publish, v1, sha256Hex(v1))
|
||||
if code != http.StatusAccepted {
|
||||
t.Fatalf("publish = %d: %s", code, body)
|
||||
}
|
||||
var task struct{ Task string }
|
||||
_ = json.Unmarshal([]byte(body), &task)
|
||||
if task.Task != base+"api/v3/imports/collections/unkin-e2e_hello-1.0.0/" {
|
||||
t.Fatalf("task = %q", task.Task)
|
||||
}
|
||||
var st map[string]any
|
||||
getJSON(t, task.Task, &st)
|
||||
if st["state"] != "completed" || st["finished_at"] == "" {
|
||||
t.Fatalf("task = %v", st)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"api/v3/imports/collections/unkin-e2e_hello-9.9.9/", ""); code != http.StatusNotFound {
|
||||
t.Errorf("unknown task = %d, want 404", code)
|
||||
}
|
||||
|
||||
for _, v := range []string{"1.1.0", "2.0.0"} {
|
||||
tb := testCollection(t, "unkin", "e2e_hello", v)
|
||||
if code, body := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
|
||||
t.Fatalf("publish %s = %d: %s", v, code, body)
|
||||
}
|
||||
}
|
||||
if code, body := ansiblePublish(t, publish, v1, ""); code != http.StatusConflict || !strings.Contains(body, "already exists") {
|
||||
t.Errorf("identical republish = %d: %s, want 409", code, body)
|
||||
}
|
||||
if code, _ := ansiblePublish(t, publish, testCollection(t, "unkin", "e2e_hello", "1.0.0")[:20], ""); code != http.StatusBadRequest {
|
||||
t.Errorf("truncated tarball = %d, want 400", code)
|
||||
}
|
||||
|
||||
var root map[string]map[string]string
|
||||
getJSON(t, base+"api/", &root)
|
||||
if root["available_versions"]["v3"] != "v3/" {
|
||||
t.Fatalf("root = %v", root)
|
||||
}
|
||||
|
||||
var page struct {
|
||||
Meta struct{ Count int }
|
||||
Links struct{ Next, Previous *string }
|
||||
Data []struct{ Version, Href string }
|
||||
}
|
||||
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/versions/?limit=2", &page)
|
||||
if page.Meta.Count != 3 || len(page.Data) != 2 || page.Data[0].Version != "2.0.0" || page.Links.Next == nil || page.Links.Previous != nil {
|
||||
t.Fatalf("page 1 = %+v", page)
|
||||
}
|
||||
next := *page.Links.Next
|
||||
if !strings.HasPrefix(next, "/api/v1/local/"+repo+"/") {
|
||||
t.Fatalf("next = %q, want root-relative", next)
|
||||
}
|
||||
next = srv.URL + next
|
||||
page.Links.Next = nil
|
||||
getJSON(t, next, &page)
|
||||
if len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next != nil {
|
||||
t.Fatalf("page 2 (%s) = %+v", next, page)
|
||||
}
|
||||
|
||||
var detail struct {
|
||||
DownloadURL string `json:"download_url"`
|
||||
Artifact struct{ Sha256 string }
|
||||
Metadata struct{ Dependencies map[string]string }
|
||||
Namespace struct{ Name string }
|
||||
Collection struct{ Name string }
|
||||
}
|
||||
getJSON(t, page.Data[0].Href, &detail)
|
||||
if detail.DownloadURL != base+"artifacts/unkin-e2e_hello-1.0.0.tar.gz" || detail.Artifact.Sha256 != sha256Hex(v1) ||
|
||||
detail.Metadata.Dependencies["ansible.posix"] != ">=1.0.0" || detail.Namespace.Name != "unkin" || detail.Collection.Name != "e2e_hello" {
|
||||
t.Fatalf("detail = %+v", detail)
|
||||
}
|
||||
if code, got := npmDo(t, http.MethodGet, detail.DownloadURL, ""); code != http.StatusOK || got != string(v1) {
|
||||
t.Fatalf("download = %d (%d bytes)", code, len(got))
|
||||
}
|
||||
|
||||
var coll map[string]any
|
||||
getJSON(t, base+"api/v3/collections/unkin/e2e_hello/", &coll)
|
||||
if coll["versions_url"] != base+"api/v3/collections/unkin/e2e_hello/versions/" || coll["updated_at"] == "" {
|
||||
t.Fatalf("collection = %v", coll)
|
||||
}
|
||||
for _, p := range []string{"api/v3/collections/unkin/nope/", "api/v3/collections/unkin/e2e_hello/versions/9.0.0/", "_index/unkin/e2e_hello/1.0.0", "other"} {
|
||||
if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound {
|
||||
t.Errorf("GET %s = %d, want 404", p, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnsiblePublishRejects(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageAnsible)
|
||||
publish := srv.URL + "/api/v1/local/" + repo + "/api/v3/artifacts/collections/"
|
||||
tb := testCollection(t, "unkin", "rej", "1.0.0")
|
||||
|
||||
if code, _ := ansiblePublish(t, publish, tb, strings.Repeat("0", 64)); code != http.StatusBadRequest {
|
||||
t.Errorf("sha256 mismatch = %d, want 400", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPost, publish, "raw"); code != http.StatusBadRequest {
|
||||
t.Errorf("non-multipart = %d, want 400", code)
|
||||
}
|
||||
old := ansibleMaxPublishBytes
|
||||
ansibleMaxPublishBytes = 64
|
||||
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("oversize = %d, want 413", code)
|
||||
}
|
||||
ansibleMaxPublishBytes = old
|
||||
generic := npmRepo(t, h, models.PackageGeneric)
|
||||
if code, _ := ansiblePublish(t, srv.URL+"/api/v1/local/"+generic+"/api/v3/artifacts/collections/", tb, ""); code != http.StatusMethodNotAllowed {
|
||||
t.Errorf("publish to generic local = %d, want 405", code)
|
||||
}
|
||||
if code, _ := ansiblePublish(t, publish, tb, ""); code != http.StatusAccepted {
|
||||
t.Errorf("publish after rejects = %d, want 202", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnsibleLocalPerHost(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageAnsible)
|
||||
if code, b := ansiblePublish(t, srv.URL+"/api/v1/local/"+repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "host", "1.0.0"), ""); code != http.StatusAccepted {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
routes := h.Routes()
|
||||
path := "/local/" + repo + "/api/v3/collections/unkin/host/versions/1.0.0/"
|
||||
r := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
r.Host = "x@evil.com"
|
||||
w := httptest.NewRecorder()
|
||||
routes.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Errorf("bad Host = %d, want 400", w.Code)
|
||||
}
|
||||
r = httptest.NewRequest(http.MethodGet, path, nil)
|
||||
r.Host = "aa.example"
|
||||
r.Header.Set("X-Forwarded-Proto", "https")
|
||||
w = httptest.NewRecorder()
|
||||
routes.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"download_url":"https://aa.example/api/v1/local/`+repo+`/artifacts/unkin-host-1.0.0.tar.gz"`) {
|
||||
t.Fatalf("detail = %d: %s", w.Code, w.Body)
|
||||
}
|
||||
if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto" {
|
||||
t.Errorf("Cache-Control %q Vary %q", cc, vary)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnsibleVirtualOverLocals(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
h.virtualEngine = virtual.NewEngine(h.db, nil)
|
||||
h.virtualEngine.AnsibleLocal = h.LocalAnsible
|
||||
a, b := npmRepo(t, h, models.PackageAnsible), npmRepo(t, h, models.PackageAnsible)
|
||||
for _, p := range []struct{ repo, version string }{{a, "1.0.0"}, {b, "1.0.0"}, {b, "2.0.0"}} {
|
||||
if code, body := ansiblePublish(t, srv.URL+"/api/v1/local/"+p.repo+"/api/v3/artifacts/collections/", testCollection(t, "unkin", "virt", p.version), ""); code != http.StatusAccepted {
|
||||
t.Fatalf("publish %s to %s = %d: %s", p.version, p.repo, code, body)
|
||||
}
|
||||
}
|
||||
virt := "v" + a
|
||||
if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageAnsible, Members: []string{a, b}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := srv.URL + "/api/v1/virtual/" + virt + "/"
|
||||
|
||||
var page struct {
|
||||
Meta struct{ Count int }
|
||||
Links struct{ Next *string }
|
||||
Data []struct{ Version, Href string }
|
||||
}
|
||||
getJSON(t, base+"api/v3/collections/unkin/virt/versions/?limit=1", &page)
|
||||
if page.Meta.Count != 2 || len(page.Data) != 1 || page.Data[0].Version != "1.0.0" || page.Links.Next == nil {
|
||||
t.Fatalf("page 1 = %+v", page)
|
||||
}
|
||||
href := page.Data[0].Href
|
||||
if !strings.HasPrefix(*page.Links.Next, "/api/v1/virtual/"+virt+"/") {
|
||||
t.Fatalf("next = %q, want root-relative", *page.Links.Next)
|
||||
}
|
||||
getJSON(t, srv.URL+*page.Links.Next, &page)
|
||||
if len(page.Data) != 1 || page.Data[0].Version != "2.0.0" || page.Links.Next != nil {
|
||||
t.Fatalf("page 2 = %+v", page)
|
||||
}
|
||||
|
||||
var detail struct {
|
||||
DownloadURL string `json:"download_url"`
|
||||
}
|
||||
getJSON(t, href, &detail)
|
||||
if detail.DownloadURL != base+a+"/artifacts/unkin-virt-1.0.0.tar.gz" {
|
||||
t.Fatalf("download_url = %q, want the first member via the virtual", detail.DownloadURL)
|
||||
}
|
||||
noFollow := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
resp, err := noFollow.Get(detail.DownloadURL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != srv.URL+"/api/v1/local/"+a+"/artifacts/unkin-virt-1.0.0.tar.gz" {
|
||||
t.Fatalf("download = %d -> %q", resp.StatusCode, resp.Header.Get("Location"))
|
||||
}
|
||||
for _, p := range []string{"api/v3/collections/unkin/nope/", "notamember/x.tar.gz"} {
|
||||
if code, _ := npmDo(t, http.MethodGet, base+p, ""); code != http.StatusNotFound {
|
||||
t.Errorf("GET %s = %d, want 404", p, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// serveVirtualAnsible serves the merged Galaxy v3 API and 302s downloads to
|
||||
// the member owning that version.
|
||||
func (h *ProxyHandler) serveVirtualAnsible(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) {
|
||||
ap := ansible.ParseAPIPath(path)
|
||||
base := proxyBaseURL + "/api/v1/virtual/" + url.PathEscape(virt.Name)
|
||||
var doc any
|
||||
var err error
|
||||
switch ap.Kind {
|
||||
case ansible.APIRoot:
|
||||
doc = ansible.Root()
|
||||
case ansible.APIVersion:
|
||||
var body []byte
|
||||
if body, err = h.virtualEngine.AnsibleVersion(r.Context(), virt, ap.Namespace, ap.Name, ap.Version, proxyBaseURL); err == nil {
|
||||
doc = json.RawMessage(body)
|
||||
}
|
||||
case ansible.APICollection, ansible.APIVersions:
|
||||
var d *virtual.AnsibleDoc
|
||||
if d, err = h.virtualEngine.AnsibleVersions(r.Context(), virt, ap.Namespace, ap.Name); err == nil {
|
||||
if ap.Kind == ansible.APICollection {
|
||||
doc = ansible.CollectionDoc(base, ap.Namespace, ap.Name, d.Refs)
|
||||
} else {
|
||||
limit, offset := ansible.Page(r.URL.Query())
|
||||
doc = ansible.VersionsPage(base, ap.Namespace, ap.Name, d.Refs, limit, offset)
|
||||
}
|
||||
}
|
||||
default:
|
||||
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL)
|
||||
switch {
|
||||
case err != nil:
|
||||
ansibleError(w, http.StatusBadRequest, err.Error())
|
||||
case !ok:
|
||||
ansibleError(w, http.StatusNotFound, "not found")
|
||||
default:
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
}
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, virtual.ErrNotFound):
|
||||
ansibleError(w, http.StatusNotFound, "not found")
|
||||
case err != nil:
|
||||
slog.Error("virtual ansible fetch failed", "virtual", virt.Name, "path", path, "error", err)
|
||||
ansibleError(w, http.StatusBadGateway, "bad gateway")
|
||||
default:
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
writeAnsibleJSON(w, http.StatusOK, doc)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
type plainProvider struct{ provider.Provider }
|
||||
|
||||
type forwardingProvider struct{ plainProvider }
|
||||
|
||||
func digits(v string) (string, bool) {
|
||||
if v == "" || strings.Trim(v, "0123456789") != "" {
|
||||
return "", false
|
||||
}
|
||||
return strings.TrimLeft(v, "0"), true
|
||||
}
|
||||
|
||||
func (forwardingProvider) ForwardedQuery(path string) map[string]provider.QueryValidator {
|
||||
if strings.HasSuffix(path, ".tar.gz") {
|
||||
return nil
|
||||
}
|
||||
return map[string]provider.QueryValidator{"limit": digits, "offset": digits}
|
||||
}
|
||||
|
||||
func TestForwardQuery(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prov provider.Provider
|
||||
path string
|
||||
query string
|
||||
want string
|
||||
}{
|
||||
{"non-forwarder keeps path", plainProvider{}, "a/versions/", "limit=10&offset=5", "a/versions/"},
|
||||
{"no query", forwardingProvider{}, "a/versions/", "", "a/versions/"},
|
||||
{"allowlisted params sorted", forwardingProvider{}, "a/versions/", "offset=5&limit=10", "a/versions/?limit=10&offset=5"},
|
||||
{"unknown params dropped", forwardingProvider{}, "a/versions/", "x=1&limit=10&cb=zz", "a/versions/?limit=10"},
|
||||
{"only unknown params", forwardingProvider{}, "a/versions/", "x=1", "a/versions/"},
|
||||
{"first value wins", forwardingProvider{}, "a/versions/", "limit=10&limit=20", "a/versions/?limit=10"},
|
||||
{"invalid value dropped", forwardingProvider{}, "a/versions/", "limit=1%202&offset=5", "a/versions/?offset=5"},
|
||||
{"value canonicalised", forwardingProvider{}, "a/versions/", "limit=0010", "a/versions/?limit=10"},
|
||||
{"path without allowlist", forwardingProvider{}, "a/b-1.0.0.tar.gz", "limit=10", "a/b-1.0.0.tar.gz"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
q, err := url.ParseQuery(tt.query)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := forwardQuery(tt.prov, tt.path, q); got != tt.want {
|
||||
t.Errorf("forwardQuery = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/gems"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// Local gems repos speak the RubyGems push/yank API and compact index. Per gem:
|
||||
// gems/<full>.gem package (immutable)
|
||||
// quick/Marshal.4.8/<full>.gemspec.rz marshalled spec (immutable)
|
||||
// _index/<name>/<full> compact index /info line; gates visibility
|
||||
// _yanked/<name>/<full> yank tombstone; blocks re-push of the version
|
||||
|
||||
var (
|
||||
gemsMaxPushBytes int64 = 256 << 20
|
||||
gemsLockWait = 30 * time.Second
|
||||
)
|
||||
|
||||
const (
|
||||
gemsIndexPrefix = "_index/"
|
||||
gemsYankedPrefix = "_yanked/"
|
||||
)
|
||||
|
||||
func gemsText(w http.ResponseWriter, status int, msg string) {
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
_, _ = io.WriteString(w, msg)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) localGemsRemote(w http.ResponseWriter, r *http.Request) (*models.Remote, bool) {
|
||||
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
|
||||
if err != nil || remote.RepoType != models.RepoTypeLocal || remote.PackageType != models.PackageGems {
|
||||
gemsText(w, http.StatusMethodNotAllowed, "only supported on local gems repositories")
|
||||
return nil, false
|
||||
}
|
||||
return remote, true
|
||||
}
|
||||
|
||||
// lockGemVersion serialises push and yank of one gem version across replicas.
|
||||
func (h *ProxyHandler) lockGemVersion(w http.ResponseWriter, r *http.Request, repo, full string) (func(), bool) {
|
||||
unlock, err := h.engine.Lock(r.Context(), repo, "_gemlock/"+full, gemsLockWait, 5*time.Minute)
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusServiceUnavailable, full+" is being pushed or yanked, retry")
|
||||
return nil, false
|
||||
}
|
||||
return unlock, true
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocalPost(w http.ResponseWriter, r *http.Request) {
|
||||
if isAnsiblePublish(chi.URLParam(r, "*")) {
|
||||
h.handleAnsiblePublish(w, r)
|
||||
return
|
||||
}
|
||||
if chi.URLParam(r, "*") != "api/v1/gems" {
|
||||
gemsText(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
remote, ok := h.localGemsRemote(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
gem, err := io.ReadAll(http.MaxBytesReader(w, r.Body, gemsMaxPushBytes))
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("gem exceeds %d bytes", gemsMaxPushBytes))
|
||||
return
|
||||
}
|
||||
spec, err := gems.ParseGem(gem)
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
quick, err := gems.QuickSpec(spec)
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
full := spec.FullName()
|
||||
unlock, ok := h.lockGemVersion(w, r, remote.Name, full)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
conflict := func() {
|
||||
gemsText(w, http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
|
||||
}
|
||||
index := gemsIndexPrefix + spec.Name + "/" + full
|
||||
for _, p := range []string{gemsYankedPrefix + spec.Name + "/" + full, index} {
|
||||
f, err := h.db.GetLocalFile(r.Context(), remote.Name, p)
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if f != nil && p == index {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
if f != nil {
|
||||
gemsText(w, http.StatusConflict, fmt.Sprintf("%s was yanked; re-pushing a yanked version is not allowed", full))
|
||||
return
|
||||
}
|
||||
}
|
||||
// The version is not live, so on failure every path is an invisible leftover and safe to drop.
|
||||
paths := []string{"gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz", index}
|
||||
fail := func(status int, msg string) {
|
||||
for _, p := range paths {
|
||||
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, p); err != nil {
|
||||
slog.Error("gem push cleanup failed", "repo", remote.Name, "path", p, "error", err)
|
||||
}
|
||||
}
|
||||
gemsText(w, status, msg)
|
||||
}
|
||||
// The index line gates visibility, so it is written last.
|
||||
for _, f := range []struct {
|
||||
path string
|
||||
data []byte
|
||||
}{
|
||||
{paths[0], gem},
|
||||
{paths[1], quick},
|
||||
} {
|
||||
ok, err := h.npmStoreImmutable(r.Context(), remote.Name, f.path, f.data, "application/octet-stream")
|
||||
if err != nil {
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
fail(http.StatusConflict, fmt.Sprintf("%s already exists; overwrites are not allowed", full))
|
||||
return
|
||||
}
|
||||
}
|
||||
line := gems.InfoLine(spec, gem)
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, index, []byte(line), "text/plain", false); err != nil {
|
||||
fail(http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("local gem pushed", "repo", remote.Name, "gem", full)
|
||||
gemsText(w, http.StatusOK, "Successfully registered gem: "+spec.Name+" ("+spec.Version+")")
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocalDelete(w http.ResponseWriter, r *http.Request) {
|
||||
if chi.URLParam(r, "*") != "api/v1/gems/yank" {
|
||||
gemsText(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
remote, ok := h.localGemsRemote(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 64<<10))
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusRequestEntityTooLarge, "yank body too large")
|
||||
return
|
||||
}
|
||||
form, err := url.ParseQuery(string(body))
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusBadRequest, "invalid form body")
|
||||
return
|
||||
}
|
||||
for k, v := range r.URL.Query() {
|
||||
if form.Get(k) == "" {
|
||||
form[k] = v
|
||||
}
|
||||
}
|
||||
spec := &gems.Spec{Name: form.Get("gem_name"), Version: form.Get("version"), Platform: form.Get("platform")}
|
||||
if spec.Platform == "" {
|
||||
spec.Platform = "ruby"
|
||||
}
|
||||
if !gems.NameRe.MatchString(spec.Name) || !gems.VersionRe.MatchString(spec.Version) || !gems.PlatformRe.MatchString(spec.Platform) {
|
||||
gemsText(w, http.StatusBadRequest, "invalid gem_name, version or platform")
|
||||
return
|
||||
}
|
||||
|
||||
full := spec.FullName()
|
||||
unlock, ok := h.lockGemVersion(w, r, remote.Name, full)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer unlock()
|
||||
index := gemsIndexPrefix + spec.Name + "/" + full
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, index)
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if existing == nil {
|
||||
gemsText(w, http.StatusNotFound, "The version "+spec.Version+" does not exist")
|
||||
return
|
||||
}
|
||||
// The tombstone keeps the yanked line and is written before any delete.
|
||||
line, err := h.readLocalBlob(r.Context(), existing.ContentHash)
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, gemsYankedPrefix+spec.Name+"/"+full, line, "text/plain", false); err != nil && !errors.Is(err, database.ErrAlreadyExists) {
|
||||
gemsText(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
for _, p := range []string{index, "gems/" + full + ".gem", "quick/Marshal.4.8/" + full + ".gemspec.rz"} {
|
||||
if err := h.db.DeleteLocalFile(context.WithoutCancel(r.Context()), remote.Name, p); err != nil {
|
||||
gemsText(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
slog.Info("local gem yanked", "repo", remote.Name, "gem", full)
|
||||
gemsText(w, http.StatusOK, "Successfully deleted gem: "+spec.Name+" ("+spec.Version+")")
|
||||
}
|
||||
|
||||
// isGemsIndexPath reports whether path is a compact index document.
|
||||
func isGemsIndexPath(path string) bool {
|
||||
return path == "versions" || path == "names" || strings.HasPrefix(path, "info/")
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveLocalGemsIndex(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) {
|
||||
body, err := h.LocalGemsIndex(r.Context(), remote.Name, path)
|
||||
if errors.Is(err, virtual.ErrNotFound) {
|
||||
gemsText(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
gemsText(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
gemsText(w, http.StatusOK, string(body))
|
||||
}
|
||||
|
||||
// LocalGemsIndex builds a local gems repo's versions, names or info/<gem>
|
||||
// document, or returns virtual.ErrNotFound.
|
||||
func (h *ProxyHandler) LocalGemsIndex(ctx context.Context, repo, path string) ([]byte, error) {
|
||||
if name, ok := strings.CutPrefix(path, "info/"); ok {
|
||||
if !gems.NameRe.MatchString(name) {
|
||||
return nil, virtual.ErrNotFound
|
||||
}
|
||||
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix+name+"/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(lines[name]) == 0 {
|
||||
return nil, virtual.ErrNotFound
|
||||
}
|
||||
return gems.InfoBody(lines[name]), nil
|
||||
}
|
||||
if path != "versions" && path != "names" {
|
||||
return nil, virtual.ErrNotFound
|
||||
}
|
||||
lines, err := h.localGemsLines(ctx, repo, gemsIndexPrefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
names := slices.Sorted(maps.Keys(lines))
|
||||
var b strings.Builder
|
||||
if path == "names" {
|
||||
b.WriteString("---\n")
|
||||
for _, n := range names {
|
||||
b.WriteString(n + "\n")
|
||||
}
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
b.WriteString(gems.VersionsHeader(time.Now()))
|
||||
for _, n := range names {
|
||||
b.WriteString(gems.VersionsLine(n, gems.InfoBody(lines[n])) + "\n")
|
||||
}
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
|
||||
// localGemsLines returns /info lines per gem name in push order.
|
||||
func (h *ProxyHandler) localGemsLines(ctx context.Context, repo, prefix string) (map[string][]string, error) {
|
||||
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, prefix)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.SortFunc(files, func(a, b database.LocalFile) int { return cmp.Compare(a.ID, b.ID) })
|
||||
out := map[string][]string{}
|
||||
for _, f := range files {
|
||||
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
|
||||
rest, ok := strings.CutPrefix(f.FilePath, prefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(prefix, gemsIndexPrefix)
|
||||
if name == "" {
|
||||
name, _, _ = strings.Cut(rest, "/")
|
||||
} else {
|
||||
name = strings.TrimSuffix(name, "/")
|
||||
}
|
||||
line, err := h.readLocalBlob(ctx, f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[name] = append(out[name], string(line))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func testGemFile(t *testing.T, name, version, platform string) []byte {
|
||||
t.Helper()
|
||||
meta := fmt.Sprintf(`--- !ruby/object:Gem::Specification
|
||||
name: %s
|
||||
version: !ruby/object:Gem::Version
|
||||
version: %s
|
||||
platform: %s
|
||||
dependencies:
|
||||
- !ruby/object:Gem::Dependency
|
||||
name: rack
|
||||
requirement: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - "~>"
|
||||
- !ruby/object:Gem::Version
|
||||
version: '3.0'
|
||||
type: :runtime
|
||||
prerelease: false
|
||||
rubygems_version: 3.5.22
|
||||
specification_version: 4
|
||||
summary: test
|
||||
`, name, version, platform)
|
||||
var gz bytes.Buffer
|
||||
zw := gzip.NewWriter(&gz)
|
||||
_, _ = zw.Write([]byte(meta))
|
||||
_ = zw.Close()
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "metadata.gz", Mode: 0o444, Size: int64(gz.Len())})
|
||||
_, _ = tw.Write(gz.Bytes())
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func sha256Hex(b []byte) string {
|
||||
s := sha256.Sum256(b)
|
||||
return hex.EncodeToString(s[:])
|
||||
}
|
||||
|
||||
func TestGemsPushIndexYank(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageGems)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
|
||||
g1 := testGemFile(t, "hello_gem", "0.1.0", "ruby")
|
||||
g2 := testGemFile(t, "hello_gem", "0.2.0", "ruby")
|
||||
gj := testGemFile(t, "hello_gem", "0.2.0", "java")
|
||||
other := testGemFile(t, "hello-gem", "1.0.0", "ruby")
|
||||
for _, g := range [][]byte{g1, g2, gj, other} {
|
||||
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g)); code != http.StatusOK {
|
||||
t.Fatalf("push = %d: %s", code, body)
|
||||
}
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)+"x"); code != http.StatusConflict {
|
||||
t.Fatalf("repush with different bytes = %d, want 409", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(g1)); code != http.StatusConflict {
|
||||
t.Fatalf("identical repush = %d, want 409", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", "not a gem"); code != http.StatusBadRequest {
|
||||
t.Fatalf("garbage push = %d, want 400", code)
|
||||
}
|
||||
|
||||
code, info := npmDo(t, http.MethodGet, base+"info/hello_gem", "")
|
||||
want := "---\n0.1.0 rack:~> 3.0|checksum:" + sha256Hex(g1) + "\n0.2.0 rack:~> 3.0|checksum:" + sha256Hex(g2) +
|
||||
"\n0.2.0-java rack:~> 3.0|checksum:" + sha256Hex(gj) + "\n"
|
||||
if code != http.StatusOK || info != want {
|
||||
t.Fatalf("info = %d\n%q\nwant\n%q", code, info, want)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodHead, base+"versions", ""); code != http.StatusOK {
|
||||
t.Fatalf("HEAD versions = %d; gem probes the compact index with HEAD", code)
|
||||
}
|
||||
_, versions := npmDo(t, http.MethodGet, base+"versions", "")
|
||||
head, body, ok := strings.Cut(versions, "\n---\n")
|
||||
if !ok || !strings.HasPrefix(head, "created_at: ") {
|
||||
t.Fatalf("versions header = %q", versions)
|
||||
}
|
||||
_, otherInfo := npmDo(t, http.MethodGet, base+"info/hello-gem", "")
|
||||
wantBody := fmt.Sprintf("hello-gem 1.0.0 %x\nhello_gem 0.1.0,0.2.0,0.2.0-java %x\n", md5.Sum([]byte(otherInfo)), md5.Sum([]byte(info)))
|
||||
if body != wantBody {
|
||||
t.Fatalf("versions body =\n%q\nwant\n%q", body, wantBody)
|
||||
}
|
||||
if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello-gem\nhello_gem\n" {
|
||||
t.Fatalf("names = %q", names)
|
||||
}
|
||||
if code, got := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusOK || got != string(gj) {
|
||||
t.Fatalf("download = %d", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"quick/Marshal.4.8/hello_gem-0.1.0.gemspec.rz", ""); code != http.StatusOK {
|
||||
t.Fatalf("quick spec = %d", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"info/missing", ""); code != http.StatusNotFound {
|
||||
t.Fatalf("missing info = %d", code)
|
||||
}
|
||||
|
||||
yank := func(form string) int {
|
||||
req, _ := http.NewRequest(http.MethodDelete, base+"api/v1/gems/yank", strings.NewReader(form))
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusOK {
|
||||
t.Fatalf("yank = %d", c)
|
||||
}
|
||||
if c := yank("gem_name=hello_gem&version=0.2.0&platform=java"); c != http.StatusNotFound {
|
||||
t.Fatalf("second yank = %d, want 404", c)
|
||||
}
|
||||
if c := yank("gem_name=../x&version=1"); c != http.StatusBadRequest {
|
||||
t.Fatalf("bad yank = %d, want 400", c)
|
||||
}
|
||||
if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") || !strings.Contains(info, "0.2.0 ") {
|
||||
t.Fatalf("info after yank = %q", info)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"gems/hello_gem-0.2.0-java.gem", ""); code != http.StatusNotFound {
|
||||
t.Fatalf("yanked download = %d, want 404", code)
|
||||
}
|
||||
if _, v := npmDo(t, http.MethodGet, base+"versions", ""); strings.Contains(v, "java") || !strings.Contains(v, "hello_gem 0.1.0,0.2.0 ") {
|
||||
t.Fatalf("versions after yank = %q", v)
|
||||
}
|
||||
for _, g := range []string{string(gj), string(gj) + "x"} {
|
||||
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", g); code != http.StatusConflict || !strings.Contains(body, "yanked") {
|
||||
t.Fatalf("push after yank = %d %q, want 409", code, body)
|
||||
}
|
||||
}
|
||||
if _, info := npmDo(t, http.MethodGet, base+"info/hello_gem", ""); strings.Contains(info, "java") {
|
||||
t.Fatalf("info after rejected re-push = %q", info)
|
||||
}
|
||||
|
||||
if c := yank("gem_name=hello-gem&version=1.0.0"); c != http.StatusOK {
|
||||
t.Fatalf("yank last version = %d", c)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"info/hello-gem", ""); code != http.StatusNotFound {
|
||||
t.Fatalf("info for fully yanked gem = %d, want 404", code)
|
||||
}
|
||||
if _, names := npmDo(t, http.MethodGet, base+"names", ""); names != "---\nhello_gem\n" {
|
||||
t.Fatalf("names after yanking every version = %q", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGemsPushRejects(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
npmLocal := npmRepo(t, h, models.PackageNPM)
|
||||
g := string(testGemFile(t, "g", "1.0.0", "ruby"))
|
||||
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+npmLocal+"/api/v1/gems", g); code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("push to npm local = %d, want 405", code)
|
||||
}
|
||||
repo := npmRepo(t, h, models.PackageGems)
|
||||
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/other", g); code != http.StatusNotFound {
|
||||
t.Fatalf("push to other path = %d, want 404", code)
|
||||
}
|
||||
old := gemsMaxPushBytes
|
||||
gemsMaxPushBytes = 64
|
||||
defer func() { gemsMaxPushBytes = old }()
|
||||
if code, _ := npmDo(t, http.MethodPost, srv.URL+"/api/v1/local/"+repo+"/api/v1/gems", g); code != http.StatusRequestEntityTooLarge {
|
||||
t.Fatalf("oversized push = %d, want 413", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGemsPushYankSerialised(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageGems)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
g := string(testGemFile(t, "race", "1.0.0", "ruby"))
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 16; i++ {
|
||||
wg.Add(2)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", g); code != http.StatusOK && code != http.StatusConflict {
|
||||
t.Errorf("push = %d: %s", code, body)
|
||||
}
|
||||
}()
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if code, body := npmDo(t, http.MethodDelete, base+"api/v1/gems/yank?gem_name=race&version=1.0.0", ""); code != http.StatusOK && code != http.StatusNotFound {
|
||||
t.Errorf("yank = %d: %s", code, body)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
present := func(p string) bool {
|
||||
f, err := h.db.GetLocalFile(context.Background(), repo, p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return f != nil
|
||||
}
|
||||
live := []string{"_index/race/race-1.0.0", "gems/race-1.0.0.gem", "quick/Marshal.4.8/race-1.0.0.gemspec.rz"}
|
||||
tomb := present("_yanked/race/race-1.0.0")
|
||||
for _, p := range live {
|
||||
if present(p) == tomb {
|
||||
t.Errorf("tombstoned=%v but %s present=%v", tomb, p, !tomb)
|
||||
}
|
||||
}
|
||||
_, info := npmDo(t, http.MethodGet, base+"info/race", "")
|
||||
if strings.Contains(info, "1.0.0") == tomb {
|
||||
t.Errorf("tombstoned=%v but info = %q", tomb, info)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGemsLockTimeoutIs503(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageGems)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
redis, err := cache.NewRedis(testRedis)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
token, err := redis.AcquireLock(context.Background(), repo, "_gemlock/busy-1.0.0", time.Minute)
|
||||
if token == "" || err != nil {
|
||||
t.Fatalf("hold lock = %q %v", token, err)
|
||||
}
|
||||
prev := gemsLockWait
|
||||
gemsLockWait = 100 * time.Millisecond
|
||||
t.Cleanup(func() { gemsLockWait = prev })
|
||||
|
||||
if code, _ := npmDo(t, http.MethodPost, base+"api/v1/gems", string(testGemFile(t, "busy", "1.0.0", "ruby"))); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("push under held lock = %d, want 503", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodDelete, base+"api/v1/gems/yank?gem_name=busy&version=1.0.0", ""); code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("yank under held lock = %d, want 503", code)
|
||||
}
|
||||
if err := redis.ReleaseLock(context.Background(), repo, "_gemlock/busy-1.0.0", token); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, body := npmDo(t, http.MethodPost, base+"api/v1/gems", string(testGemFile(t, "busy", "1.0.0", "ruby"))); code != http.StatusOK {
|
||||
t.Fatalf("push after release = %d: %s", code, body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// serveVirtualGems serves the merged compact index and 302s downloads to the
|
||||
// member owning that version.
|
||||
func (h *ProxyHandler) serveVirtualGems(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) {
|
||||
var body []byte
|
||||
var loc string
|
||||
var err error
|
||||
if isGemsIndexPath(path) {
|
||||
body, _, err = h.virtualEngine.FetchGems(r.Context(), virt, path)
|
||||
} else {
|
||||
loc, err = h.virtualEngine.GemsRedirect(r.Context(), virt, path, r.URL.RawQuery, proxyBaseURL)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, virtual.ErrNotFound):
|
||||
gemsText(w, http.StatusNotFound, "not found")
|
||||
case errors.Is(err, virtual.ErrBadPath):
|
||||
gemsText(w, http.StatusBadRequest, err.Error())
|
||||
case err != nil:
|
||||
slog.Error("virtual gems fetch failed", "virtual", virt.Name, "path", path, "error", err)
|
||||
gemsText(w, http.StatusBadGateway, "bad gateway")
|
||||
case loc != "":
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
default:
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
gemsText(w, http.StatusOK, string(body))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// serveVirtualGoProxy serves the merged @v/list and 302s @latest and version
|
||||
// files to the member owning that version.
|
||||
func (h *ProxyHandler) serveVirtualGoProxy(w http.ResponseWriter, r *http.Request, virt models.Virtual, path string) {
|
||||
body, loc, err := h.virtualEngine.GoProxy(r.Context(), virt, path, r.URL.RawQuery)
|
||||
switch {
|
||||
case errors.Is(err, virtual.ErrNotFound):
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
case errors.Is(err, virtual.ErrBadPath):
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
case err != nil:
|
||||
slog.Error("virtual goproxy fetch failed", "virtual", virt.Name, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
case loc != "":
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
default:
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// Local npm repos speak the npm registry publish/read protocol. Per package:
|
||||
// <pkg>/-/<base>-<version>.tgz tarball (immutable)
|
||||
// <pkg>/-/<base>-<version>.json version manifest (immutable)
|
||||
// <pkg>/-/dist-tags/<tag> version string (mutable)
|
||||
|
||||
var (
|
||||
npmNameRe = regexp.MustCompile(`^(@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*$`)
|
||||
npmRefRe = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z.+_-]*$`)
|
||||
npmSemverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`)
|
||||
|
||||
npmMaxPublishBytes int64 = 256 << 20
|
||||
)
|
||||
|
||||
func (h *ProxyHandler) localNPMRemote(r *http.Request, name string) (*models.Remote, bool) {
|
||||
remote, err := h.db.GetRemote(r.Context(), name)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageNPM
|
||||
}
|
||||
|
||||
// npmPackageName returns the package a packument/publish path addresses, or ""
|
||||
// when the path is something else (e.g. a tarball under <pkg>/-/).
|
||||
func npmPackageName(raw string) string {
|
||||
p, err := url.PathUnescape(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
p = strings.TrimSuffix(p, "/")
|
||||
if !npmNameRe.MatchString(p) {
|
||||
return ""
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
type npmRelease struct {
|
||||
version string
|
||||
manifest []byte
|
||||
tarball []byte
|
||||
tags []string
|
||||
}
|
||||
|
||||
func parseNPMPublish(body io.Reader, pkg string) (*npmRelease, error) {
|
||||
var doc struct {
|
||||
Name string `json:"name"`
|
||||
DistTags map[string]string `json:"dist-tags"`
|
||||
Versions map[string]json.RawMessage `json:"versions"`
|
||||
Attachments map[string]struct {
|
||||
Data string `json:"data"`
|
||||
} `json:"_attachments"`
|
||||
}
|
||||
if err := json.NewDecoder(body).Decode(&doc); err != nil {
|
||||
return nil, fmt.Errorf("invalid publish body: %w", err)
|
||||
}
|
||||
if doc.Name != pkg {
|
||||
return nil, fmt.Errorf("body name %q does not match %q", doc.Name, pkg)
|
||||
}
|
||||
if len(doc.Versions) != 1 || len(doc.Attachments) != 1 {
|
||||
return nil, errors.New("publish must carry exactly one version and one tarball")
|
||||
}
|
||||
|
||||
rel := &npmRelease{}
|
||||
for v, m := range doc.Versions {
|
||||
rel.version, rel.manifest = v, m
|
||||
}
|
||||
if !npmSemverRe.MatchString(rel.version) {
|
||||
return nil, fmt.Errorf("invalid version %q", rel.version)
|
||||
}
|
||||
var meta struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Dist struct {
|
||||
Integrity string `json:"integrity"`
|
||||
Shasum string `json:"shasum"`
|
||||
} `json:"dist"`
|
||||
}
|
||||
if err := json.Unmarshal(rel.manifest, &meta); err != nil {
|
||||
return nil, fmt.Errorf("invalid version manifest: %w", err)
|
||||
}
|
||||
if meta.Name != pkg || meta.Version != rel.version {
|
||||
return nil, errors.New("version manifest name/version mismatch")
|
||||
}
|
||||
|
||||
for _, a := range doc.Attachments {
|
||||
tgz, err := base64.StdEncoding.DecodeString(a.Data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid tarball encoding: %w", err)
|
||||
}
|
||||
rel.tarball = tgz
|
||||
}
|
||||
if err := verifyNPMDist(rel.tarball, meta.Dist.Integrity, meta.Dist.Shasum); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for tag, v := range doc.DistTags {
|
||||
if !npmRefRe.MatchString(tag) || v != rel.version {
|
||||
return nil, fmt.Errorf("dist-tag %q must point at the published version", tag)
|
||||
}
|
||||
rel.tags = append(rel.tags, tag)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func verifyNPMDist(tgz []byte, integrity, shasum string) error {
|
||||
if b64, ok := strings.CutPrefix(integrity, "sha512-"); ok {
|
||||
sum := sha512.Sum512(tgz)
|
||||
if base64.StdEncoding.EncodeToString(sum[:]) != b64 {
|
||||
return errors.New("tarball does not match dist.integrity")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if shasum != "" {
|
||||
sum := sha1.Sum(tgz)
|
||||
if hex.EncodeToString(sum[:]) != shasum {
|
||||
return errors.New("tarball does not match dist.shasum")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("version manifest carries no sha512 integrity or shasum")
|
||||
}
|
||||
|
||||
func npmError(w http.ResponseWriter, status int, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleLocalPut(w http.ResponseWriter, r *http.Request) {
|
||||
remote, ok := h.localNPMRemote(r, chi.URLParam(r, "localName"))
|
||||
if !ok {
|
||||
npmError(w, http.StatusMethodNotAllowed, "publish is only supported on local npm repositories")
|
||||
return
|
||||
}
|
||||
pkg := npmPackageName(chi.URLParam(r, "*"))
|
||||
if pkg == "" {
|
||||
npmError(w, http.StatusBadRequest, "invalid package name")
|
||||
return
|
||||
}
|
||||
rel, err := parseNPMPublish(http.MaxBytesReader(w, r.Body, npmMaxPublishBytes), pkg)
|
||||
if err != nil {
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
npmError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("publish body exceeds %d bytes", npmMaxPublishBytes))
|
||||
return
|
||||
}
|
||||
npmError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
base := pkg[strings.LastIndex(pkg, "/")+1:]
|
||||
stem := pkg + "/-/" + base + "-" + rel.version
|
||||
// The manifest gates visibility and tags may only name visible versions, so
|
||||
// the order is tarball, manifest, tags. A tarball left by a failed publish is
|
||||
// accepted again only with identical bytes; an existing manifest always conflicts.
|
||||
conflict := func() {
|
||||
npmError(w, http.StatusConflict, fmt.Sprintf("%s@%s already exists; overwrites are not allowed", pkg, rel.version))
|
||||
}
|
||||
ok, err = h.npmStoreImmutable(r.Context(), remote.Name, stem+".tgz", rel.tarball, "application/octet-stream")
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, stem+".json", rel.manifest, "application/json", false); err != nil {
|
||||
if errors.Is(err, database.ErrAlreadyExists) {
|
||||
conflict()
|
||||
return
|
||||
}
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
for _, tag := range rel.tags {
|
||||
if _, err := h.npmStore(r.Context(), remote.Name, pkg+"/-/dist-tags/"+tag, []byte(rel.version), "text/plain", true); err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
slog.Info("local npm package published", "repo", remote.Name, "package", pkg, "version", rel.version)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "id": pkg})
|
||||
}
|
||||
|
||||
// npmStoreImmutable reports false when path already holds different bytes.
|
||||
func (h *ProxyHandler) npmStoreImmutable(ctx context.Context, repo, path string, data []byte, contentType string) (bool, error) {
|
||||
hash, err := h.npmStore(ctx, repo, path, data, contentType, false)
|
||||
if !errors.Is(err, database.ErrAlreadyExists) {
|
||||
return err == nil, err
|
||||
}
|
||||
existing, err := h.db.GetLocalFile(ctx, repo, path)
|
||||
if err != nil || existing == nil {
|
||||
return false, fmt.Errorf("read existing %s: %v", path, err)
|
||||
}
|
||||
return existing.ContentHash == hash, nil
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) npmStore(ctx context.Context, repo, path string, data []byte, contentType string, mutable bool) (string, error) {
|
||||
result, err := h.cas.Store(ctx, bytes.NewReader(data), contentType)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if mutable {
|
||||
return result.ContentHash, h.db.UpsertLocalFile(ctx, repo, path, result.ContentHash)
|
||||
}
|
||||
return result.ContentHash, h.db.CreateLocalFile(ctx, repo, path, result.ContentHash)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) readLocalBlob(ctx context.Context, contentHash string) ([]byte, error) {
|
||||
reader, _, err := h.store.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer reader.Close()
|
||||
return io.ReadAll(reader)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request, remote *models.Remote, pkg string) {
|
||||
files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, pkg+"/-/")
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
base, ok := requestBaseURL(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) }
|
||||
tarballBase := base + "/api/v1/local/" + remote.Name + "/"
|
||||
doc, err := buildNPMPackument(pkg, files, read, tarballBase)
|
||||
if err != nil {
|
||||
npmError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
if doc == nil {
|
||||
npmError(w, http.StatusNotFound, "not found")
|
||||
return
|
||||
}
|
||||
setNPMPerHostHeaders(w)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("X-Artifact-Source", "local")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_ = json.NewEncoder(w).Encode(doc)
|
||||
}
|
||||
|
||||
// setNPMPerHostHeaders keeps shared caches off packuments whose tarball URLs
|
||||
// are built from the request host and Accept (abbreviated vs full).
|
||||
func setNPMPerHostHeaders(w http.ResponseWriter) {
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Vary", "Host, X-Forwarded-Proto, Accept")
|
||||
}
|
||||
|
||||
// buildNPMPackument returns nil when pkg has no published versions.
|
||||
func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) {
|
||||
prefix := pkg + "/-/"
|
||||
versions := map[string]any{}
|
||||
tags := map[string]string{}
|
||||
for _, f := range files {
|
||||
// LIKE treats '_' as a wildcard, so the prefix match can over-select.
|
||||
rest, ok := strings.CutPrefix(f.FilePath, prefix)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if tag, isTag := strings.CutPrefix(rest, "dist-tags/"); isTag {
|
||||
v, err := read(f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tags[tag] = string(v)
|
||||
continue
|
||||
}
|
||||
stem, isManifest := strings.CutSuffix(rest, ".json")
|
||||
if !isManifest {
|
||||
continue
|
||||
}
|
||||
raw, err := read(f.ContentHash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(raw, &m); err != nil {
|
||||
return nil, fmt.Errorf("manifest %s: %w", f.FilePath, err)
|
||||
}
|
||||
dist, _ := m["dist"].(map[string]any)
|
||||
if dist == nil {
|
||||
dist = map[string]any{}
|
||||
m["dist"] = dist
|
||||
}
|
||||
dist["tarball"] = tarballBase + prefix + stem + ".tgz"
|
||||
v, _ := m["version"].(string)
|
||||
versions[v] = m
|
||||
}
|
||||
if len(versions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
for tag, v := range tags {
|
||||
if _, ok := versions[v]; !ok {
|
||||
delete(tags, tag)
|
||||
}
|
||||
}
|
||||
return map[string]any{"_id": pkg, "name": pkg, "dist-tags": tags, "versions": versions}, nil
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/cache"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
var (
|
||||
testDSN string
|
||||
testMinio testsupport.MinioConn
|
||||
testRedis string
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
ctx := context.Background()
|
||||
dsn, termPG, err := testsupport.StartPostgres(ctx)
|
||||
if err != nil {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
conn, termMinio, err := testsupport.StartMinio(ctx)
|
||||
if err != nil {
|
||||
termPG()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
redisURL, termRedis, err := testsupport.StartRedis(ctx)
|
||||
if err != nil {
|
||||
termMinio()
|
||||
termPG()
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
testDSN, testMinio, testRedis = dsn, conn, redisURL
|
||||
code := m.Run()
|
||||
termRedis()
|
||||
termMinio()
|
||||
termPG()
|
||||
os.Exit(code)
|
||||
}
|
||||
|
||||
func npmTestServer(t *testing.T) (*ProxyHandler, *httptest.Server) {
|
||||
t.Helper()
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(db.Close)
|
||||
var store *storage.S3
|
||||
for i := 0; i < 20; i++ {
|
||||
if store, err = storage.NewS3(testMinio.Endpoint, testMinio.AccessKey, testMinio.SecretKey, "npm", false, ""); err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
redis, err := cache.NewRedis(testRedis)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h := NewProxyHandler(proxy.NewEngine(db, redis, store), nil, db, store, nil)
|
||||
r := chi.NewRouter()
|
||||
r.Mount("/api/v1", h.Routes())
|
||||
srv := httptest.NewServer(r)
|
||||
t.Cleanup(srv.Close)
|
||||
return h, srv
|
||||
}
|
||||
|
||||
func npmRepo(t *testing.T, h *ProxyHandler, pt models.PackageType) string {
|
||||
t.Helper()
|
||||
name := fmt.Sprintf("npm-%s-%d", pt, time.Now().UnixNano())
|
||||
if err := h.db.CreateRemote(context.Background(), &models.Remote{Name: name, PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func npmDo(t *testing.T, method, url, body string) (int, string) {
|
||||
t.Helper()
|
||||
req, _ := http.NewRequest(method, url, strings.NewReader(body))
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return resp.StatusCode, string(b)
|
||||
}
|
||||
|
||||
func packumentVersions(t *testing.T, url string) map[string]any {
|
||||
t.Helper()
|
||||
code, body := npmDo(t, http.MethodGet, url, "")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("GET %s = %d: %s", url, code, body)
|
||||
}
|
||||
var doc struct {
|
||||
Versions map[string]any `json:"versions"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return doc.Versions
|
||||
}
|
||||
|
||||
func packumentTags(t *testing.T, url string) map[string]string {
|
||||
t.Helper()
|
||||
_, body := npmDo(t, http.MethodGet, url, "")
|
||||
var doc struct {
|
||||
Tags map[string]string `json:"dist-tags"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(body), &doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return doc.Tags
|
||||
}
|
||||
|
||||
func concurrentPuts(t *testing.T, url string, bodies ...string) []int {
|
||||
t.Helper()
|
||||
var wg sync.WaitGroup
|
||||
codes := make([]int, len(bodies))
|
||||
for i, b := range bodies {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
codes[i], _ = npmDo(t, http.MethodPut, url, b)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return codes
|
||||
}
|
||||
|
||||
func TestNPMPublishHandler(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
repo := npmRepo(t, h, models.PackageNPM)
|
||||
base := srv.URL + "/api/v1/local/" + repo + "/"
|
||||
|
||||
t.Run("scoped name via %2f", func(t *testing.T) {
|
||||
body := publishBody(t, "@acme/widget", []byte("scoped tgz"), publishOpts{})
|
||||
if code, b := npmDo(t, http.MethodPut, base+"@acme%2fwidget", body); code != http.StatusCreated {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
v := packumentVersions(t, base+"@acme%2fwidget")
|
||||
dist := v["1.0.0"].(map[string]any)["dist"].(map[string]any)
|
||||
tarball := dist["tarball"].(string)
|
||||
if !strings.HasPrefix(tarball, base+"@acme/widget/-/widget-1.0.0.tgz") {
|
||||
t.Fatalf("tarball = %s", tarball)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodGet, tarball, ""); code != http.StatusOK || b != "scoped tgz" {
|
||||
t.Fatalf("tarball fetch = %d %q", code, b)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("republish", func(t *testing.T) {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("first publish = %d: %s", code, b)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("different"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("different bytes = %d, want 409", code)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v2"), publishOpts{version: "1.1.0"})); code != http.StatusCreated {
|
||||
t.Fatalf("publish 1.1.0 = %d: %s", code, b)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("identical republish = %d, want 409", code)
|
||||
}
|
||||
if tags := packumentTags(t, base+"repub"); tags["latest"] != "1.1.0" {
|
||||
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("rejected republish leaves tags", func(t *testing.T) {
|
||||
for _, v := range []string{"1.0.0", "1.1.0"} {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz "+v), publishOpts{version: v})); code != http.StatusCreated {
|
||||
t.Fatalf("publish %s = %d: %s", v, code, b)
|
||||
}
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz 1.0.0"), publishOpts{shasum: "x"})); code != http.StatusConflict {
|
||||
t.Fatalf("re-PUT 1.0.0 = %d, want 409", code)
|
||||
}
|
||||
if tags := packumentTags(t, base+"rej"); tags["latest"] != "1.1.0" {
|
||||
t.Errorf("latest = %q, want 1.1.0", tags["latest"])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("mid-publish window", func(t *testing.T) {
|
||||
if code, b := npmDo(t, http.MethodPut, base+"mid", publishBody(t, "mid", []byte("v1"), publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
ctx := context.Background()
|
||||
if _, err := h.npmStore(ctx, repo, "mid/-/mid-1.0.1.tgz", []byte("v2"), "application/octet-stream", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := packumentTags(t, base+"mid"); tags["latest"] != "1.0.0" {
|
||||
t.Errorf("tarball-only 1.0.1: latest = %q", tags["latest"])
|
||||
}
|
||||
if _, err := h.npmStore(ctx, repo, "mid/-/dist-tags/latest", []byte("1.0.1"), "text/plain", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := packumentTags(t, base+"mid"); tags["latest"] == "1.0.1" {
|
||||
t.Error("latest points at invisible 1.0.1")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent identical", func(t *testing.T) {
|
||||
body := publishBody(t, "same", []byte("same tgz"), publishOpts{})
|
||||
codes := concurrentPuts(t, base+"same", body, body)
|
||||
if codes[0]+codes[1] != http.StatusCreated+http.StatusConflict || codes[0] == codes[1] {
|
||||
t.Fatalf("codes = %v, want one 201 and one 409", codes)
|
||||
}
|
||||
if tags := packumentTags(t, base+"same"); len(tags) != 1 || tags["latest"] != "1.0.0" {
|
||||
t.Errorf("tags = %v", tags)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent different manifests", func(t *testing.T) {
|
||||
tgz := []byte("race tgz")
|
||||
codes := concurrentPuts(t, base+"race",
|
||||
publishBody(t, "race", tgz, publishOpts{shasum: "a", tags: map[string]string{"a": "1.0.0"}}),
|
||||
publishBody(t, "race", tgz, publishOpts{shasum: "b", tags: map[string]string{"b": "1.0.0"}}))
|
||||
won := map[int]string{}
|
||||
for i, c := range codes {
|
||||
won[c] = []string{"a", "b"}[i]
|
||||
}
|
||||
if len(won) != 2 || won[http.StatusCreated] == "" || won[http.StatusConflict] == "" {
|
||||
t.Fatalf("codes = %v, want one 201 and one 409", codes)
|
||||
}
|
||||
v := packumentVersions(t, base+"race")["1.0.0"].(map[string]any)
|
||||
if got := v["dist"].(map[string]any)["shasum"]; got != won[http.StatusCreated] {
|
||||
t.Errorf("manifest shasum = %v, winner %s", got, won[http.StatusCreated])
|
||||
}
|
||||
tags := packumentTags(t, base+"race")
|
||||
if len(tags) != 1 || tags[won[http.StatusCreated]] != "1.0.0" {
|
||||
t.Errorf("tags = %v, winner %s", tags, won[http.StatusCreated])
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("retry after tarball-only write", func(t *testing.T) {
|
||||
tgz := []byte("half published")
|
||||
if _, err := h.npmStore(context.Background(), repo, "retry/-/retry-1.0.0.tgz", tgz, "application/octet-stream", false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodGet, base+"retry", ""); code != http.StatusNotFound {
|
||||
t.Fatalf("tarball-only version visible: %d", code)
|
||||
}
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", []byte("other bytes"), publishOpts{})); code != http.StatusConflict {
|
||||
t.Errorf("retry with different bytes = %d, want 409", code)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", tgz, publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("retry with identical bytes = %d: %s", code, b)
|
||||
}
|
||||
if _, ok := packumentVersions(t, base+"retry")["1.0.0"]; !ok {
|
||||
t.Error("retried version missing from packument")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("concurrent versions", func(t *testing.T) {
|
||||
codes := concurrentPuts(t, base+"conc",
|
||||
publishBody(t, "conc", []byte("tgz 1.0.0"), publishOpts{version: "1.0.0"}),
|
||||
publishBody(t, "conc", []byte("tgz 2.0.0"), publishOpts{version: "2.0.0"}))
|
||||
if codes[0] != http.StatusCreated || codes[1] != http.StatusCreated {
|
||||
t.Fatalf("codes = %v", codes)
|
||||
}
|
||||
v := packumentVersions(t, base+"conc")
|
||||
if _, ok := v["1.0.0"]; !ok {
|
||||
t.Error("1.0.0 missing")
|
||||
}
|
||||
if _, ok := v["2.0.0"]; !ok {
|
||||
t.Error("2.0.0 missing")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("body too large", func(t *testing.T) {
|
||||
old := npmMaxPublishBytes
|
||||
npmMaxPublishBytes = 64
|
||||
defer func() { npmMaxPublishBytes = old }()
|
||||
if code, _ := npmDo(t, http.MethodPut, base+"big", publishBody(t, "big", []byte("x"), publishOpts{})); code != http.StatusRequestEntityTooLarge {
|
||||
t.Errorf("oversized body = %d, want 413", code)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("non-npm local", func(t *testing.T) {
|
||||
generic := npmRepo(t, h, models.PackageGeneric)
|
||||
if code, _ := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+generic+"/pkg", publishBody(t, "pkg", []byte("x"), publishOpts{})); code != http.StatusMethodNotAllowed {
|
||||
t.Errorf("publish to generic local = %d, want 405", code)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNPMPackumentPerHost(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
h.virtualEngine = virtual.NewEngine(h.db, nil)
|
||||
h.virtualEngine.NPMLocal = h.LocalNPMPackument
|
||||
repo := npmRepo(t, h, models.PackageNPM)
|
||||
virt := "v" + repo
|
||||
if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackageNPM, Members: []string{repo}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, b := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+repo+"/hostpkg", publishBody(t, "hostpkg", []byte("tgz"), publishOpts{})); code != http.StatusCreated {
|
||||
t.Fatalf("publish = %d: %s", code, b)
|
||||
}
|
||||
routes := h.Routes()
|
||||
get := func(path, host, proto string) *httptest.ResponseRecorder {
|
||||
r := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
r.Host = host
|
||||
if proto != "" {
|
||||
r.Header.Set("X-Forwarded-Proto", proto)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
routes.ServeHTTP(w, r)
|
||||
return w
|
||||
}
|
||||
for _, path := range []string{"/local/" + repo + "/hostpkg", "/virtual/" + virt + "/hostpkg"} {
|
||||
for _, host := range []string{"x@evil.com", "evil.example:"} {
|
||||
if w := get(path, host, ""); w.Code != http.StatusBadRequest {
|
||||
t.Errorf("%s Host %q = %d, want 400", path, host, w.Code)
|
||||
}
|
||||
}
|
||||
w := get(path, "aa.example", "https")
|
||||
if w.Code != http.StatusOK || !strings.Contains(w.Body.String(), `"tarball":"https://aa.example/api/v1/`) {
|
||||
t.Fatalf("%s = %d: %s", path, w.Code, w.Body)
|
||||
}
|
||||
if cc, vary := w.Header().Get("Cache-Control"), w.Header().Get("Vary"); cc != "no-store" || vary != "Host, X-Forwarded-Proto, Accept" {
|
||||
t.Errorf("%s Cache-Control %q Vary %q", path, cc, vary)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"crypto/sha1"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
func TestNPMPackageName(t *testing.T) {
|
||||
tests := map[string]string{
|
||||
"left-pad": "left-pad",
|
||||
"left-pad/": "left-pad",
|
||||
"@acme%2fwidget": "@acme/widget",
|
||||
"@acme%2Fwidget": "@acme/widget",
|
||||
"@acme/widget": "@acme/widget",
|
||||
"left-pad/-/left-pad.tgz": "",
|
||||
"@acme/widget/-/w-1.tgz": "",
|
||||
"a/b": "",
|
||||
"../etc": "",
|
||||
"Upper": "",
|
||||
"": "",
|
||||
}
|
||||
for in, want := range tests {
|
||||
if got := npmPackageName(in); got != want {
|
||||
t.Errorf("npmPackageName(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type publishOpts struct {
|
||||
bodyName, manifestName, version, integrity, shasum string
|
||||
tags map[string]string
|
||||
extraVersion bool
|
||||
}
|
||||
|
||||
func publishBody(t *testing.T, pkg string, tgz []byte, o publishOpts) string {
|
||||
t.Helper()
|
||||
sha := sha512.Sum512(tgz)
|
||||
sh1 := sha1.Sum(tgz)
|
||||
if o.bodyName == "" {
|
||||
o.bodyName = pkg
|
||||
}
|
||||
if o.manifestName == "" {
|
||||
o.manifestName = pkg
|
||||
}
|
||||
if o.version == "" {
|
||||
o.version = "1.0.0"
|
||||
}
|
||||
if o.integrity == "" {
|
||||
o.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha[:])
|
||||
}
|
||||
if o.shasum == "" {
|
||||
o.shasum = hex.EncodeToString(sh1[:])
|
||||
}
|
||||
if o.tags == nil {
|
||||
o.tags = map[string]string{"latest": o.version}
|
||||
}
|
||||
versions := map[string]any{o.version: map[string]any{
|
||||
"name": o.manifestName, "version": o.version, "main": "index.js",
|
||||
"dist": map[string]any{"integrity": o.integrity, "shasum": o.shasum, "tarball": "http://elsewhere/x.tgz"},
|
||||
}}
|
||||
if o.extraVersion {
|
||||
versions["2.0.0"] = map[string]any{"name": pkg, "version": "2.0.0"}
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{
|
||||
"_id": pkg, "name": o.bodyName, "dist-tags": o.tags, "versions": versions,
|
||||
"_attachments": map[string]any{pkg + "-" + o.version + ".tgz": map[string]any{
|
||||
"content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz),
|
||||
}},
|
||||
})
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func TestParseNPMPublish(t *testing.T) {
|
||||
tgz := []byte("fake tarball bytes")
|
||||
|
||||
rel, err := parseNPMPublish(strings.NewReader(publishBody(t, "@acme/widget", tgz, publishOpts{tags: map[string]string{"latest": "1.0.0", "beta": "1.0.0"}})), "@acme/widget")
|
||||
if err != nil {
|
||||
t.Fatalf("valid scoped publish rejected: %v", err)
|
||||
}
|
||||
if rel.version != "1.0.0" || string(rel.tarball) != string(tgz) || len(rel.tags) != 2 {
|
||||
t.Fatalf("unexpected release: %+v", rel)
|
||||
}
|
||||
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none", shasum: hex.EncodeToString(make([]byte, 20))})), "left-pad"); err == nil {
|
||||
t.Fatal("shasum fallback should reject a mismatched tarball")
|
||||
}
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none"})), "left-pad"); err != nil {
|
||||
t.Fatalf("matching shasum fallback rejected: %v", err)
|
||||
}
|
||||
|
||||
bad := map[string]publishOpts{
|
||||
"body name mismatch": {bodyName: "other"},
|
||||
"manifest name mismatch": {manifestName: "other"},
|
||||
"integrity mismatch": {integrity: "sha512-AAAA"},
|
||||
"path-like version": {version: "../1.0.0"},
|
||||
"non-semver version": {version: "1..2"},
|
||||
"leading-zero version": {version: "01.0.0"},
|
||||
"tag at another version": {tags: map[string]string{"latest": "9.9.9"}},
|
||||
"path-like tag": {tags: map[string]string{"../x": "1.0.0"}},
|
||||
"multiple versions at once": {extraVersion: true},
|
||||
}
|
||||
for name, o := range bad {
|
||||
if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, o)), "left-pad"); err == nil {
|
||||
t.Errorf("%s: expected rejection", name)
|
||||
}
|
||||
}
|
||||
if _, err := parseNPMPublish(strings.NewReader(`{"name":"left-pad","versions":{}}`), "left-pad"); err == nil {
|
||||
t.Error("deprecate-style body without attachments should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildNPMPackument(t *testing.T) {
|
||||
blobs := map[string]string{
|
||||
"h1": `{"name":"my_pkg","version":"1.0.0","dist":{"shasum":"aa","tarball":"http://elsewhere/x.tgz"}}`,
|
||||
"h2": `{"name":"my_pkg","version":"1.1.0"}`,
|
||||
"h3": "1.1.0",
|
||||
"h4": "2.0.0",
|
||||
"hx": `{"name":"myXpkg","version":"6.6.6"}`,
|
||||
}
|
||||
read := func(h string) ([]byte, error) {
|
||||
b, ok := blobs[h]
|
||||
if !ok {
|
||||
return nil, errors.New("missing blob")
|
||||
}
|
||||
return []byte(b), nil
|
||||
}
|
||||
files := []database.LocalFile{
|
||||
{FilePath: "my_pkg/-/dist-tags/latest", ContentHash: "h3"},
|
||||
{FilePath: "my_pkg/-/dist-tags/next", ContentHash: "h4"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.0.0.json", ContentHash: "h1"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.0.0.tgz", ContentHash: "nope"},
|
||||
{FilePath: "my_pkg/-/my_pkg-1.1.0.json", ContentHash: "h2"},
|
||||
{FilePath: "myXpkg/-/myXpkg-6.6.6.json", ContentHash: "hx"},
|
||||
}
|
||||
|
||||
doc, err := buildNPMPackument("my_pkg", files, read, "http://reg/api/v1/local/npm/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tags := doc["dist-tags"].(map[string]string); tags["latest"] != "1.1.0" || len(tags) != 1 {
|
||||
t.Errorf("dist-tags = %v", tags)
|
||||
}
|
||||
versions := doc["versions"].(map[string]any)
|
||||
if len(versions) != 2 {
|
||||
t.Fatalf("versions = %v (LIKE over-match must be dropped)", versions)
|
||||
}
|
||||
v1 := versions["1.0.0"].(map[string]any)["dist"].(map[string]any)
|
||||
if v1["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.0.0.tgz" || v1["shasum"] != "aa" {
|
||||
t.Errorf("1.0.0 dist = %v", v1)
|
||||
}
|
||||
if versions["1.1.0"].(map[string]any)["dist"].(map[string]any)["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.1.0.tgz" {
|
||||
t.Error("1.1.0 missing rewritten tarball")
|
||||
}
|
||||
|
||||
if doc, err := buildNPMPackument("absent", files, read, "x/"); err != nil || doc != nil {
|
||||
t.Errorf("absent package: doc=%v err=%v", doc, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
// LocalNPMPackument serves a local npm repo's packument to npm virtuals, with
|
||||
// tarball paths relative to the repo.
|
||||
func (h *ProxyHandler) LocalNPMPackument(ctx context.Context, repo, pkg string) ([]byte, error) {
|
||||
files, err := h.db.ListLocalFilesByPrefix(ctx, repo, pkg+"/-/")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
read := func(hash string) ([]byte, error) { return h.readLocalBlob(ctx, hash) }
|
||||
doc, err := buildNPMPackument(pkg, files, read, "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if doc == nil {
|
||||
return nil, virtual.ErrNotFound
|
||||
}
|
||||
return json.Marshal(doc)
|
||||
}
|
||||
+207
-34
@@ -6,16 +6,22 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type ProxyHandler struct {
|
||||
@@ -41,8 +47,14 @@ func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *da
|
||||
func (h *ProxyHandler) Routes() chi.Router {
|
||||
r := chi.NewRouter()
|
||||
r.Get("/remote/{remoteName}/*", h.handleProxy)
|
||||
r.Head("/remote/{remoteName}/*", h.handleProxyHead)
|
||||
r.Get("/local/{localName}/*", h.handleLocal)
|
||||
r.Head("/local/{localName}/*", h.handleLocal)
|
||||
r.Put("/local/{localName}/*", h.handleLocalPut)
|
||||
r.Post("/local/{localName}/*", h.handleLocalPost)
|
||||
r.Delete("/local/{localName}/*", h.handleLocalDelete)
|
||||
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
r.Head("/virtual/{virtualName}/*", h.handleVirtual)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -68,39 +80,51 @@ func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
// remoteFor resolves the remote and provider named in the request, writing the
|
||||
// error response itself when either is missing.
|
||||
func (h *ProxyHandler) remoteFor(w http.ResponseWriter, r *http.Request) (*models.Remote, provider.Provider, bool) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
return
|
||||
return nil, nil, false
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
return nil, nil, false
|
||||
}
|
||||
return remote, prov, true
|
||||
}
|
||||
|
||||
// serveRemote lets metadata-only remotes (e.g. github_rpm) synthesize their own
|
||||
// responses and redirect package downloads instead of proxying bytes.
|
||||
func (h *ProxyHandler) serveRemote(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) bool {
|
||||
rs, ok := prov.(provider.RemoteServer)
|
||||
return ok && rs.ServeRemote(w, r, *remote, path, fmt.Sprintf("%s://%s", Scheme(r), r.Host), h.db)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
remote, prov, ok := h.remoteFor(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
|
||||
// redirect package downloads to a backend remote instead of proxying bytes.
|
||||
if rs, ok := prov.(provider.RemoteServer); ok {
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
|
||||
return
|
||||
}
|
||||
path := chi.URLParam(r, "*")
|
||||
if h.serveRemote(w, r, remote, prov, path) {
|
||||
return
|
||||
}
|
||||
h.serveFetch(w, r, remote, prov, path)
|
||||
}
|
||||
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
|
||||
func (h *ProxyHandler) serveFetch(w http.ResponseWriter, r *http.Request, remote *models.Remote, prov provider.Provider, path string) {
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, forwardQuery(prov, path, r.URL.Query()), prov, r.Header)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
if errors.As(err, &proxyErr) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
|
||||
slog.Error("proxy fetch failed", "remote", remote.Name, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
@@ -108,37 +132,43 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
var body io.Reader = result.Reader
|
||||
if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
|
||||
raw, err := io.ReadAll(result.Reader)
|
||||
if err != nil {
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
raw, ok := rewriteServed(w, r, sr, *remote, result.Reader)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
|
||||
body, result.Size = bytes.NewReader(raw), int64(len(raw))
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", result.ContentType)
|
||||
w.Header().Set("X-Artifact-Source", result.Source)
|
||||
if result.Size > 0 {
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", result.Size))
|
||||
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
io.Copy(w, body)
|
||||
}
|
||||
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
remoteName := chi.URLParam(r, "remoteName")
|
||||
path := chi.URLParam(r, "*")
|
||||
func rewritesMutable(prov provider.Provider, path string) bool {
|
||||
_, sr := prov.(provider.ServeRewriter)
|
||||
_, rr := prov.(provider.ResponseRewriter)
|
||||
return (sr || rr) && prov.Classify(path) == provider.Mutable
|
||||
}
|
||||
|
||||
remote, err := h.db.GetRemote(r.Context(), remoteName)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
|
||||
// handleProxyHead answers HEAD like GET without a body. Responses a provider
|
||||
// synthesizes or rewrites per request go through the GET path so their headers
|
||||
// match; everything else is answered from store metadata or an upstream HEAD.
|
||||
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
remote, prov, ok := h.remoteFor(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
|
||||
path := chi.URLParam(r, "*")
|
||||
if h.serveRemote(w, r, remote, prov, path) {
|
||||
return
|
||||
}
|
||||
if rewritesMutable(prov, path) {
|
||||
h.serveFetch(w, r, remote, prov, path)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -149,7 +179,7 @@ func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, proxyErr.Message, proxyErr.Status)
|
||||
return
|
||||
}
|
||||
slog.Error("proxy head failed", "remote", remoteName, "path", path, "error", err)
|
||||
slog.Error("proxy head failed", "remote", remote.Name, "path", path, "error", err)
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
@@ -172,7 +202,27 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
proxyBaseURL, ok := requestBaseURL(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
if virt.PackageType == models.PackageGems {
|
||||
h.serveVirtualGems(w, r, *virt, path, proxyBaseURL)
|
||||
return
|
||||
}
|
||||
if virt.PackageType == models.PackageAnsible {
|
||||
h.serveVirtualAnsible(w, r, *virt, path, proxyBaseURL)
|
||||
return
|
||||
}
|
||||
if virt.PackageType == models.PackagePuppet {
|
||||
h.serveVirtualPuppet(w, r, *virt, path, proxyBaseURL)
|
||||
return
|
||||
}
|
||||
if virt.PackageType == models.PackageGoProxy {
|
||||
h.serveVirtualGoProxy(w, r, *virt, path)
|
||||
return
|
||||
}
|
||||
|
||||
loc, ok, err := h.virtualEngine.MemberRedirect(r.Context(), *virt, path, r.URL.RawQuery, proxyBaseURL)
|
||||
if err != nil {
|
||||
@@ -184,7 +234,14 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
var body []byte
|
||||
var contentType string
|
||||
if virt.PackageType == models.PackageNPM {
|
||||
abbreviated := strings.Contains(r.Header.Get("Accept"), virtual.NPMAbbreviatedType)
|
||||
body, contentType, err = h.virtualEngine.FetchNPM(r.Context(), *virt, path, proxyBaseURL, abbreviated)
|
||||
} else {
|
||||
body, contentType, err = h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
|
||||
}
|
||||
if errors.Is(err, virtual.ErrNotFound) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
@@ -195,6 +252,9 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if virt.PackageType == models.PackageNPM {
|
||||
setNPMPerHostHeaders(w)
|
||||
}
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
@@ -211,6 +271,34 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if remote.PackageType == models.PackageNPM {
|
||||
if pkg := npmPackageName(path); pkg != "" {
|
||||
h.serveNPMPackument(w, r, remote, pkg)
|
||||
return
|
||||
}
|
||||
}
|
||||
if remote.PackageType == models.PackageGems {
|
||||
if isGemsIndexPath(path) {
|
||||
h.serveLocalGemsIndex(w, r, remote, path)
|
||||
return
|
||||
}
|
||||
if strings.HasPrefix(path, gemsIndexPrefix) || strings.HasPrefix(path, gemsYankedPrefix) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if remote.PackageType == models.PackageAnsible {
|
||||
if ap := ansible.ParseAPIPath(path); ap.Kind != ansible.APIUnknown {
|
||||
h.serveLocalAnsible(w, r, remote, ap)
|
||||
return
|
||||
}
|
||||
if !strings.HasPrefix(path, ansible.ArtifactPrefix) {
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
prov, _ := provider.Get(remote.PackageType)
|
||||
if indexer, ok := prov.(provider.LocalIndexer); ok {
|
||||
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
|
||||
@@ -249,12 +337,97 @@ func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, re
|
||||
io.Copy(w, reader)
|
||||
}
|
||||
|
||||
func scheme(r *http.Request) string {
|
||||
// Scheme returns the client-facing scheme: TLS, else the first
|
||||
// X-Forwarded-Proto value when it is http or https, else http.
|
||||
func Scheme(r *http.Request) string {
|
||||
if r.TLS != nil {
|
||||
return "https"
|
||||
}
|
||||
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
|
||||
fwd, _, _ := strings.Cut(r.Header.Get("X-Forwarded-Proto"), ",")
|
||||
if fwd = strings.ToLower(strings.TrimSpace(fwd)); fwd == "https" {
|
||||
return fwd
|
||||
}
|
||||
return "http"
|
||||
}
|
||||
|
||||
// forwardQuery appends the provider's validated query params to path in a
|
||||
// canonical (sorted, single-valued) form, since the result is the cache key.
|
||||
func forwardQuery(prov provider.Provider, path string, query url.Values) string {
|
||||
f, ok := prov.(provider.QueryForwarder)
|
||||
if !ok {
|
||||
return path
|
||||
}
|
||||
kept := url.Values{}
|
||||
for name, valid := range f.ForwardedQuery(path) {
|
||||
if v, ok := valid(query.Get(name)); ok {
|
||||
kept.Set(name, v)
|
||||
}
|
||||
}
|
||||
if len(kept) == 0 {
|
||||
return path
|
||||
}
|
||||
return path + "?" + kept.Encode()
|
||||
}
|
||||
|
||||
// rewriteServed rewrites a cached body for the requesting host and marks the
|
||||
// response uncacheable by shared caches, since it varies per Host.
|
||||
func rewriteServed(w http.ResponseWriter, r *http.Request, sr provider.ServeRewriter, remote models.Remote, src io.Reader) ([]byte, bool) {
|
||||
base, ok := requestBaseURL(w, r)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
raw, err := io.ReadAll(src)
|
||||
if err != nil {
|
||||
http.Error(w, "bad gateway", http.StatusBadGateway)
|
||||
return nil, false
|
||||
}
|
||||
raw = sr.RewriteServed(raw, remote, base)
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(raw)))
|
||||
return raw, true
|
||||
}
|
||||
|
||||
// requestBaseURL returns the client-facing scheme://host, answering 400 on an
|
||||
// invalid Host.
|
||||
func requestBaseURL(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
if !validHost(r.Host) {
|
||||
http.Error(w, "invalid host header", http.StatusBadRequest)
|
||||
return "", false
|
||||
}
|
||||
return Scheme(r) + "://" + r.Host, true
|
||||
}
|
||||
|
||||
// validHost accepts host[:port] where host is a DNS name or IP literal.
|
||||
func validHost(hostport string) bool {
|
||||
host, port := hostport, ""
|
||||
if h, p, err := net.SplitHostPort(hostport); err == nil {
|
||||
if p == "" {
|
||||
return false
|
||||
}
|
||||
host, port = h, p
|
||||
} else if strings.HasPrefix(hostport, "[") {
|
||||
return strings.HasSuffix(hostport, "]") && net.ParseIP(hostport[1:len(hostport)-1]) != nil
|
||||
}
|
||||
if port != "" {
|
||||
if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if net.ParseIP(host) != nil {
|
||||
return true
|
||||
}
|
||||
if strings.HasPrefix(hostport, "[") {
|
||||
return false
|
||||
}
|
||||
if host == "" || len(host) > 253 {
|
||||
return false
|
||||
}
|
||||
for _, label := range strings.Split(strings.TrimSuffix(host, "."), ".") {
|
||||
if label == "" || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' ||
|
||||
strings.Trim(strings.ToLower(label), "abcdefghijklmnopqrstuvwxyz0123456789-_") != "" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/puppet"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// serveVirtualPuppet serves the merged Forge v3 module and release listings
|
||||
// and 302s release details and downloads to the member owning that release.
|
||||
func (h *ProxyHandler) serveVirtualPuppet(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) {
|
||||
var doc any
|
||||
var err error
|
||||
switch {
|
||||
case path == "v3/releases":
|
||||
q := r.URL.Query()
|
||||
slug := puppet.NormalizeSlug(q.Get("module"))
|
||||
if !puppet.ValidSlug(slug) {
|
||||
forgeError(w, http.StatusBadRequest, "module is required")
|
||||
return
|
||||
}
|
||||
var d *virtual.PuppetDoc
|
||||
d, err = h.virtualEngine.PuppetReleases(r.Context(), virt, slug)
|
||||
releases := []any{}
|
||||
if errors.Is(err, virtual.ErrNotFound) {
|
||||
err = nil
|
||||
} else if err == nil {
|
||||
for _, rel := range d.Releases {
|
||||
releases = append(releases, rel)
|
||||
}
|
||||
}
|
||||
doc = puppet.ReleasesPage(slug, releases, q)
|
||||
case strings.HasPrefix(path, "v3/modules/"):
|
||||
slug := puppet.NormalizeSlug(strings.TrimPrefix(path, "v3/modules/"))
|
||||
if !puppet.ValidSlug(slug) {
|
||||
puppet.NotFound(w)
|
||||
return
|
||||
}
|
||||
var d *virtual.PuppetDoc
|
||||
if d, err = h.virtualEngine.PuppetReleases(r.Context(), virt, slug); err == nil {
|
||||
doc, err = virtual.PuppetModule(d)
|
||||
}
|
||||
default:
|
||||
var loc string
|
||||
loc, err = h.virtualEngine.PuppetRedirect(r.Context(), virt, path, proxyBaseURL)
|
||||
if err == nil {
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("Vary", "Host, X-Forwarded-Proto")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, virtual.ErrNotFound):
|
||||
puppet.NotFound(w)
|
||||
case errors.Is(err, virtual.ErrBadPath):
|
||||
forgeError(w, http.StatusBadRequest, err.Error())
|
||||
case err != nil:
|
||||
slog.Error("virtual puppet fetch failed", "virtual", virt.Name, "path", path, "error", err)
|
||||
forgeError(w, http.StatusBadGateway, "bad gateway")
|
||||
default:
|
||||
w.Header().Set("X-Artifact-Source", "virtual")
|
||||
puppet.WriteJSON(w, doc)
|
||||
}
|
||||
}
|
||||
|
||||
func forgeError(w http.ResponseWriter, code int, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"message": http.StatusText(code), "errors": []string{msg}})
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestPuppetVirtualOverLocals(t *testing.T) {
|
||||
h, srv := npmTestServer(t)
|
||||
h.virtualEngine = virtual.NewEngine(h.db, nil)
|
||||
a, b := npmRepo(t, h, models.PackagePuppet), npmRepo(t, h, models.PackagePuppet)
|
||||
for _, p := range []struct{ repo, version string }{{a, "0.1.0"}, {b, "0.1.0"}, {b, "0.2.0"}} {
|
||||
file := "v3/files/unkin-hello-" + p.version + ".tar.gz"
|
||||
meta := &provider.PuppetMetadata{RepoName: p.repo, FilePath: file, ContentHash: "sha256:00", Slug: "unkin-hello", Version: p.version, MetadataJSON: `{"name":"unkin-hello","version":"` + p.version + `"}`}
|
||||
if err := h.db.InsertPuppetMetadata(context.Background(), meta); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
virt := "v" + a
|
||||
if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackagePuppet, Members: []string{a, b}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
base := srv.URL + "/api/v1/virtual/" + virt
|
||||
|
||||
var page struct {
|
||||
Pagination struct {
|
||||
Total int
|
||||
Next *string
|
||||
}
|
||||
Results []struct {
|
||||
Version string
|
||||
FileURI string `json:"file_uri"`
|
||||
}
|
||||
}
|
||||
getJSON(t, base+"/v3/releases?module=unkin/hello&limit=1", &page)
|
||||
if page.Pagination.Total != 2 || len(page.Results) != 1 || page.Results[0].Version != "0.2.0" || page.Pagination.Next == nil {
|
||||
t.Fatalf("page 1 = %+v", page)
|
||||
}
|
||||
if *page.Pagination.Next != "/v3/releases?limit=1&module=unkin-hello&offset=1" || page.Results[0].FileURI != "/v3/files/unkin-hello-0.2.0.tar.gz" {
|
||||
t.Fatalf("links not relative to the forge root: next %q file_uri %q", *page.Pagination.Next, page.Results[0].FileURI)
|
||||
}
|
||||
getJSON(t, base+*page.Pagination.Next, &page)
|
||||
if len(page.Results) != 1 || page.Results[0].Version != "0.1.0" || page.Pagination.Next != nil {
|
||||
t.Fatalf("page 2 = %+v", page)
|
||||
}
|
||||
|
||||
var mod struct {
|
||||
CurrentRelease struct{ Version string } `json:"current_release"`
|
||||
Releases []struct{ Version string }
|
||||
}
|
||||
getJSON(t, base+"/v3/modules/unkin-hello", &mod)
|
||||
if mod.CurrentRelease.Version != "0.2.0" || len(mod.Releases) != 2 {
|
||||
t.Fatalf("module = %+v", mod)
|
||||
}
|
||||
|
||||
noRedirect := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }}
|
||||
for p, want := range map[string]string{
|
||||
"/v3/files/unkin-hello-0.1.0.tar.gz": "/api/v1/local/" + a + "/v3/files/unkin-hello-0.1.0.tar.gz",
|
||||
"/v3/releases/unkin-hello-0.2.0": "/api/v1/local/" + b + "/v3/releases/unkin-hello-0.2.0",
|
||||
} {
|
||||
resp, err := noRedirect.Get(base + p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != srv.URL+want || resp.Header.Get("Cache-Control") != "no-store" {
|
||||
t.Errorf("%s = %d %q %q, want 302 to %s", p, resp.StatusCode, resp.Header.Get("Location"), resp.Header.Get("Cache-Control"), want)
|
||||
}
|
||||
}
|
||||
|
||||
getJSON(t, base+"/v3/releases?module=unkin-absent", &page)
|
||||
if page.Pagination.Total != 0 || len(page.Results) != 0 {
|
||||
t.Fatalf("absent module = %+v", page)
|
||||
}
|
||||
for p, want := range map[string]int{
|
||||
"/v3/releases": http.StatusBadRequest,
|
||||
"/v3/modules/unkin-absent": http.StatusNotFound,
|
||||
"/v3/files/unkin-hello-0.9.0.tar.gz": http.StatusNotFound,
|
||||
} {
|
||||
if code, body := npmDo(t, http.MethodGet, base+p, ""); code != want || !strings.Contains(body, `"errors"`) {
|
||||
t.Errorf("%s = %d %s, want %d", p, code, body, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestValidHost(t *testing.T) {
|
||||
for _, h := range []string{"aa.example", "aa.example:8443", "artifactapi.k8s.syd1.au.unkin.net", "10.0.0.1:80", "[::1]:8080", "[::1]", "::1", "localhost", "my_host.local"} {
|
||||
if !validHost(h) {
|
||||
t.Errorf("validHost(%q) = false, want true", h)
|
||||
}
|
||||
}
|
||||
for _, h := range []string{"", "evil\".example", "a b", "aa.example:0", "aa.example:99999", "aa.example:x", "aa.example:", ":8080", "-bad.example", "a..b", "[::1", "[nope]:80", "a/b", "a<script>", strings.Repeat("a", 64) + ".example"} {
|
||||
if validHost(h) {
|
||||
t.Errorf("validHost(%q) = true, want false", h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
|
||||
|
||||
const serviceIndex = `{"resources":[{"@id":"https://api.nuget.org/v3-flatcontainer/"}]}`
|
||||
|
||||
func rewrite(t *testing.T, host, proto string) (*httptest.ResponseRecorder, string, bool) {
|
||||
t.Helper()
|
||||
r := httptest.NewRequest(http.MethodGet, "/api/v1/remote/nuget-org/index.json", nil)
|
||||
r.Host = host
|
||||
if proto != "" {
|
||||
r.Header.Set("X-Forwarded-Proto", proto)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
raw, ok := rewriteServed(w, r, &nuget.Provider{}, nugetOrg, strings.NewReader(serviceIndex))
|
||||
return w, string(raw), ok
|
||||
}
|
||||
|
||||
func TestRewriteServedHeaders(t *testing.T) {
|
||||
w, body, ok := rewrite(t, "aa.example:8443", "https")
|
||||
if !ok {
|
||||
t.Fatal("valid request rejected")
|
||||
}
|
||||
if want := `{"resources":[{"@id":"https://aa.example:8443/api/v1/remote/nuget-org/v3-flatcontainer/"}]}`; body != want {
|
||||
t.Errorf("body = %s, want %s", body, want)
|
||||
}
|
||||
if got := w.Header().Get("Cache-Control"); got != "no-store" {
|
||||
t.Errorf("Cache-Control = %q, want no-store", got)
|
||||
}
|
||||
if got := w.Header().Get("Vary"); got != "Host, X-Forwarded-Proto" {
|
||||
t.Errorf("Vary = %q", got)
|
||||
}
|
||||
if got := w.Header().Get("Content-Length"); got != "91" || len(body) != 91 {
|
||||
t.Errorf("Content-Length = %q, body len %d", got, len(body))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteServedIgnoresBogusProto(t *testing.T) {
|
||||
_, plain, _ := rewrite(t, "aa.example", "")
|
||||
_, bogus, _ := rewrite(t, "aa.example", "javascript")
|
||||
if bogus != plain || !strings.Contains(bogus, `"http://aa.example/`) {
|
||||
t.Errorf("bogus X-Forwarded-Proto changed the body: %s", bogus)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteServedRejectsInvalidHost(t *testing.T) {
|
||||
w, _, ok := rewrite(t, `evil"},{"x":"`, "")
|
||||
if ok || w.Code != http.StatusBadRequest {
|
||||
t.Errorf("invalid host: ok=%v code=%d, want rejected with 400", ok, w.Code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package v1
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/npm"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/nuget"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
)
|
||||
|
||||
func TestRoutesRemoteHead(t *testing.T) {
|
||||
r := (&ProxyHandler{}).Routes()
|
||||
for _, m := range []string{http.MethodGet, http.MethodHead} {
|
||||
if !r.Match(chi.NewRouteContext(), m, "/remote/myremote/some/path") {
|
||||
t.Errorf("%s /remote/myremote/some/path not routed", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewritesMutable(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
prov provider.Provider
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"npm packument", &npm.Provider{}, "left-pad", true},
|
||||
{"npm tarball", &npm.Provider{}, "left-pad/-/left-pad-1.3.0.tgz", false},
|
||||
{"nuget index", &nuget.Provider{}, "index.json", true},
|
||||
{"rpm repodata", &rpm.Provider{}, "repodata/repomd.xml", false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
if got := rewritesMutable(tc.prov, tc.path); got != tc.want {
|
||||
t.Errorf("%s: rewritesMutable = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,14 +7,28 @@ import (
|
||||
)
|
||||
|
||||
func TestScheme(t *testing.T) {
|
||||
if got := scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
|
||||
if got := Scheme(&http.Request{TLS: &tls.ConnectionState{}}); got != "https" {
|
||||
t.Errorf("TLS request scheme = %q, want https", got)
|
||||
}
|
||||
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {"https"}}}
|
||||
if got := scheme(r); got != "https" {
|
||||
if got := Scheme(r); got != "https" {
|
||||
t.Errorf("X-Forwarded-Proto scheme = %q, want https", got)
|
||||
}
|
||||
if got := scheme(&http.Request{Header: http.Header{}}); got != "http" {
|
||||
for fwd, want := range map[string]string{
|
||||
"https, http": "https",
|
||||
"http, https": "http",
|
||||
"HTTPS": "https",
|
||||
" https ": "https",
|
||||
"javascript": "http",
|
||||
"ftp": "http",
|
||||
"https;evil": "http",
|
||||
} {
|
||||
r := &http.Request{Header: http.Header{"X-Forwarded-Proto": {fwd}}}
|
||||
if got := Scheme(r); got != want {
|
||||
t.Errorf("X-Forwarded-Proto %q scheme = %q, want %s", fwd, got, want)
|
||||
}
|
||||
}
|
||||
if got := Scheme(&http.Request{Header: http.Header{}}); got != "http" {
|
||||
t.Errorf("default scheme = %q, want http", got)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
@@ -83,7 +84,22 @@ func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, r
|
||||
return
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), r.Body, contentType)
|
||||
var body io.Reader = r.Body
|
||||
if cv, ok := prov.(provider.ContentValidator); ok {
|
||||
f, err := spoolBody(r.Body)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("read upload: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
defer discard(f)
|
||||
if err := cv.ValidateContent(storagePath, f.Name()); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
body = f
|
||||
}
|
||||
|
||||
result, err := h.cas.Store(r.Context(), body, contentType)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -110,6 +126,27 @@ func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, r
|
||||
writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes))
|
||||
}
|
||||
|
||||
func spoolBody(r io.Reader) (*os.File, error) {
|
||||
f, err := os.CreateTemp("", "artifactapi-upload-*")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := io.Copy(f, r); err != nil {
|
||||
discard(f)
|
||||
return nil, err
|
||||
}
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
discard(f)
|
||||
return nil, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func discard(f *os.File) {
|
||||
_ = f.Close()
|
||||
_ = os.Remove(f.Name())
|
||||
}
|
||||
|
||||
func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
|
||||
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, filePath)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// TestLocalUploadContentRejected verifies a ContentValidator provider rejects
|
||||
// bad bytes with 400 before anything is stored.
|
||||
func TestLocalUploadContentRejected(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
ctx := context.Background()
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
const repo = "go-local-validate"
|
||||
if err := db.CreateRemote(ctx, &models.Remote{Name: repo, PackageType: models.PackageGoProxy, RepoType: models.RepoTypeLocal}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
router := chi.NewRouter()
|
||||
router.Route("/remotes/{name}/files", func(r chi.Router) {
|
||||
r.Put("/*", NewLocalHandler(db, nil).Routes().ServeHTTP)
|
||||
})
|
||||
|
||||
cases := map[string]string{
|
||||
"example.unkin.net/hello/@v/v0.1.0.zip": "not a zip",
|
||||
"example.unkin.net/hello/@v/v0.1.0.mod": "module example.unkin.net/other\n",
|
||||
"example.unkin.net/hello/@v/v0.1.0.info": `{"Version":"v9.9.9"}`,
|
||||
"example.unkin.net/Hello/@v/v0.1.0.mod": "module example.unkin.net/Hello\n",
|
||||
}
|
||||
for path, body := range cases {
|
||||
w := httptest.NewRecorder()
|
||||
router.ServeHTTP(w, httptest.NewRequest("PUT", "/remotes/"+repo+"/files/"+path, strings.NewReader(body)))
|
||||
if w.Code != 400 {
|
||||
t.Errorf("%s: status %d, want 400 (%s)", path, w.Code, w.Body.String())
|
||||
}
|
||||
if f, _ := db.GetLocalFile(ctx, repo, path); f != nil {
|
||||
t.Errorf("%s: rejected upload was recorded", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
-10
@@ -67,8 +67,12 @@ func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
}
|
||||
|
||||
// defaultMutableTTL matches the remotes.mutable_ttl column default; a create
|
||||
// that omits mutable_ttl gets it, an explicit 0 still means never expire.
|
||||
const defaultMutableTTL = 3600
|
||||
|
||||
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
var remote models.Remote
|
||||
remote := models.Remote{MutableTTL: defaultMutableTTL}
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
@@ -114,7 +118,15 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
name := chi.URLParam(r, "name")
|
||||
// Capture the current backend before the update so we can tell whether the
|
||||
// remote's base_url (its upstream) changed. A read failure just means we
|
||||
// skip the freshness flush; it must not block the update.
|
||||
existing, err := h.db.GetRemote(r.Context(), name)
|
||||
oldKnown := err == nil
|
||||
var remote models.Remote
|
||||
if oldKnown {
|
||||
remote.MutableTTL = existing.MutableTTL
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
|
||||
http.Error(w, "invalid json", http.StatusBadRequest)
|
||||
return
|
||||
@@ -132,13 +144,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// Capture the current backend before the update so we can tell whether the
|
||||
// remote's base_url (its upstream) changed. A read failure just means we
|
||||
// skip the freshness flush; it must not block the update.
|
||||
oldBaseURL, oldKnown := "", false
|
||||
if existing, err := h.db.GetRemote(r.Context(), name); err == nil {
|
||||
oldBaseURL, oldKnown = existing.BaseURL, true
|
||||
}
|
||||
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
@@ -147,13 +152,13 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
|
||||
// Release / APKINDEX): purge it so the next request re-fetches from the new
|
||||
// upstream instead of serving stale data until TTL expiry. A flush failure
|
||||
// is logged but does not fail the request — the DB update already landed.
|
||||
if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil {
|
||||
if oldKnown && existing.BaseURL != remote.BaseURL && h.cache != nil {
|
||||
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
|
||||
slog.Warn("flush cached metadata after base_url change failed",
|
||||
"remote", name, "error", err)
|
||||
} else {
|
||||
slog.Info("flushed cached metadata after base_url change",
|
||||
"remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL)
|
||||
"remote", name, "old_base_url", existing.BaseURL, "new_base_url", remote.BaseURL)
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, remote)
|
||||
@@ -165,6 +170,11 @@ func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if h.cache != nil {
|
||||
if err := h.cache.FlushRemote(r.Context(), name); err != nil {
|
||||
slog.Warn("flush cached metadata after delete failed", "remote", name, "error", err)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
|
||||
@@ -94,3 +94,28 @@ func TestUpdateFlushFailureStillSucceeds(t *testing.T) {
|
||||
t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls)
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting a remote flushes its cached state so a re-created remote of the
|
||||
// same name starts cold; a flush error does not fail the delete.
|
||||
func TestDeleteFlushesCache(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
for _, ff := range []*fakeFlusher{{}, {err: errors.New("redis down")}} {
|
||||
const name = "rpm-flush-delete"
|
||||
seedRemote(t, db, name, "https://example.com/repo")
|
||||
h := NewRemotesHandler(db, ff, nil).Routes()
|
||||
if c := do(t, h, "DELETE", "/"+name, ""); c != 204 {
|
||||
t.Fatalf("delete = %d, want 204", c)
|
||||
}
|
||||
if len(ff.calls) != 1 || ff.calls[0] != name {
|
||||
t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package v2
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
)
|
||||
|
||||
// An omitted mutable_ttl defaults to 3600 on create and keeps the stored value
|
||||
// on update; an explicit 0 is stored as 0 in both.
|
||||
func TestMutableTTLOmittedVsZero(t *testing.T) {
|
||||
if testDSN == "" {
|
||||
t.Skip("Docker unavailable")
|
||||
}
|
||||
db, err := database.New(testDSN)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
h := NewRemotesHandler(db, nil, nil).Routes()
|
||||
ttl := func(name string) int {
|
||||
t.Helper()
|
||||
r, err := db.GetRemote(context.Background(), name)
|
||||
if err != nil {
|
||||
t.Fatalf("get %s: %v", name, err)
|
||||
}
|
||||
return r.MutableTTL
|
||||
}
|
||||
const body = `"package_type":"rpm","base_url":"https://example.com/repo"`
|
||||
|
||||
steps := []struct {
|
||||
method, path, json string
|
||||
want int
|
||||
}{
|
||||
{"POST", "/", `{"name":"ttl-omit",` + body + `}`, 3600},
|
||||
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":120}`, 120},
|
||||
{"PUT", "/ttl-omit", `{` + body + `}`, 120},
|
||||
{"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":0}`, 0},
|
||||
{"PUT", "/ttl-omit", `{` + body + `}`, 0},
|
||||
}
|
||||
for i, s := range steps {
|
||||
if c := do(t, h, s.method, s.path, s.json); c/100 != 2 {
|
||||
t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c)
|
||||
}
|
||||
if got := ttl("ttl-omit"); got != s.want {
|
||||
t.Errorf("step %d: mutable_ttl = %d, want %d", i, got, s.want)
|
||||
}
|
||||
}
|
||||
|
||||
if c := do(t, h, "POST", "/", `{"name":"ttl-zero",`+body+`,"mutable_ttl":0}`); c != 201 {
|
||||
t.Fatalf("create explicit 0 = %d", c)
|
||||
}
|
||||
if got := ttl("ttl-zero"); got != 0 {
|
||||
t.Errorf("explicit 0 stored as %d", got)
|
||||
}
|
||||
}
|
||||
Vendored
+49
-7
@@ -61,18 +61,54 @@ func TestTTL(t *testing.T) {
|
||||
func TestLock(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
|
||||
if err != nil || !ok {
|
||||
t.Fatalf("first acquire should succeed: %v %v", ok, err)
|
||||
token, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
|
||||
if err != nil || token == "" {
|
||||
t.Fatalf("first acquire should succeed: %q %v", token, err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok {
|
||||
if again, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); again != "" {
|
||||
t.Error("second acquire should fail while held")
|
||||
}
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil {
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "lockpath", "not-the-owner"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok {
|
||||
t.Error("acquire should succeed after release")
|
||||
if again, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); again != "" {
|
||||
t.Error("release with a foreign token must not drop the lock")
|
||||
}
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "lockpath", token); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
|
||||
if next == "" {
|
||||
t.Fatal("acquire should succeed after release")
|
||||
}
|
||||
if next == token {
|
||||
t.Error("tokens must differ between holders")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleReleaseKeepsNextHolder(t *testing.T) {
|
||||
requireRedis(t)
|
||||
ctx := context.Background()
|
||||
stale, err := testRedis.AcquireLock(ctx, "r", "stale", 50*time.Millisecond)
|
||||
if err != nil || stale == "" {
|
||||
t.Fatalf("acquire: %q %v", stale, err)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
fresh, err := testRedis.AcquireLock(ctx, "r", "stale", time.Minute)
|
||||
if err != nil || fresh == "" {
|
||||
t.Fatalf("acquire after expiry: %q %v", fresh, err)
|
||||
}
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "stale", stale); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := testRedis.AcquireLock(ctx, "r", "stale", time.Minute); again != "" {
|
||||
t.Fatal("stale holder's release dropped the second holder's lock")
|
||||
}
|
||||
if err := testRedis.ReleaseLock(ctx, "r", "stale", fresh); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if again, _ := testRedis.AcquireLock(ctx, "r", "stale", time.Minute); again == "" {
|
||||
t.Error("owner release should free the lock")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +122,12 @@ func TestETagAndToken(t *testing.T) {
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` {
|
||||
t.Errorf("etag = %q", v)
|
||||
}
|
||||
if err := testRedis.SetETag(ctx, "r", "p", "", 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v, _ := testRedis.GetETag(ctx, "r", "p"); v != "" {
|
||||
t.Errorf("empty etag should drop the stored one, got %q", v)
|
||||
}
|
||||
|
||||
if v, _ := testRedis.GetToken(ctx, "missing"); v != "" {
|
||||
t.Error("missing token should be empty")
|
||||
|
||||
Vendored
+18
-5
@@ -2,6 +2,7 @@ package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -46,19 +47,31 @@ func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error)
|
||||
return exists > 0, nil
|
||||
}
|
||||
|
||||
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
|
||||
// AcquireLock returns the owner token on success and "" when the lock is held.
|
||||
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (string, error) {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
|
||||
return ok, err
|
||||
token := rand.Text()
|
||||
ok, err := r.client.SetNX(ctx, key, token, ttl).Result()
|
||||
if err != nil || !ok {
|
||||
return "", err
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
|
||||
func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
|
||||
var releaseScript = redis.NewScript(`if redis.call('get',KEYS[1])==ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end`)
|
||||
|
||||
// ReleaseLock deletes the lock only while token still owns it.
|
||||
func (r *Redis) ReleaseLock(ctx context.Context, remote, path, token string) error {
|
||||
key := fmt.Sprintf("lock:%s:%s", remote, path)
|
||||
return r.client.Del(ctx, key).Err()
|
||||
return releaseScript.Run(ctx, r.client, []string{key}, token).Err()
|
||||
}
|
||||
|
||||
// SetETag stores etag for path, or drops the stored one when etag is empty.
|
||||
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
|
||||
key := fmt.Sprintf("etag:%s:%s", remote, path)
|
||||
if etag == "" {
|
||||
return r.client.Del(ctx, key).Err()
|
||||
}
|
||||
return r.client.Set(ctx, key, etag, ttl).Err()
|
||||
}
|
||||
|
||||
|
||||
@@ -207,3 +207,72 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
|
||||
t.Fatal("claimed inside freshness window after a successful sync")
|
||||
}
|
||||
}
|
||||
|
||||
// Deleting a remote drops its sync state, so a recreated remote of the same name
|
||||
// starts clean instead of inheriting the old ETag, sync time and retry state.
|
||||
func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
|
||||
requireDB(t)
|
||||
for _, tc := range []struct {
|
||||
table string
|
||||
pt models.PackageType
|
||||
claim func(name string) (bool, string, error)
|
||||
release func(name string, res provider.SyncResult) error
|
||||
}{
|
||||
{"github_rpm_sync_state", models.PackageGitHubRPM,
|
||||
func(n string) (bool, string, error) {
|
||||
return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
|
||||
},
|
||||
func(n string, res provider.SyncResult) error {
|
||||
return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
|
||||
}},
|
||||
{"github_deb_sync_state", models.PackageGitHubDeb,
|
||||
func(n string) (bool, string, error) {
|
||||
return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
|
||||
},
|
||||
func(n string, res provider.SyncResult) error {
|
||||
return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
|
||||
}},
|
||||
{"github_alpine_sync_state", models.PackageGitHubAlpine,
|
||||
func(n string) (bool, string, error) {
|
||||
return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
|
||||
},
|
||||
func(n string, res provider.SyncResult) error {
|
||||
return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.table, func(t *testing.T) {
|
||||
name := "gh-recreate-" + time.Now().Format("150405.000000")
|
||||
create := func() {
|
||||
t.Helper()
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("create remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
create()
|
||||
if ok, _, err := tc.claim(name); err != nil || !ok {
|
||||
t.Fatalf("claim: ok=%v err=%v", ok, err)
|
||||
}
|
||||
if err := tc.release(name, provider.SyncResult{Etag: `"old"`}); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
if err := testDB.DeleteRemote(ctx(), name); err != nil {
|
||||
t.Fatalf("delete remote: %v", err)
|
||||
}
|
||||
var rows int
|
||||
if err := testDB.Pool.QueryRow(ctx(), `SELECT count(*) FROM `+tc.table+` WHERE remote_name = $1`, name).Scan(&rows); err != nil || rows != 0 {
|
||||
t.Fatalf("sync state left after delete: rows=%d err=%v", rows, err)
|
||||
}
|
||||
|
||||
create()
|
||||
ok, etag, err := tc.claim(name)
|
||||
if err != nil || !ok || etag != "" {
|
||||
t.Fatalf("recreated remote claim ok=%v etag=%q err=%v, want a fresh claim with no etag", ok, etag, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml)
|
||||
VALUES ($1,$2,$3,$4,$5,$6)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at
|
||||
FROM helm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, file_path
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.HelmMetadata
|
||||
for rows.Next() {
|
||||
var m provider.HelmMetadata
|
||||
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to
|
||||
// one module slug (empty slug lists every module).
|
||||
func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at
|
||||
FROM puppet_metadata
|
||||
WHERE repo_name = $1 AND ($2 = '' OR slug = $2)
|
||||
ORDER BY slug, file_path
|
||||
`, repoName, slug)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.PuppetMetadata
|
||||
for rows.Next() {
|
||||
var m provider.PuppetMetadata
|
||||
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/gems"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
|
||||
@@ -0,0 +1,3 @@
|
||||
package all
|
||||
|
||||
import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
|
||||
@@ -372,6 +372,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
// A transiently failed asset fails the scan so the ETag is not advanced and
|
||||
// the retry re-derives it; an invalid package is skipped as no retry fixes it.
|
||||
var failed []error
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
@@ -400,10 +403,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
if !errors.Is(err, provider.ErrInvalidPackage) {
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_alpine: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
continue
|
||||
}
|
||||
slog.Info("github_alpine: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
@@ -415,7 +422,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return newEtag, true, nil
|
||||
return newEtag, true, errors.Join(failed...)
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
@@ -496,7 +503,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
|
||||
return nil, err
|
||||
}
|
||||
if meta.Name == "" || meta.Arch == "" {
|
||||
return nil, errors.New(".PKGINFO missing pkgname/arch")
|
||||
return nil, fmt.Errorf("%w: .PKGINFO missing pkgname/arch", provider.ErrInvalidPackage)
|
||||
}
|
||||
|
||||
meta.RepoName = remote.Name
|
||||
@@ -526,13 +533,13 @@ func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote,
|
||||
}
|
||||
meta, complete, perr := pkginfoFromPrefix(body)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
|
||||
return nil, fmt.Errorf("%w: parse apk .PKGINFO: %w", provider.ErrInvalidPackage, perr)
|
||||
}
|
||||
if complete {
|
||||
return meta, nil
|
||||
}
|
||||
if full || n >= p.headerMax {
|
||||
return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
|
||||
return nil, fmt.Errorf("%w: .PKGINFO not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
|
||||
}
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
@@ -594,7 +601,7 @@ func pkginfoFromPrefix(prefix []byte) (meta *provider.AlpineMetadata, complete b
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
// response holds the whole object.
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
@@ -616,15 +623,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
return provider.ReadPrefix(resp, n)
|
||||
}
|
||||
|
||||
// assetPath is the package's location relative to github.com — the path the
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -88,15 +89,17 @@ type githubFixture struct {
|
||||
notModHit int
|
||||
releaseAuth string
|
||||
assetAuth string
|
||||
assetFail map[string]int // asset filename -> downloads left to answer 500
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
apkBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
apkBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
assetFail: map[string]int{},
|
||||
}
|
||||
f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
|
||||
|
||||
@@ -146,6 +149,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
if f.assetFail[name] > 0 {
|
||||
f.assetFail[name]--
|
||||
f.mu.Unlock()
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
@@ -495,3 +504,13 @@ func readAPKIndex(t *testing.T, gzBytes []byte) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubDeriveAssetMissingPkgnameIsInvalid(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.apkBytes["nameless-1-r0.apk"] = testsupport.MinimalApk("", "1-r0", "x86_64")
|
||||
asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2.3/nameless-1-r0.apk"}
|
||||
_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
|
||||
if !errors.Is(err, provider.ErrInvalidPackage) {
|
||||
t.Fatalf("want ErrInvalidPackage, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
|
||||
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
|
||||
}
|
||||
}
|
||||
|
||||
// A transiently failed asset fails the scan: the ETag is not advanced, a retry
|
||||
// is scheduled, and the retry against the unchanged release list derives it. An
|
||||
// invalid package is skipped instead, since no retry can fix it.
|
||||
func TestSyncerFailedAssetRetry(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
corrupt bool
|
||||
wantRetry bool
|
||||
}{
|
||||
{name: "transient", wantRetry: true},
|
||||
{name: "invalid package", corrupt: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
if tc.corrupt {
|
||||
fx.apkBytes["other-9-r0.apk"] = []byte("not a package")
|
||||
} else {
|
||||
fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
|
||||
fx.assetFail["other-9-r0.apk"] = 1
|
||||
}
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
remote := fx.remote()
|
||||
bg := context.Background()
|
||||
|
||||
s.process(bg, syncJob{remote: remote, prime: true})
|
||||
_, pending := store.retryAt[remote.Name]
|
||||
if pending != tc.wantRetry {
|
||||
t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
|
||||
}
|
||||
if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
|
||||
t.Fatalf("etag saved=%v after first scan", saved)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("healthy asset not served: %d rows", len(rows))
|
||||
}
|
||||
if !tc.wantRetry {
|
||||
return
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
|
||||
store.mu.Unlock()
|
||||
s.process(bg, syncJob{remote: remote})
|
||||
if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
|
||||
t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 2 {
|
||||
t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
// Package ansible proxies an Ansible Galaxy server (v3 collections API):
|
||||
// mutable JSON metadata whose links are rewritten back to the remote, plus
|
||||
// immutable collection tarballs.
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if isArtifact(path) {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
if isArtifact(path) {
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
const maxPageLimit = 100
|
||||
|
||||
// ForwardedQuery keeps limit/offset pagination on metadata paths only.
|
||||
func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator {
|
||||
if isArtifact(path) {
|
||||
return nil
|
||||
}
|
||||
return map[string]provider.QueryValidator{"limit": pageLimit, "offset": pageOffset}
|
||||
}
|
||||
|
||||
func pageOffset(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
return strconv.FormatUint(n, 10), err == nil
|
||||
}
|
||||
|
||||
func pageLimit(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
if err != nil || n == 0 {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatUint(min(n, maxPageLimit), 10), true
|
||||
}
|
||||
|
||||
// RewriteResponse points href, download_url and links.* at the remote. Links
|
||||
// stay root-relative, which ansible-galaxy joins onto the server URL, so the
|
||||
// cached body is independent of the request host.
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
|
||||
dec := json.NewDecoder(bytes.NewReader(body))
|
||||
dec.UseNumber()
|
||||
var doc any
|
||||
if err := dec.Decode(&doc); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
r := rewriter{
|
||||
prefix: provider.RemoteMountPath + url.PathEscape(remote.Name) + "/",
|
||||
base: strings.TrimRight(remote.BaseURL, "/") + "/",
|
||||
}
|
||||
r.walk(doc)
|
||||
var out bytes.Buffer
|
||||
enc := json.NewEncoder(&out)
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out.Bytes(), nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
type rewriter struct{ prefix, base string }
|
||||
|
||||
func (r rewriter) walk(v any) {
|
||||
switch v := v.(type) {
|
||||
case []any:
|
||||
for _, e := range v {
|
||||
r.walk(e)
|
||||
}
|
||||
case map[string]any:
|
||||
for k, e := range v {
|
||||
switch k {
|
||||
case "href", "download_url":
|
||||
v[k] = r.link(e)
|
||||
case "links":
|
||||
if links, ok := e.(map[string]any); ok {
|
||||
for lk, l := range links {
|
||||
links[lk] = r.link(l)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
r.walk(v[k])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r rewriter) link(v any) any {
|
||||
s, ok := v.(string)
|
||||
switch {
|
||||
case !ok:
|
||||
return v
|
||||
case strings.HasPrefix(s, "/api/"):
|
||||
return r.prefix + s[1:]
|
||||
case strings.HasPrefix(s, r.base):
|
||||
return r.prefix + strings.TrimPrefix(s, r.base)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func isArtifact(path string) bool {
|
||||
path, _, _ = strings.Cut(path, "?")
|
||||
return strings.Contains(path, "/artifacts/") && strings.HasSuffix(path, ".tar.gz")
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const tarball = "api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz"
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"api/", provider.Mutable},
|
||||
{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
|
||||
{"api/v3/collections/community/general/versions/?x=.tar.gz", provider.Mutable},
|
||||
{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
|
||||
{"api/v3/collections/ansible/posix/readme.tar.gz", provider.Mutable},
|
||||
{tarball, provider.Immutable},
|
||||
{tarball + "?limit=1", provider.Immutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentTypeIgnoresQuery(t *testing.T) {
|
||||
if got := (&Provider{}).ContentType("api/v3/collections/a/b/versions/?x=.tar.gz"); got != "application/json" {
|
||||
t.Errorf("ContentType = %q, want application/json", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestForwardedQuery(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if got := p.ForwardedQuery(tarball); got != nil {
|
||||
t.Errorf("tarball ForwardedQuery = %v, want nil", got)
|
||||
}
|
||||
q := p.ForwardedQuery("api/v3/collections/a/b/versions/")
|
||||
if len(q) != 2 || q["limit"] == nil || q["offset"] == nil {
|
||||
t.Fatalf("metadata ForwardedQuery keys = %v, want limit and offset", q)
|
||||
}
|
||||
tests := []struct {
|
||||
param, in, want string
|
||||
ok bool
|
||||
}{
|
||||
{"limit", "10", "10", true},
|
||||
{"limit", "007", "7", true},
|
||||
{"limit", "100", "100", true},
|
||||
{"limit", "5000", "100", true},
|
||||
{"limit", "0", "", false},
|
||||
{"limit", "", "", false},
|
||||
{"limit", "-1", "", false},
|
||||
{"limit", "+5", "", false},
|
||||
{"limit", "1e3", "", false},
|
||||
{"limit", "10 ", "", false},
|
||||
{"offset", "0", "0", true},
|
||||
{"offset", "000", "0", true},
|
||||
{"offset", "0100", "100", true},
|
||||
{"offset", "abc", "", false},
|
||||
{"offset", "99999999999999999999", "", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got, ok := q[tt.param](tt.in)
|
||||
if ok != tt.ok || (ok && got != tt.want) {
|
||||
t.Errorf("%s(%q) = %q, %v, want %q, %v", tt.param, tt.in, got, ok, tt.want, tt.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func rewrite(t *testing.T, remote models.Remote, body string) map[string]any {
|
||||
t.Helper()
|
||||
out, err := (&Provider{}).RewriteResponse([]byte(body), remote, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got map[string]any
|
||||
if err := json.Unmarshal(out, &got); err != nil {
|
||||
t.Fatalf("rewritten body is not JSON: %v: %s", err, out)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestRewriteResponse(t *testing.T) {
|
||||
remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
|
||||
got := rewrite(t, remote, `{
|
||||
"links": {"next": "/api/v3/collections/a/b/versions/?limit=10&offset=10", "previous": null},
|
||||
"data": [{
|
||||
"href": "/api/v3/collections/a/b/versions/1.0.0/",
|
||||
"download_url": "https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",
|
||||
"description": "/api/ is where it lives",
|
||||
"docs": "/api/v3/not-a-link/",
|
||||
"metadata": {"repository": "https://github.com/a/b", "homepage": "https://galaxy.ansible.com/ui/"}
|
||||
}]
|
||||
}`)
|
||||
pre := "/api/v1/remote/galaxy/"
|
||||
links := got["links"].(map[string]any)
|
||||
if want := pre + "api/v3/collections/a/b/versions/?limit=10&offset=10"; links["next"] != want {
|
||||
t.Errorf("links.next = %v, want %q", links["next"], want)
|
||||
}
|
||||
if links["previous"] != nil {
|
||||
t.Errorf("links.previous = %v, want null", links["previous"])
|
||||
}
|
||||
item := got["data"].([]any)[0].(map[string]any)
|
||||
if want := pre + "api/v3/collections/a/b/versions/1.0.0/"; item["href"] != want {
|
||||
t.Errorf("href = %v, want %q", item["href"], want)
|
||||
}
|
||||
if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; item["download_url"] != want {
|
||||
t.Errorf("download_url = %v, want %q", item["download_url"], want)
|
||||
}
|
||||
for k, want := range map[string]string{"description": "/api/ is where it lives", "docs": "/api/v3/not-a-link/"} {
|
||||
if item[k] != want {
|
||||
t.Errorf("%s rewritten to %v", k, item[k])
|
||||
}
|
||||
}
|
||||
meta := item["metadata"].(map[string]any)
|
||||
if meta["repository"] != "https://github.com/a/b" || meta["homepage"] != "https://galaxy.ansible.com/ui/" {
|
||||
t.Errorf("unrelated URLs rewritten: %v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseEscapesRemoteName(t *testing.T) {
|
||||
got := rewrite(t, models.Remote{Name: `a"b/c`, BaseURL: "https://g/"}, `{"href":"/api/x/"}`)
|
||||
if want := "/api/v1/remote/a%22b%2Fc/api/x/"; got["href"] != want {
|
||||
t.Errorf("href = %v, want %q", got["href"], want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseKeepsLargeNumbers(t *testing.T) {
|
||||
out, err := (&Provider{}).RewriteResponse([]byte(`{"n":12345678901234567890}`), models.Remote{Name: "g"}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := "{\"n\":12345678901234567890}\n"; string(out) != want {
|
||||
t.Errorf("got %q, want %q", out, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteResponseSkipsNonJSON(t *testing.T) {
|
||||
out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
|
||||
if err != nil || out != nil {
|
||||
t.Errorf("got %q, %v; want nil, nil", out, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,347 @@
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Local ansible repos speak the Galaxy v3 publish and install API. Per version:
|
||||
// artifacts/<ns>-<name>-<version>.tar.gz collection tarball (immutable)
|
||||
// _index/<ns>/<name>/<version> Entry JSON; gates visibility
|
||||
|
||||
const (
|
||||
IndexPrefix = "_index/"
|
||||
ArtifactPrefix = "artifacts/"
|
||||
PublishPath = "api/v3/artifacts/collections/"
|
||||
)
|
||||
|
||||
var (
|
||||
NameRe = regexp.MustCompile(`^[a-z][a-z0-9_]{0,63}$`)
|
||||
VersionRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$`)
|
||||
)
|
||||
|
||||
var (
|
||||
MaxManifestBytes int64 = 1 << 20
|
||||
MaxScanBytes int64 = 512 << 20
|
||||
)
|
||||
|
||||
// Entry is one published collection version.
|
||||
type Entry struct {
|
||||
Namespace string `json:"namespace"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Size int64 `json:"size"`
|
||||
Metadata json.RawMessage `json:"metadata"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func (e Entry) Filename() string { return e.Namespace + "-" + e.Name + "-" + e.Version + ".tar.gz" }
|
||||
|
||||
// IndexPath is the entry's hidden index file.
|
||||
func (e Entry) IndexPath() string { return IndexPrefix + e.Namespace + "/" + e.Name + "/" + e.Version }
|
||||
|
||||
// TaskID names the import task returned on publish.
|
||||
func (e Entry) TaskID() string { return e.Namespace + "-" + e.Name + "-" + e.Version }
|
||||
|
||||
// ParseCollection reads collection_info from a collection tarball's top-level
|
||||
// MANIFEST.json. Every member is checked, so a tarball whose paths or links
|
||||
// escape the collection root is rejected even after the manifest.
|
||||
func ParseCollection(r io.Reader) (*Entry, error) {
|
||||
zr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("not a gzip tarball: %w", err)
|
||||
}
|
||||
scan := &io.LimitedReader{R: zr, N: MaxScanBytes}
|
||||
tr := tar.NewReader(scan)
|
||||
var e *Entry
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if scan.N <= 0 {
|
||||
return nil, errors.New("collection exceeds decompressed size limit")
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read tarball: %w", err)
|
||||
}
|
||||
name, err := memberPath(hdr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if name != "MANIFEST.json" || hdr.Typeflag != tar.TypeReg {
|
||||
continue
|
||||
}
|
||||
if e != nil {
|
||||
return nil, errors.New("duplicate MANIFEST.json")
|
||||
}
|
||||
if e, err = parseManifest(io.LimitReader(tr, MaxManifestBytes+1)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if e == nil {
|
||||
return nil, errors.New("no MANIFEST.json in collection tarball")
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// memberPath returns hdr's cleaned name, rejecting absolute or escaping
|
||||
// paths and links, and anything but files, directories and symlinks.
|
||||
func memberPath(hdr *tar.Header) (string, error) {
|
||||
name := path.Clean(strings.TrimPrefix(hdr.Name, "./"))
|
||||
if escapes(name) {
|
||||
return "", fmt.Errorf("tarball member %q escapes the collection root", hdr.Name)
|
||||
}
|
||||
switch hdr.Typeflag {
|
||||
case tar.TypeReg, tar.TypeDir:
|
||||
case tar.TypeSymlink:
|
||||
if path.IsAbs(hdr.Linkname) || escapes(path.Join(path.Dir(name), hdr.Linkname)) {
|
||||
return "", fmt.Errorf("tarball symlink %q escapes the collection root", hdr.Name)
|
||||
}
|
||||
default:
|
||||
return "", fmt.Errorf("tarball member %q has unsupported type %q", hdr.Name, hdr.Typeflag)
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
func escapes(p string) bool {
|
||||
return path.IsAbs(p) || p == ".." || strings.HasPrefix(p, "../") || strings.Contains(p, "\\")
|
||||
}
|
||||
|
||||
func parseManifest(r io.Reader) (*Entry, error) {
|
||||
raw, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read MANIFEST.json: %w", err)
|
||||
}
|
||||
if int64(len(raw)) > MaxManifestBytes {
|
||||
return nil, fmt.Errorf("MANIFEST.json exceeds %d bytes", MaxManifestBytes)
|
||||
}
|
||||
var doc struct {
|
||||
CollectionInfo map[string]json.RawMessage `json:"collection_info"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &doc); err != nil || doc.CollectionInfo == nil {
|
||||
return nil, errors.New("MANIFEST.json has no collection_info")
|
||||
}
|
||||
info := doc.CollectionInfo
|
||||
e := &Entry{}
|
||||
for k, dst := range map[string]*string{"namespace": &e.Namespace, "name": &e.Name, "version": &e.Version} {
|
||||
if err := json.Unmarshal(info[k], dst); err != nil {
|
||||
return nil, fmt.Errorf("collection_info.%s is not a string", k)
|
||||
}
|
||||
}
|
||||
if !NameRe.MatchString(e.Namespace) || !NameRe.MatchString(e.Name) || !VersionRe.MatchString(e.Version) {
|
||||
return nil, fmt.Errorf("invalid collection %q.%q version %q", e.Namespace, e.Name, e.Version)
|
||||
}
|
||||
deps := map[string]string{}
|
||||
if d := info["dependencies"]; len(d) > 0 && string(d) != "null" {
|
||||
if err := json.Unmarshal(d, &deps); err != nil {
|
||||
return nil, errors.New("collection_info.dependencies must map collection names to version ranges")
|
||||
}
|
||||
}
|
||||
for k := range deps {
|
||||
ns, n, ok := strings.Cut(k, ".")
|
||||
if !ok || !NameRe.MatchString(ns) || !NameRe.MatchString(n) {
|
||||
return nil, fmt.Errorf("invalid dependency %q", k)
|
||||
}
|
||||
}
|
||||
info["dependencies"], _ = json.Marshal(deps)
|
||||
if e.Metadata, err = json.Marshal(info); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// APIPath is a parsed Galaxy v3 request path below a repo.
|
||||
type APIPath struct {
|
||||
Kind APIKind
|
||||
Namespace, Name, Version string
|
||||
TaskID string
|
||||
}
|
||||
|
||||
type APIKind int
|
||||
|
||||
const (
|
||||
APIUnknown APIKind = iota
|
||||
APIRoot
|
||||
APICollection
|
||||
APIVersions
|
||||
APIVersion
|
||||
APIImport
|
||||
)
|
||||
|
||||
// ParseAPIPath parses api/, api/v3/collections/<ns>/<name>/[versions/[<v>/]]
|
||||
// and api/v3/imports/collections/<id>/, with or without trailing slashes.
|
||||
func ParseAPIPath(p string) APIPath {
|
||||
p = strings.TrimSuffix(p, "/")
|
||||
if p == "api" {
|
||||
return APIPath{Kind: APIRoot}
|
||||
}
|
||||
if id, ok := strings.CutPrefix(p, "api/v3/imports/collections/"); ok && id != "" && !strings.Contains(id, "/") {
|
||||
return APIPath{Kind: APIImport, TaskID: id}
|
||||
}
|
||||
rest, ok := strings.CutPrefix(p, "api/v3/collections/")
|
||||
if !ok {
|
||||
return APIPath{}
|
||||
}
|
||||
segs := strings.Split(rest, "/")
|
||||
if len(segs) < 2 || !NameRe.MatchString(segs[0]) || !NameRe.MatchString(segs[1]) {
|
||||
return APIPath{}
|
||||
}
|
||||
ap := APIPath{Namespace: segs[0], Name: segs[1]}
|
||||
switch {
|
||||
case len(segs) == 2:
|
||||
ap.Kind = APICollection
|
||||
case len(segs) == 3 && segs[2] == "versions":
|
||||
ap.Kind = APIVersions
|
||||
case len(segs) == 4 && segs[2] == "versions" && VersionRe.MatchString(segs[3]):
|
||||
ap.Kind, ap.Version = APIVersion, segs[3]
|
||||
default:
|
||||
return APIPath{}
|
||||
}
|
||||
return ap
|
||||
}
|
||||
|
||||
// Root is the API root ansible-galaxy discovers.
|
||||
func Root() map[string]any {
|
||||
return map[string]any{"available_versions": map[string]string{"v3": "v3/"}}
|
||||
}
|
||||
|
||||
func CollectionPath(ns, name string) string { return "/api/v3/collections/" + ns + "/" + name + "/" }
|
||||
|
||||
func VersionPath(ns, name, version string) string {
|
||||
return CollectionPath(ns, name) + "versions/" + url.PathEscape(version) + "/"
|
||||
}
|
||||
|
||||
// VersionRef is one entry of a versions list.
|
||||
type VersionRef struct {
|
||||
Version string `json:"version"`
|
||||
CreatedAt string `json:"created_at,omitempty"`
|
||||
UpdatedAt string `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
func (e Entry) Ref() VersionRef {
|
||||
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
|
||||
return VersionRef{Version: e.Version, CreatedAt: t, UpdatedAt: t}
|
||||
}
|
||||
|
||||
// CollectionDoc describes a collection from its versions; base is the repo
|
||||
// URL the hrefs are built on.
|
||||
func CollectionDoc(base, ns, name string, refs []VersionRef) map[string]any {
|
||||
var created, updated time.Time
|
||||
for _, r := range refs {
|
||||
if t, err := time.Parse(time.RFC3339Nano, r.CreatedAt); err == nil && (created.IsZero() || t.Before(created)) {
|
||||
created = t
|
||||
}
|
||||
if t, err := time.Parse(time.RFC3339Nano, r.UpdatedAt); err == nil && t.After(updated) {
|
||||
updated = t
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"href": base + CollectionPath(ns, name),
|
||||
"namespace": ns,
|
||||
"name": name,
|
||||
"deprecated": false,
|
||||
"versions_url": base + CollectionPath(ns, name) + "versions/",
|
||||
"created_at": formatTime(created),
|
||||
"updated_at": formatTime(updated),
|
||||
}
|
||||
}
|
||||
|
||||
func formatTime(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Page reads limit/offset as the remote forwards them, defaulting to 100/0.
|
||||
func Page(q url.Values) (limit, offset int) {
|
||||
limit, offset = maxPageLimit, 0
|
||||
if v, ok := pageLimit(q.Get("limit")); ok {
|
||||
limit, _ = strconv.Atoi(v)
|
||||
}
|
||||
if v, ok := pageOffset(q.Get("offset")); ok {
|
||||
offset, _ = strconv.Atoi(v)
|
||||
}
|
||||
return limit, offset
|
||||
}
|
||||
|
||||
// VersionsPage is one page of refs in Galaxy v3 list form. Links are
|
||||
// root-relative like Galaxy's, since ansible-galaxy splices them onto the
|
||||
// server URL.
|
||||
func VersionsPage(base, ns, name string, refs []VersionRef, limit, offset int) map[string]any {
|
||||
list := rootPath(base) + CollectionPath(ns, name) + "versions/"
|
||||
link := func(off int) any {
|
||||
return list + "?" + url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}.Encode()
|
||||
}
|
||||
offset = min(offset, len(refs))
|
||||
var previous, next any
|
||||
if offset > 0 {
|
||||
previous = link(max(offset-limit, 0))
|
||||
}
|
||||
if offset+limit < len(refs) {
|
||||
next = link(offset + limit)
|
||||
}
|
||||
data := []map[string]any{}
|
||||
for _, r := range refs[offset:min(offset+limit, len(refs))] {
|
||||
data = append(data, map[string]any{
|
||||
"version": r.Version,
|
||||
"href": base + VersionPath(ns, name, r.Version),
|
||||
"created_at": r.CreatedAt,
|
||||
"updated_at": r.UpdatedAt,
|
||||
"marks": []string{},
|
||||
})
|
||||
}
|
||||
return map[string]any{
|
||||
"meta": map[string]any{"count": len(refs)},
|
||||
"links": map[string]any{"first": link(0), "previous": previous, "next": next, "last": link(max(len(refs)-1, 0) / limit * limit)},
|
||||
"data": data,
|
||||
}
|
||||
}
|
||||
|
||||
// rootPath drops the scheme and host from base.
|
||||
func rootPath(base string) string {
|
||||
if u, err := url.Parse(base); err == nil {
|
||||
return u.EscapedPath()
|
||||
}
|
||||
return base
|
||||
}
|
||||
|
||||
// VersionDoc is a version's detail; the tarball is served from the repo at
|
||||
// artifacts/<filename>.
|
||||
func VersionDoc(base string, e Entry) map[string]any {
|
||||
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
|
||||
return map[string]any{
|
||||
"href": base + VersionPath(e.Namespace, e.Name, e.Version),
|
||||
"namespace": map[string]string{"name": e.Namespace},
|
||||
"collection": map[string]string{"name": e.Name},
|
||||
"name": e.Name,
|
||||
"version": e.Version,
|
||||
"download_url": base + "/" + ArtifactPrefix + e.Filename(),
|
||||
"artifact": map[string]any{"filename": e.Filename(), "sha256": e.SHA256, "size": e.Size},
|
||||
"metadata": e.Metadata,
|
||||
"signatures": []string{},
|
||||
"created_at": t,
|
||||
"updated_at": t,
|
||||
}
|
||||
}
|
||||
|
||||
// ImportTask is a finished import: publishing completes synchronously.
|
||||
func ImportTask(e Entry) map[string]any {
|
||||
t := e.CreatedAt.UTC().Format(time.RFC3339Nano)
|
||||
return map[string]any{
|
||||
"id": e.TaskID(), "state": "completed", "created_at": t, "started_at": t, "finished_at": t,
|
||||
"messages": []any{}, "error": nil,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package ansible
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type member struct {
|
||||
hdr tar.Header
|
||||
body string
|
||||
}
|
||||
|
||||
func collectionTar(t *testing.T, members ...member) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for _, m := range members {
|
||||
h := m.hdr
|
||||
if h.Typeflag == 0 {
|
||||
h.Typeflag = tar.TypeReg
|
||||
}
|
||||
if h.Typeflag == tar.TypeReg {
|
||||
h.Size = int64(len(m.body))
|
||||
}
|
||||
h.Mode = 0o644
|
||||
if err := tw.WriteHeader(&h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = tw.Write([]byte(m.body))
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func manifest(ns, name, version, deps string) member {
|
||||
return member{tar.Header{Name: "MANIFEST.json"}, `{"collection_info":{"namespace":"` + ns + `","name":"` + name +
|
||||
`","version":"` + version + `","authors":["a"],"dependencies":` + deps + `}}`}
|
||||
}
|
||||
|
||||
func TestParseCollection(t *testing.T) {
|
||||
e, err := ParseCollection(bytes.NewReader(collectionTar(t,
|
||||
member{tar.Header{Name: "plugins/", Typeflag: tar.TypeDir}, ""},
|
||||
member{tar.Header{Name: "plugins/modules/hello.py"}, "x"},
|
||||
member{tar.Header{Name: "docs/link", Typeflag: tar.TypeSymlink, Linkname: "../plugins/modules/hello.py"}, ""},
|
||||
manifest("unkin", "e2e_hello", "1.0.0-rc.1+b1", `{"ansible.posix":">=1.0.0"}`),
|
||||
)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Namespace != "unkin" || e.Name != "e2e_hello" || e.Version != "1.0.0-rc.1+b1" || e.Filename() != "unkin-e2e_hello-1.0.0-rc.1+b1.tar.gz" {
|
||||
t.Fatalf("entry = %+v", e)
|
||||
}
|
||||
var md struct {
|
||||
Authors []string `json:"authors"`
|
||||
Dependencies map[string]string `json:"dependencies"`
|
||||
}
|
||||
if err := json.Unmarshal(e.Metadata, &md); err != nil || md.Dependencies["ansible.posix"] != ">=1.0.0" || len(md.Authors) != 1 {
|
||||
t.Fatalf("metadata = %s (%v)", e.Metadata, err)
|
||||
}
|
||||
|
||||
e, err = ParseCollection(bytes.NewReader(collectionTar(t, manifest("a", "b", "1.0.0", "null"))))
|
||||
if err != nil || !strings.Contains(string(e.Metadata), `"dependencies":{}`) {
|
||||
t.Fatalf("null dependencies = %s (%v), want {}", e.Metadata, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCollectionRejects(t *testing.T) {
|
||||
ok := manifest("a", "b", "1.0.0", "{}")
|
||||
tests := []struct {
|
||||
name string
|
||||
members []member
|
||||
want string
|
||||
}{
|
||||
{"no manifest", []member{{tar.Header{Name: "README.md"}, "x"}}, "no MANIFEST.json"},
|
||||
{"nested manifest only", []member{{tar.Header{Name: "x/MANIFEST.json"}, ok.body}}, "no MANIFEST.json"},
|
||||
{"dotdot", []member{ok, {tar.Header{Name: "../evil"}, "x"}}, "escapes"},
|
||||
{"dotdot after clean", []member{ok, {tar.Header{Name: "a/../../evil"}, "x"}}, "escapes"},
|
||||
{"absolute", []member{ok, {tar.Header{Name: "/etc/passwd"}, "x"}}, "escapes"},
|
||||
{"symlink escape", []member{ok, {tar.Header{Name: "a/l", Typeflag: tar.TypeSymlink, Linkname: "../../x"}, ""}}, "symlink"},
|
||||
{"absolute symlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeSymlink, Linkname: "/etc"}, ""}}, "symlink"},
|
||||
{"hardlink", []member{ok, {tar.Header{Name: "l", Typeflag: tar.TypeLink, Linkname: "MANIFEST.json"}, ""}}, "unsupported type"},
|
||||
{"duplicate manifest", []member{ok, ok}, "duplicate"},
|
||||
{"bad namespace", []member{manifest("Bad", "b", "1.0.0", "{}")}, "invalid collection"},
|
||||
{"bad version", []member{manifest("a", "b", "1.0", "{}")}, "invalid collection"},
|
||||
{"bad dependency", []member{manifest("a", "b", "1.0.0", `{"nodot":"*"}`)}, "invalid dependency"},
|
||||
{"list dependencies", []member{manifest("a", "b", "1.0.0", `["x.y"]`)}, "dependencies must map"},
|
||||
{"huge manifest", []member{{tar.Header{Name: "MANIFEST.json"}, strings.Repeat(" ", int(MaxManifestBytes)+1)}}, "exceeds"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := ParseCollection(bytes.NewReader(collectionTar(t, tt.members...)))
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) {
|
||||
t.Fatalf("err = %v, want %q", err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := ParseCollection(strings.NewReader("not gzip")); err == nil {
|
||||
t.Error("non-gzip input accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCollectionScanLimit(t *testing.T) {
|
||||
old := MaxScanBytes
|
||||
MaxScanBytes = 4096
|
||||
defer func() { MaxScanBytes = old }()
|
||||
_, err := ParseCollection(bytes.NewReader(collectionTar(t,
|
||||
member{tar.Header{Name: "big"}, strings.Repeat("x", 8192)},
|
||||
manifest("a", "b", "1.0.0", "{}"),
|
||||
)))
|
||||
if err == nil || !strings.Contains(err.Error(), "decompressed size limit") {
|
||||
t.Fatalf("err = %v, want decompressed size limit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAPIPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
in string
|
||||
want APIPath
|
||||
}{
|
||||
{"api", APIPath{Kind: APIRoot}},
|
||||
{"api/", APIPath{Kind: APIRoot}},
|
||||
{"api/v3/collections/unkin/e2e_hello/", APIPath{Kind: APICollection, Namespace: "unkin", Name: "e2e_hello"}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions", APIPath{Kind: APIVersions, Namespace: "unkin", Name: "e2e_hello"}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions/1.0.0/", APIPath{Kind: APIVersion, Namespace: "unkin", Name: "e2e_hello", Version: "1.0.0"}},
|
||||
{"api/v3/imports/collections/unkin-e2e_hello-1.0.0/", APIPath{Kind: APIImport, TaskID: "unkin-e2e_hello-1.0.0"}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions/1.0/", APIPath{}},
|
||||
{"api/v3/collections/../x/", APIPath{}},
|
||||
{"api/v3/collections/unkin/e2e_hello/versions/1.0.0/extra/", APIPath{}},
|
||||
{"artifacts/unkin-e2e_hello-1.0.0.tar.gz", APIPath{}},
|
||||
{"api/v3/imports/collections/a/b/", APIPath{}},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := ParseAPIPath(tt.in); got != tt.want {
|
||||
t.Errorf("ParseAPIPath(%q) = %+v, want %+v", tt.in, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionsPage(t *testing.T) {
|
||||
refs := []VersionRef{{Version: "3.0.0"}, {Version: "2.0.0"}, {Version: "1.0.0"}}
|
||||
list := "/r/api/v3/collections/a/b/versions/"
|
||||
links := func(p map[string]any) map[string]any { return p["links"].(map[string]any) }
|
||||
versions := func(p map[string]any) (out []string) {
|
||||
for _, d := range p["data"].([]map[string]any) {
|
||||
out = append(out, d["version"].(string))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
p := VersionsPage("https://h/r", "a", "b", refs, 2, 0)
|
||||
if got := versions(p); strings.Join(got, ",") != "3.0.0,2.0.0" {
|
||||
t.Fatalf("page 1 = %v", got)
|
||||
}
|
||||
if l := links(p); l["previous"] != nil || l["next"] != list+"?limit=2&offset=2" || l["last"] != list+"?limit=2&offset=2" {
|
||||
t.Fatalf("page 1 links = %v", l)
|
||||
}
|
||||
if p["meta"].(map[string]any)["count"] != 3 {
|
||||
t.Fatalf("count = %v", p["meta"])
|
||||
}
|
||||
|
||||
p = VersionsPage("https://h/r", "a", "b", refs, 2, 2)
|
||||
if got := versions(p); strings.Join(got, ",") != "1.0.0" {
|
||||
t.Fatalf("page 2 = %v", got)
|
||||
}
|
||||
if l := links(p); l["next"] != nil || l["previous"] != list+"?limit=2&offset=0" {
|
||||
t.Fatalf("page 2 links = %v", l)
|
||||
}
|
||||
|
||||
if got := versions(VersionsPage("", "a", "b", refs, 2, 99)); len(got) != 0 {
|
||||
t.Fatalf("offset past end = %v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestVersionsPageFollow pages through more than 100 versions the way
|
||||
// ansible-galaxy does: old clients splice next onto the versions URL in place
|
||||
// of its path, new ones urljoin it onto the server.
|
||||
func TestVersionsPageFollow(t *testing.T) {
|
||||
var refs []VersionRef
|
||||
for i := 104; i >= 0; i-- {
|
||||
refs = append(refs, VersionRef{Version: fmt.Sprintf("1.0.%d", i)})
|
||||
}
|
||||
server := "https://h:8443/api/v1/local/r%2Bx/"
|
||||
base := strings.TrimSuffix(server, "/")
|
||||
versionsURL := server + "api/v3/collections/a/b/versions/"
|
||||
for _, follow := range map[string]func(string) string{
|
||||
"replace": func(link string) string {
|
||||
u, _ := url.Parse(versionsURL)
|
||||
return strings.Replace(versionsURL, u.EscapedPath(), link, 1)
|
||||
},
|
||||
"urljoin": func(link string) string {
|
||||
s, _ := url.Parse(server)
|
||||
l, _ := url.Parse(link)
|
||||
return s.ResolveReference(l).String()
|
||||
},
|
||||
} {
|
||||
var got []string
|
||||
next := versionsURL + "?limit=100"
|
||||
for pages := 0; next != ""; pages++ {
|
||||
if pages > 3 {
|
||||
t.Fatal("pagination does not terminate")
|
||||
}
|
||||
u, err := url.Parse(next)
|
||||
if err != nil || u.Host != "h:8443" || !strings.HasPrefix(u.EscapedPath(), "/api/v1/local/r%2Bx/api/v3/") {
|
||||
t.Fatalf("followed link %q is not on the server", next)
|
||||
}
|
||||
limit, offset := Page(u.Query())
|
||||
p := VersionsPage(base, "a", "b", refs, limit, offset)
|
||||
for _, d := range p["data"].([]map[string]any) {
|
||||
got = append(got, d["version"].(string))
|
||||
}
|
||||
next = ""
|
||||
if l, ok := p["links"].(map[string]any)["next"].(string); ok {
|
||||
next = follow(l)
|
||||
}
|
||||
}
|
||||
if len(got) != 105 || got[0] != "1.0.104" || got[104] != "1.0.0" {
|
||||
t.Fatalf("collected %d versions, first %v", len(got), got[:1])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCollectionDocTimes(t *testing.T) {
|
||||
d := CollectionDoc("", "a", "b", []VersionRef{
|
||||
{CreatedAt: "2026-01-01T00:00:05Z", UpdatedAt: "2026-01-01T00:00:05Z"},
|
||||
{CreatedAt: "2026-01-01T00:00:05.5Z", UpdatedAt: "2026-01-01T00:00:05.5Z"},
|
||||
{CreatedAt: "2026-01-01T00:00:04.9Z", UpdatedAt: "2026-01-01T00:00:04.9Z"},
|
||||
})
|
||||
if d["created_at"] != "2026-01-01T00:00:04.9Z" || d["updated_at"] != "2026-01-01T00:00:05.5Z" {
|
||||
t.Fatalf("created/updated = %v/%v", d["created_at"], d["updated_at"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPage(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
q string
|
||||
limit, offset int
|
||||
}{
|
||||
{"", 100, 0},
|
||||
{"limit=5&offset=10", 5, 10},
|
||||
{"limit=5000", 100, 0},
|
||||
{"limit=0&offset=-1", 100, 0},
|
||||
} {
|
||||
q, _ := url.ParseQuery(tt.q)
|
||||
if l, o := Page(q); l != tt.limit || o != tt.offset {
|
||||
t.Errorf("Page(%q) = %d,%d want %d,%d", tt.q, l, o, tt.limit, tt.offset)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package provider
|
||||
|
||||
// ContentValidator checks an upload's bytes (spooled to localFile) before it is stored.
|
||||
type ContentValidator interface {
|
||||
ValidateContent(storagePath, localFile string) error
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
)
|
||||
|
||||
type FileStore interface {
|
||||
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
|
||||
ListPackages(ctx context.Context, repoName string) ([]string, error)
|
||||
}
|
||||
|
||||
type BlobReader interface {
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package provider
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type LocalIndexer interface {
|
||||
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
|
||||
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
package provider
|
||||
|
||||
type LocalUploader interface {
|
||||
ValidateUpload(filePath string) (storagePath, contentType string, err error)
|
||||
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
|
||||
// feeds generated repodata) after a local file is removed.
|
||||
type PostDeleteHook interface {
|
||||
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type PostUploadHook interface {
|
||||
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package provider
|
||||
|
||||
// QueryForwarder is a remote provider whose upstream answers depend on the
|
||||
// query string (e.g. paginated JSON APIs). Only the named params for path are
|
||||
// forwarded upstream and kept in the cache key, each as the canonical value
|
||||
// its validator returns; a value the validator rejects is dropped.
|
||||
type QueryForwarder interface {
|
||||
ForwardedQuery(path string) map[string]QueryValidator
|
||||
}
|
||||
|
||||
// QueryValidator canonicalises a raw query value, reporting false to drop it.
|
||||
type QueryValidator func(v string) (string, bool)
|
||||
@@ -0,0 +1,25 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// RemoteServer lets a remote provider fully answer a request itself instead of
|
||||
// going through the byte-proxy engine. It is the remote-side analog of
|
||||
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
|
||||
// repodata from derived per-asset metadata and to redirect package downloads to
|
||||
// a backend remote, without ever precaching the packages. Returning false lets
|
||||
// the normal proxy path take over.
|
||||
type RemoteServer interface {
|
||||
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
|
||||
}
|
||||
|
||||
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
|
||||
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
|
||||
type RemoteMetadataStore interface {
|
||||
RPMMetadataReader
|
||||
MetadataStore
|
||||
MetadataDeleter
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package provider
|
||||
|
||||
// ResponseRewriter marks providers whose RewriteResponse changes mutable
|
||||
// bodies, so the served length is only known after a fetch.
|
||||
type ResponseRewriter interface {
|
||||
RewritesResponse()
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ServeRewriter rewrites a cached mutable remote response per request, for
|
||||
// indexes whose URLs must name the requesting host and so cannot be rewritten
|
||||
// before caching.
|
||||
type ServeRewriter interface {
|
||||
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
|
||||
}
|
||||
@@ -333,6 +333,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
// A transiently failed asset fails the scan so the ETag is not advanced and
|
||||
// the retry re-derives it; an invalid package is skipped as no retry fixes it.
|
||||
var failed []error
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
@@ -361,10 +364,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_deb: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
if !errors.Is(err, provider.ErrInvalidPackage) {
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := store.InsertDebMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_deb: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
continue
|
||||
}
|
||||
slog.Info("github_deb: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
|
||||
@@ -376,7 +383,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return newEtag, true, nil
|
||||
return newEtag, true, errors.Join(failed...)
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
@@ -468,7 +475,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
|
||||
Size: asset.Size,
|
||||
}
|
||||
if meta.Name == "" {
|
||||
return nil, errors.New("control missing Package field")
|
||||
return nil, fmt.Errorf("%w: control missing Package field", provider.ErrInvalidPackage)
|
||||
}
|
||||
|
||||
// The Packages SHA256 must be the sha256 of the whole .deb. Prefer GitHub's
|
||||
@@ -503,13 +510,13 @@ func (p *GitHubProvider) fetchControl(ctx context.Context, remote models.Remote,
|
||||
}
|
||||
control, complete, perr := controlFromPrefix(body)
|
||||
if perr != nil {
|
||||
return "", fmt.Errorf("parse deb control: %w", perr)
|
||||
return "", fmt.Errorf("%w: parse deb control: %w", provider.ErrInvalidPackage, perr)
|
||||
}
|
||||
if complete {
|
||||
return control, nil
|
||||
}
|
||||
if full || n >= p.headerMax {
|
||||
return "", fmt.Errorf("control.tar not found within %d bytes of %s", n, downloadURL)
|
||||
return "", fmt.Errorf("%w: control.tar not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
|
||||
}
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
@@ -567,7 +574,7 @@ func controlFromPrefix(prefix []byte) (control string, complete bool, err error)
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
// response holds the whole object.
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
@@ -589,15 +596,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
return provider.ReadPrefix(resp, n)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
|
||||
@@ -620,7 +621,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -81,15 +82,17 @@ type githubFixture struct {
|
||||
notModHit int
|
||||
releaseAuth string
|
||||
assetAuth string
|
||||
assetFail map[string]int // asset filename -> downloads left to answer 500
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
debBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
debBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
assetFail: map[string]int{},
|
||||
}
|
||||
f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
|
||||
|
||||
@@ -139,6 +142,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
if f.assetFail[name] > 0 {
|
||||
f.assetFail[name]--
|
||||
f.mu.Unlock()
|
||||
http.Error(w, "boom", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
@@ -460,3 +469,13 @@ func TestGitHubAssetPatternFilter(t *testing.T) {
|
||||
t.Fatalf("pattern filter failed, rows=%+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubDeriveAssetMissingPackageIsInvalid(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.debBytes["nameless_1_amd64.deb"] = testsupport.MinimalDeb("", "1", "amd64")
|
||||
asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2-3/nameless_1_amd64.deb", Digest: "sha256:00"}
|
||||
_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
|
||||
if !errors.Is(err, provider.ErrInvalidPackage) {
|
||||
t.Fatalf("want ErrInvalidPackage, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -308,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
|
||||
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
|
||||
}
|
||||
}
|
||||
|
||||
// A transiently failed asset fails the scan: the ETag is not advanced, a retry
|
||||
// is scheduled, and the retry against the unchanged release list derives it. An
|
||||
// invalid package is skipped instead, since no retry can fix it.
|
||||
func TestSyncerFailedAssetRetry(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
corrupt bool
|
||||
wantRetry bool
|
||||
}{
|
||||
{name: "transient", wantRetry: true},
|
||||
{name: "invalid package", corrupt: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
if tc.corrupt {
|
||||
fx.debBytes["other_9-9_arm64.deb"] = []byte("not a package")
|
||||
} else {
|
||||
fx.debBytes["other_9-9_arm64.deb"] = testsupport.MinimalDeb("other", "9-9", "arm64")
|
||||
fx.assetFail["other_9-9_arm64.deb"] = 1
|
||||
}
|
||||
store := newFakeSyncStore()
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
remote := fx.remote()
|
||||
bg := context.Background()
|
||||
|
||||
s.process(bg, syncJob{remote: remote, prime: true})
|
||||
_, pending := store.retryAt[remote.Name]
|
||||
if pending != tc.wantRetry {
|
||||
t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
|
||||
}
|
||||
if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
|
||||
t.Fatalf("etag saved=%v after first scan", saved)
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("healthy asset not served: %d rows", len(rows))
|
||||
}
|
||||
if !tc.wantRetry {
|
||||
return
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
|
||||
store.mu.Unlock()
|
||||
s.process(bg, syncJob{remote: remote})
|
||||
if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
|
||||
t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 2 {
|
||||
t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
// Package gems proxies a RubyGems source: mutable compact index (versions,
|
||||
// info/*, names) and legacy specs indexes, plus immutable .gem and
|
||||
// quick/Marshal.4.8 gemspec downloads.
|
||||
package gems
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageGems }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
if isGem(path) || isQuickSpec(path) {
|
||||
return provider.Immutable
|
||||
}
|
||||
return provider.Mutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
path = strings.TrimLeft(path, "/")
|
||||
if path == "versions" || path == "names" || strings.HasPrefix(path, "info/") {
|
||||
return "text/plain; charset=utf-8"
|
||||
}
|
||||
if strings.HasPrefix(path, "api/v1/") && strings.HasSuffix(path, ".json") {
|
||||
return "application/json"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func isGem(path string) bool {
|
||||
path = strings.TrimLeft(path, "/")
|
||||
return strings.HasPrefix(path, "gems/") && strings.HasSuffix(path, ".gem")
|
||||
}
|
||||
|
||||
func isQuickSpec(path string) bool {
|
||||
path = strings.TrimLeft(path, "/")
|
||||
return strings.HasPrefix(path, "quick/Marshal.4.8/") && strings.HasSuffix(path, ".gemspec.rz")
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package gems
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func TestClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"gems/rack-3.1.8.gem", provider.Immutable},
|
||||
{"/gems/nokogiri-1.16.7-x86_64-linux.gem", provider.Immutable},
|
||||
{"quick/Marshal.4.8/rack-3.1.8.gemspec.rz", provider.Immutable},
|
||||
{"versions", provider.Mutable},
|
||||
{"names", provider.Mutable},
|
||||
{"info/rack", provider.Mutable},
|
||||
{"info/gems", provider.Mutable},
|
||||
{"specs.4.8.gz", provider.Mutable},
|
||||
{"latest_specs.4.8.gz", provider.Mutable},
|
||||
{"prerelease_specs.4.8.gz", provider.Mutable},
|
||||
{"api/v1/dependencies", provider.Mutable},
|
||||
{"quick/Marshal.4.8/", provider.Mutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := map[string]string{
|
||||
"versions": "text/plain; charset=utf-8",
|
||||
"info/rack": "text/plain; charset=utf-8",
|
||||
"names": "text/plain; charset=utf-8",
|
||||
"gems/rack-3.1.8.gem": "application/octet-stream",
|
||||
"specs.4.8.gz": "application/octet-stream",
|
||||
"api/v1/gems/rack.json": "application/json",
|
||||
"api/v1/dependencies.json": "application/json",
|
||||
"api/v1/dependencies": "application/octet-stream",
|
||||
}
|
||||
for path, want := range tests {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
r := models.Remote{BaseURL: "https://rubygems.org/"}
|
||||
if got := p.UpstreamURL(r, "/info/rack"); got != "https://rubygems.org/info/rack" {
|
||||
t.Errorf("UpstreamURL = %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
package gems
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"compress/zlib"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var (
|
||||
NameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*$`)
|
||||
VersionRe = regexp.MustCompile(`^[0-9]+(\.[0-9A-Za-z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$`)
|
||||
PlatformRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+$`)
|
||||
opRe = regexp.MustCompile(`^(=|!=|>|<|>=|<=|~>)$`)
|
||||
)
|
||||
|
||||
const maxMetadataBytes = 8 << 20
|
||||
|
||||
type Requirement [][2]string
|
||||
|
||||
type Dependency struct {
|
||||
Name string
|
||||
Requirement Requirement
|
||||
Type string
|
||||
Prerelease bool
|
||||
}
|
||||
|
||||
// Spec is the subset of a gemspec the compact index and quick spec need.
|
||||
type Spec struct {
|
||||
Name, Version, Platform string
|
||||
Summary, RubyGemsVer string
|
||||
SpecVersion int
|
||||
Date time.Time
|
||||
Authors, Licenses []string
|
||||
Email, Description string
|
||||
Homepage string
|
||||
Metadata map[string]string
|
||||
Dependencies []Dependency
|
||||
RequiredRuby Requirement
|
||||
RequiredRubyGems Requirement
|
||||
}
|
||||
|
||||
// FullName is the gem's file stem: name-version, plus -platform when not ruby.
|
||||
func (s *Spec) FullName() string {
|
||||
if s.Platform == "" || s.Platform == "ruby" {
|
||||
return s.Name + "-" + s.Version
|
||||
}
|
||||
return s.Name + "-" + s.Version + "-" + s.Platform
|
||||
}
|
||||
|
||||
type yamlReq struct {
|
||||
Requirements [][]yaml.Node `yaml:"requirements"`
|
||||
}
|
||||
|
||||
func (r *yamlReq) decode() (Requirement, error) {
|
||||
if r == nil {
|
||||
return nil, nil
|
||||
}
|
||||
var out Requirement
|
||||
for _, pair := range r.Requirements {
|
||||
var op string
|
||||
var ver struct {
|
||||
Version string `yaml:"version"`
|
||||
}
|
||||
if len(pair) != 2 || pair[0].Decode(&op) != nil || pair[1].Decode(&ver) != nil {
|
||||
return nil, errors.New("malformed requirement")
|
||||
}
|
||||
if !opRe.MatchString(op) || !VersionRe.MatchString(ver.Version) {
|
||||
return nil, fmt.Errorf("invalid requirement %q %q", op, ver.Version)
|
||||
}
|
||||
out = append(out, [2]string{op, ver.Version})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// String renders the requirement as the compact index does: "op ver&op ver".
|
||||
func (r Requirement) String() string {
|
||||
parts := make([]string, len(r))
|
||||
for i, p := range r {
|
||||
parts[i] = p[0] + " " + p[1]
|
||||
}
|
||||
return strings.Join(parts, "&")
|
||||
}
|
||||
|
||||
func (r Requirement) trivial() bool {
|
||||
return len(r) == 0 || len(r) == 1 && r[0] == [2]string{">=", "0"}
|
||||
}
|
||||
|
||||
// ParseGem reads the gemspec from a .gem archive (a tar holding metadata.gz).
|
||||
func ParseGem(gem []byte) (*Spec, error) {
|
||||
tr := tar.NewReader(bytes.NewReader(gem))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil, errors.New("gem has no metadata.gz")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read gem: %w", err)
|
||||
}
|
||||
if hdr.Name != "metadata.gz" {
|
||||
continue
|
||||
}
|
||||
zr, err := gzip.NewReader(tr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("metadata.gz: %w", err)
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(zr, maxMetadataBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("metadata.gz: %w", err)
|
||||
}
|
||||
if len(raw) > maxMetadataBytes {
|
||||
return nil, fmt.Errorf("metadata exceeds %d bytes", maxMetadataBytes)
|
||||
}
|
||||
return parseMetadata(raw)
|
||||
}
|
||||
}
|
||||
|
||||
func parseMetadata(raw []byte) (*Spec, error) {
|
||||
var doc struct {
|
||||
Name string `yaml:"name"`
|
||||
Version struct {
|
||||
Version string `yaml:"version"`
|
||||
} `yaml:"version"`
|
||||
Platform string `yaml:"platform"`
|
||||
Summary string `yaml:"summary"`
|
||||
Authors []string `yaml:"authors"`
|
||||
Email yaml.Node `yaml:"email"`
|
||||
Description string `yaml:"description"`
|
||||
Homepage string `yaml:"homepage"`
|
||||
Licenses []string `yaml:"licenses"`
|
||||
Metadata map[string]string `yaml:"metadata"`
|
||||
Date string `yaml:"date"`
|
||||
RubyGemsVer string `yaml:"rubygems_version"`
|
||||
SpecVersion int `yaml:"specification_version"`
|
||||
Dependencies []struct {
|
||||
Name string `yaml:"name"`
|
||||
Requirement *yamlReq `yaml:"requirement"`
|
||||
VersionReqs *yamlReq `yaml:"version_requirements"`
|
||||
Type string `yaml:"type"`
|
||||
Prerelease bool `yaml:"prerelease"`
|
||||
} `yaml:"dependencies"`
|
||||
RequiredRuby *yamlReq `yaml:"required_ruby_version"`
|
||||
RequiredRubyGems *yamlReq `yaml:"required_rubygems_version"`
|
||||
}
|
||||
if err := yaml.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("gemspec yaml: %w", err)
|
||||
}
|
||||
s := &Spec{
|
||||
Name: doc.Name, Version: doc.Version.Version, Platform: doc.Platform,
|
||||
Summary: doc.Summary, Authors: doc.Authors, Description: doc.Description,
|
||||
Homepage: doc.Homepage, Licenses: doc.Licenses, Metadata: doc.Metadata,
|
||||
RubyGemsVer: doc.RubyGemsVer, SpecVersion: doc.SpecVersion,
|
||||
}
|
||||
if s.Platform == "" {
|
||||
s.Platform = "ruby"
|
||||
}
|
||||
if !NameRe.MatchString(s.Name) || !VersionRe.MatchString(s.Version) || !PlatformRe.MatchString(s.Platform) {
|
||||
return nil, fmt.Errorf("invalid gem name/version/platform %q %q %q", s.Name, s.Version, s.Platform)
|
||||
}
|
||||
if doc.Email.Kind == yaml.ScalarNode {
|
||||
s.Email = doc.Email.Value
|
||||
}
|
||||
if t, err := time.Parse("2006-01-02", strings.SplitN(doc.Date, " ", 2)[0]); err == nil {
|
||||
s.Date = t
|
||||
}
|
||||
var err error
|
||||
if s.RequiredRuby, err = doc.RequiredRuby.decode(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.RequiredRubyGems, err = doc.RequiredRubyGems.decode(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, d := range doc.Dependencies {
|
||||
if !NameRe.MatchString(d.Name) {
|
||||
return nil, fmt.Errorf("invalid dependency name %q", d.Name)
|
||||
}
|
||||
req := d.Requirement
|
||||
if req == nil {
|
||||
req = d.VersionReqs
|
||||
}
|
||||
r, err := req.decode()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dependency %s: %w", d.Name, err)
|
||||
}
|
||||
typ := strings.TrimPrefix(d.Type, ":")
|
||||
if typ == "" {
|
||||
typ = "runtime"
|
||||
}
|
||||
s.Dependencies = append(s.Dependencies, Dependency{Name: d.Name, Requirement: r, Type: typ, Prerelease: d.Prerelease})
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// InfoLine renders the spec's compact index /info line; checksum is the
|
||||
// .gem's sha256 hex.
|
||||
func InfoLine(s *Spec, gem []byte) string {
|
||||
sum := sha256.Sum256(gem)
|
||||
var deps []string
|
||||
for _, d := range s.Dependencies {
|
||||
if d.Type == "runtime" {
|
||||
r := d.Requirement
|
||||
if len(r) == 0 {
|
||||
r = Requirement{{">=", "0"}}
|
||||
}
|
||||
deps = append(deps, d.Name+":"+r.String())
|
||||
}
|
||||
}
|
||||
key := s.Version
|
||||
if s.Platform != "ruby" {
|
||||
key += "-" + s.Platform
|
||||
}
|
||||
line := key + " " + strings.Join(deps, ",") + "|checksum:" + hex.EncodeToString(sum[:])
|
||||
if !s.RequiredRuby.trivial() {
|
||||
line += ",ruby:" + s.RequiredRuby.String()
|
||||
}
|
||||
if !s.RequiredRubyGems.trivial() {
|
||||
line += ",rubygems:" + s.RequiredRubyGems.String()
|
||||
}
|
||||
return line
|
||||
}
|
||||
|
||||
// InfoBody renders an /info document from its lines.
|
||||
func InfoBody(lines []string) []byte {
|
||||
var b strings.Builder
|
||||
b.WriteString("---\n")
|
||||
for _, l := range lines {
|
||||
b.WriteString(l)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return []byte(b.String())
|
||||
}
|
||||
|
||||
// InfoKey returns an /info line's version[-platform] token.
|
||||
func InfoKey(line string) string {
|
||||
k, _, _ := strings.Cut(line, " ")
|
||||
return k
|
||||
}
|
||||
|
||||
// InfoLines returns an /info body's version lines.
|
||||
func InfoLines(body []byte) []string {
|
||||
var out []string
|
||||
for _, l := range strings.Split(string(body), "\n") {
|
||||
if l != "" && l != "---" {
|
||||
out = append(out, l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func MD5Hex(b []byte) string {
|
||||
sum := md5.Sum(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
// VersionsLine renders one /versions entry from a gem's /info body.
|
||||
func VersionsLine(name string, info []byte) string {
|
||||
var keys []string
|
||||
for _, l := range InfoLines(info) {
|
||||
keys = append(keys, InfoKey(l))
|
||||
}
|
||||
return name + " " + strings.Join(keys, ",") + " " + MD5Hex(info)
|
||||
}
|
||||
|
||||
// VersionsHeader starts a /versions document.
|
||||
func VersionsHeader(at time.Time) string {
|
||||
return "created_at: " + at.UTC().Format(time.RFC3339) + "\n---\n"
|
||||
}
|
||||
|
||||
// QuickSpec returns the deflated Marshal.4.8 dump of the spec, as served at
|
||||
// quick/Marshal.4.8/<full>.gemspec.rz.
|
||||
func QuickSpec(s *Spec) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
zw := zlib.NewWriter(&buf)
|
||||
if _, err := zw.Write(MarshalSpec(s)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// MarshalSpec mirrors Gem::Specification#_dump wrapped by Marshal.dump.
|
||||
func MarshalSpec(s *Spec) []byte {
|
||||
var in rbMarshal
|
||||
in.WriteString("\x04\x08")
|
||||
in.array(19)
|
||||
in.str(s.RubyGemsVer)
|
||||
in.WriteByte('i')
|
||||
in.fixnum(s.SpecVersion)
|
||||
in.str(s.Name)
|
||||
in.version(s.Version)
|
||||
in.time(s.Date)
|
||||
in.str(s.Summary)
|
||||
in.requirement(s.RequiredRuby)
|
||||
in.requirement(s.RequiredRubyGems)
|
||||
in.str(s.Platform)
|
||||
in.array(len(s.Dependencies))
|
||||
for _, d := range s.Dependencies {
|
||||
in.WriteByte('o')
|
||||
in.sym("Gem::Dependency")
|
||||
in.fixnum(4)
|
||||
in.sym("@name")
|
||||
in.str(d.Name)
|
||||
in.sym("@requirement")
|
||||
in.requirement(d.Requirement)
|
||||
in.sym("@type")
|
||||
in.sym(d.Type)
|
||||
in.sym("@prerelease")
|
||||
in.boolean(d.Prerelease)
|
||||
}
|
||||
in.str("")
|
||||
in.optStr(s.Email)
|
||||
in.strs(s.Authors)
|
||||
in.optStr(s.Description)
|
||||
in.optStr(s.Homepage)
|
||||
in.boolean(true)
|
||||
in.str(s.Platform)
|
||||
in.strs(s.Licenses)
|
||||
in.WriteByte('{')
|
||||
in.fixnum(len(s.Metadata))
|
||||
for _, k := range slices.Sorted(maps.Keys(s.Metadata)) {
|
||||
in.str(k)
|
||||
in.str(s.Metadata[k])
|
||||
}
|
||||
|
||||
var out rbMarshal
|
||||
out.WriteString("\x04\x08u")
|
||||
out.sym("Gem::Specification")
|
||||
out.fixnum(in.Len())
|
||||
out.Write(in.Bytes())
|
||||
return out.Bytes()
|
||||
}
|
||||
|
||||
// rbMarshal writes the Ruby Marshal 4.8 subset gemspecs use. It never emits
|
||||
// object or symbol links, so every symbol is written in full.
|
||||
type rbMarshal struct{ bytes.Buffer }
|
||||
|
||||
func (m *rbMarshal) fixnum(n int) {
|
||||
switch {
|
||||
case n == 0:
|
||||
m.WriteByte(0)
|
||||
case n > 0 && n < 123:
|
||||
m.WriteByte(byte(n + 5))
|
||||
case n < 0 && n > -124:
|
||||
m.WriteByte(byte(n - 5))
|
||||
default:
|
||||
var b []byte
|
||||
for x := n; len(b) < 4; x >>= 8 {
|
||||
b = append(b, byte(x))
|
||||
if (n >= 0 && x>>8 == 0) || (n < 0 && x>>8 == -1) {
|
||||
break
|
||||
}
|
||||
}
|
||||
if n >= 0 {
|
||||
m.WriteByte(byte(len(b)))
|
||||
} else {
|
||||
m.WriteByte(byte(-len(b)))
|
||||
}
|
||||
m.Write(b)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *rbMarshal) raw(s string) {
|
||||
m.fixnum(len(s))
|
||||
m.WriteString(s)
|
||||
}
|
||||
|
||||
func (m *rbMarshal) sym(s string) {
|
||||
m.WriteByte(':')
|
||||
m.raw(s)
|
||||
}
|
||||
|
||||
// str writes a UTF-8 String (an ivar-wrapped string with E=true).
|
||||
func (m *rbMarshal) str(s string) {
|
||||
m.WriteString("I\"")
|
||||
m.raw(s)
|
||||
m.fixnum(1)
|
||||
m.sym("E")
|
||||
m.WriteByte('T')
|
||||
}
|
||||
|
||||
func (m *rbMarshal) optStr(s string) {
|
||||
if s == "" {
|
||||
m.WriteByte('0')
|
||||
return
|
||||
}
|
||||
m.str(s)
|
||||
}
|
||||
|
||||
func (m *rbMarshal) strs(ss []string) {
|
||||
m.array(len(ss))
|
||||
for _, s := range ss {
|
||||
m.str(s)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *rbMarshal) array(n int) {
|
||||
m.WriteByte('[')
|
||||
m.fixnum(n)
|
||||
}
|
||||
|
||||
func (m *rbMarshal) boolean(b bool) {
|
||||
if b {
|
||||
m.WriteByte('T')
|
||||
} else {
|
||||
m.WriteByte('F')
|
||||
}
|
||||
}
|
||||
|
||||
func (m *rbMarshal) version(v string) {
|
||||
m.WriteByte('U')
|
||||
m.sym("Gem::Version")
|
||||
m.array(1)
|
||||
m.str(v)
|
||||
}
|
||||
|
||||
func (m *rbMarshal) requirement(r Requirement) {
|
||||
if len(r) == 0 {
|
||||
r = Requirement{{">=", "0"}}
|
||||
}
|
||||
m.WriteByte('U')
|
||||
m.sym("Gem::Requirement")
|
||||
m.array(1)
|
||||
m.array(len(r))
|
||||
for _, p := range r {
|
||||
m.array(2)
|
||||
m.str(p[0])
|
||||
m.version(p[1])
|
||||
}
|
||||
}
|
||||
|
||||
// time writes a UTC Time with day resolution, as Time#_dump lays it out.
|
||||
func (m *rbMarshal) time(t time.Time) {
|
||||
t = t.UTC()
|
||||
// Time#_dump holds year-1900 in 16 bits.
|
||||
if t.IsZero() || t.Year() < 1900 || t.Year() > 1900+0xffff {
|
||||
t = time.Unix(0, 0).UTC()
|
||||
}
|
||||
p := uint32(1)<<31 | uint32(1)<<30 | uint32(t.Year()-1900)<<14 | uint32(t.Month()-1)<<10 | uint32(t.Day())<<5 | uint32(t.Hour())
|
||||
s := uint32(t.Minute())<<26 | uint32(t.Second())<<20
|
||||
m.WriteString("Iu")
|
||||
m.sym("Time")
|
||||
m.fixnum(8)
|
||||
for _, w := range []uint32{p, s} {
|
||||
m.Write([]byte{byte(w), byte(w >> 8), byte(w >> 16), byte(w >> 24)})
|
||||
}
|
||||
m.fixnum(1)
|
||||
m.sym("zone")
|
||||
m.WriteString("I\"")
|
||||
m.raw("UTC")
|
||||
m.fixnum(1)
|
||||
m.sym("E")
|
||||
m.WriteByte('F')
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package gems
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"compress/zlib"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const testMetadata = `--- !ruby/object:Gem::Specification
|
||||
name: unkin-e2e-hello
|
||||
version: !ruby/object:Gem::Version
|
||||
version: 0.1.0
|
||||
platform: %PLATFORM%
|
||||
authors:
|
||||
- e2e
|
||||
date: 2026-10-17 00:00:00.000000000 Z
|
||||
dependencies:
|
||||
- !ruby/object:Gem::Dependency
|
||||
name: rack
|
||||
requirement: &1 !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '2.0'
|
||||
- - "<"
|
||||
- !ruby/object:Gem::Version
|
||||
version: '4'
|
||||
type: :runtime
|
||||
prerelease: false
|
||||
version_requirements: *1
|
||||
- !ruby/object:Gem::Dependency
|
||||
name: json
|
||||
version_requirements: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - "~>"
|
||||
- !ruby/object:Gem::Version
|
||||
version: '2.1'
|
||||
type: :runtime
|
||||
prerelease: false
|
||||
- !ruby/object:Gem::Dependency
|
||||
name: rake
|
||||
requirement: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '0'
|
||||
type: :development
|
||||
prerelease: false
|
||||
email:
|
||||
licenses:
|
||||
- MIT
|
||||
metadata:
|
||||
source_code_uri: https://example.com
|
||||
required_ruby_version: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '2.7'
|
||||
required_rubygems_version: !ruby/object:Gem::Requirement
|
||||
requirements:
|
||||
- - ">="
|
||||
- !ruby/object:Gem::Version
|
||||
version: '0'
|
||||
rubygems_version: 3.5.22
|
||||
specification_version: 4
|
||||
summary: hi
|
||||
`
|
||||
|
||||
func buildGem(t *testing.T, metadata []byte) []byte {
|
||||
t.Helper()
|
||||
var gz bytes.Buffer
|
||||
zw := gzip.NewWriter(&gz)
|
||||
_, _ = zw.Write(metadata)
|
||||
_ = zw.Close()
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
for name, body := range map[string][]byte{"data.tar.gz": []byte("x"), "metadata.gz": gz.Bytes()} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o444, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = tw.Write(body)
|
||||
}
|
||||
_ = tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func testGem(t *testing.T, platform string) []byte {
|
||||
return buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", platform, 1)))
|
||||
}
|
||||
|
||||
func TestParseGemAndInfoLine(t *testing.T) {
|
||||
gem := testGem(t, "ruby")
|
||||
s, err := ParseGem(gem)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if s.FullName() != "unkin-e2e-hello-0.1.0" || !s.Date.Equal(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC)) {
|
||||
t.Fatalf("spec = %+v", s)
|
||||
}
|
||||
if len(s.Dependencies) != 3 || s.Dependencies[1].Requirement.String() != "~> 2.1" {
|
||||
t.Fatalf("deps = %+v", s.Dependencies)
|
||||
}
|
||||
sum := sha256.Sum256(gem)
|
||||
want := "0.1.0 rack:>= 2.0&< 4,json:~> 2.1|checksum:" + hex.EncodeToString(sum[:]) + ",ruby:>= 2.7"
|
||||
if got := InfoLine(s, gem); got != want {
|
||||
t.Fatalf("InfoLine =\n %s\nwant\n %s", got, want)
|
||||
}
|
||||
|
||||
plat, err := ParseGem(testGem(t, "x86_64-linux"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if plat.FullName() != "unkin-e2e-hello-0.1.0-x86_64-linux" || InfoKey(InfoLine(plat, nil)) != "0.1.0-x86_64-linux" {
|
||||
t.Fatalf("platform gem: %s / %s", plat.FullName(), InfoLine(plat, nil))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseGemRejects(t *testing.T) {
|
||||
big := []byte(testMetadata + "description: " + strings.Repeat("a", maxMetadataBytes) + "\n")
|
||||
cases := map[string][]byte{
|
||||
"not a tar": []byte("garbage"),
|
||||
"no metadata": func() []byte { var b bytes.Buffer; tw := tar.NewWriter(&b); _ = tw.Close(); return b.Bytes() }(),
|
||||
"bad name": buildGem(t, []byte(strings.Replace(testMetadata, "name: unkin-e2e-hello", "name: ../x", 1))),
|
||||
"bad version": buildGem(t, []byte(strings.Replace(testMetadata, "version: 0.1.0", "version: 'a b'", 1))),
|
||||
"bad dep op": buildGem(t, []byte(strings.Replace(testMetadata, `- - "~>"`, `- - "=~"`, 1))),
|
||||
"metadata bomb": buildGem(t, big),
|
||||
"bad platform": buildGem(t, []byte(strings.Replace(testMetadata, "%PLATFORM%", "x/y", 1))),
|
||||
"bad dependency": buildGem(t, []byte(strings.Replace(testMetadata, "name: json", "name: 'a|b'", 1))),
|
||||
}
|
||||
for name, gem := range cases {
|
||||
if _, err := ParseGem(gem); err == nil {
|
||||
t.Errorf("%s: accepted", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionsLineMD5(t *testing.T) {
|
||||
info := InfoBody([]string{"0.1.0 |checksum:aa", "0.2.0-java |checksum:bb"})
|
||||
if string(info) != "---\n0.1.0 |checksum:aa\n0.2.0-java |checksum:bb\n" {
|
||||
t.Fatalf("InfoBody = %q", info)
|
||||
}
|
||||
if got := VersionsLine("g", info); got != "g 0.1.0,0.2.0-java "+MD5Hex(info) {
|
||||
t.Fatalf("VersionsLine = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Byte layouts below are taken from Ruby 3.3's Marshal.dump.
|
||||
func TestRubyMarshalEncoding(t *testing.T) {
|
||||
for n, want := range map[int]string{0: "\x00", 4: "\x09", 122: "\x7f", 123: "\x01\x7b", 255: "\x01\xff", 256: "\x02\x00\x01", 65536: "\x03\x00\x00\x01", -1: "\xfa", -123: "\x80"} {
|
||||
var m rbMarshal
|
||||
m.fixnum(n)
|
||||
if m.String() != want {
|
||||
t.Errorf("fixnum(%d) = %x, want %x", n, m.String(), want)
|
||||
}
|
||||
}
|
||||
var m rbMarshal
|
||||
m.str("3.5.22")
|
||||
if m.String() != "I\"\x0b3.5.22\x06:\x06ET" {
|
||||
t.Errorf("str = %q", m.String())
|
||||
}
|
||||
m.Reset()
|
||||
m.version("0.1.0")
|
||||
if m.String() != "U:\x11Gem::Version[\x06I\"\x0a0.1.0\x06:\x06ET" {
|
||||
t.Errorf("version = %q", m.String())
|
||||
}
|
||||
m.Reset()
|
||||
m.time(time.Date(2026, 10, 17, 0, 0, 0, 0, time.UTC))
|
||||
if want := "Iu:\x09Time\x0d\x20\xa6\x1f\xc0\x00\x00\x00\x00\x06:\x09zoneI\"\x08UTC\x06:\x06EF"; m.String() != want {
|
||||
t.Errorf("time = %q, want %q", m.String(), want)
|
||||
}
|
||||
var epoch rbMarshal
|
||||
epoch.time(time.Time{})
|
||||
for _, y := range []int{1, 1899, 1900 + 0x10000} {
|
||||
m.Reset()
|
||||
m.time(time.Date(y, 1, 1, 0, 0, 0, 0, time.UTC))
|
||||
if m.String() != epoch.String() {
|
||||
t.Errorf("time(year %d) = %q, want the epoch encoding %q", y, m.String(), epoch.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuickSpec(t *testing.T) {
|
||||
s, err := ParseGem(testGem(t, "ruby"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q, err := QuickSpec(s)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q2, _ := QuickSpec(s)
|
||||
if !bytes.Equal(q, q2) {
|
||||
t.Fatal("QuickSpec is not deterministic")
|
||||
}
|
||||
zr, err := zlib.NewReader(bytes.NewReader(q))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, _ := io.ReadAll(zr)
|
||||
head := "\x04\x08u:\x17Gem::Specification"
|
||||
if !strings.HasPrefix(string(raw), head) {
|
||||
t.Fatalf("quick spec prefix = %q", raw[:30])
|
||||
}
|
||||
inner := string(raw[len(head):])
|
||||
at := strings.Index(inner, "\x04\x08[\x18")
|
||||
var length rbMarshal
|
||||
length.fixnum(len(inner) - at)
|
||||
if at < 1 || inner[:at] != length.String() {
|
||||
t.Fatalf("inner dump header = %q", inner[:8])
|
||||
}
|
||||
for _, want := range []string{"o:\x14Gem::Dependency", ":\x0cruntime", ":\x10development", "source_code_uri"} {
|
||||
if !strings.Contains(string(raw), want) {
|
||||
t.Errorf("quick spec missing %q", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/mod/modfile"
|
||||
"golang.org/x/mod/module"
|
||||
"golang.org/x/mod/semver"
|
||||
modzip "golang.org/x/mod/zip"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
type versionFile struct {
|
||||
module string
|
||||
version string
|
||||
ext string
|
||||
}
|
||||
|
||||
// parseVersionFile splits "<escaped module>/@v/<version>.<ext>" into its parts.
|
||||
func parseVersionFile(path string) (versionFile, error) {
|
||||
escMod, file, ok := strings.Cut(strings.TrimPrefix(path, "/"), "/@v/")
|
||||
if !ok || escMod == "" || strings.Contains(file, "/") {
|
||||
return versionFile{}, fmt.Errorf("path %q must be <module>/@v/<version>.{zip,mod,info}", path)
|
||||
}
|
||||
dot := strings.LastIndex(file, ".")
|
||||
if dot <= 0 {
|
||||
return versionFile{}, fmt.Errorf("path %q must end in .zip, .mod or .info", path)
|
||||
}
|
||||
ext := file[dot+1:]
|
||||
if ext != "zip" && ext != "mod" && ext != "info" {
|
||||
return versionFile{}, fmt.Errorf("path %q must end in .zip, .mod or .info", path)
|
||||
}
|
||||
mod, err := module.UnescapePath(escMod)
|
||||
if err != nil {
|
||||
return versionFile{}, fmt.Errorf("module path must be case-encoded (uppercase as !lower): %w", err)
|
||||
}
|
||||
ver, err := module.UnescapeVersion(file[:dot])
|
||||
if err != nil {
|
||||
return versionFile{}, err
|
||||
}
|
||||
if err := module.Check(mod, ver); err != nil {
|
||||
return versionFile{}, err
|
||||
}
|
||||
if semver.Canonical(ver) != ver {
|
||||
return versionFile{}, fmt.Errorf("version %q is not canonical semver", ver)
|
||||
}
|
||||
return versionFile{module: mod, version: ver, ext: ext}, nil
|
||||
}
|
||||
|
||||
func (v versionFile) storagePath() string {
|
||||
escMod, _ := module.EscapePath(v.module)
|
||||
escVer, _ := module.EscapeVersion(v.version)
|
||||
return escMod + "/@v/" + escVer + "." + v.ext
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
vf, err := parseVersionFile(filePath)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
sp := vf.storagePath()
|
||||
return sp, p.ContentType(sp), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateContent(storagePath, localFile string) error {
|
||||
vf, err := parseVersionFile(storagePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch vf.ext {
|
||||
case "zip":
|
||||
if _, err := modzip.CheckZip(module.Version{Path: vf.module, Version: vf.version}, localFile); err != nil {
|
||||
return fmt.Errorf("invalid module zip (files must sit under %s@%s/): %w", vf.module, vf.version, err)
|
||||
}
|
||||
case "mod":
|
||||
data, err := os.ReadFile(localFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mf, err := modfile.ParseLax("go.mod", data, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid go.mod: %w", err)
|
||||
}
|
||||
if mf.Module == nil || mf.Module.Mod.Path != vf.module {
|
||||
return fmt.Errorf("go.mod must declare module %s", vf.module)
|
||||
}
|
||||
case "info":
|
||||
data, err := os.ReadFile(localFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var info struct{ Version string }
|
||||
if err := json.Unmarshal(data, &info); err != nil {
|
||||
return fmt.Errorf("invalid .info JSON: %w", err)
|
||||
}
|
||||
if info.Version != vf.version {
|
||||
return fmt.Errorf(".info Version %q does not match %s", info.Version, vf.version)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
vf, _ := parseVersionFile(storagePath)
|
||||
return map[string]any{
|
||||
"module": vf.module,
|
||||
"version": vf.version,
|
||||
"path": storagePath,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
// versions returns the canonical versions of escMod that have a .mod uploaded, sorted ascending.
|
||||
func versions(ctx context.Context, files provider.FileStore, repoName, escMod string) (versions []string, infos map[string]bool, err error) {
|
||||
entries, err := files.ListFilesByPrefix(ctx, repoName, escMod+"/@v/")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
infos = map[string]bool{}
|
||||
for _, e := range entries {
|
||||
vf, err := parseVersionFile(e.FilePath)
|
||||
if err != nil || vf.storagePath() != e.FilePath {
|
||||
continue
|
||||
}
|
||||
switch vf.ext {
|
||||
case "mod":
|
||||
versions = append(versions, vf.version)
|
||||
case "info":
|
||||
infos[vf.version] = true
|
||||
}
|
||||
}
|
||||
semver.Sort(versions)
|
||||
return versions, infos, nil
|
||||
}
|
||||
|
||||
// Latest is the highest release in sorted vs, else the highest prerelease.
|
||||
func Latest(vs []string) string {
|
||||
for i := len(vs) - 1; i >= 0; i-- {
|
||||
if semver.Prerelease(vs[i]) == "" {
|
||||
return vs[i]
|
||||
}
|
||||
}
|
||||
if len(vs) == 0 {
|
||||
return ""
|
||||
}
|
||||
return vs[len(vs)-1]
|
||||
}
|
||||
|
||||
func infoJSON(version string) []byte {
|
||||
b, _ := json.Marshal(map[string]string{"Version": version})
|
||||
return b
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
switch {
|
||||
case strings.HasSuffix(path, "/@v/list"), strings.HasSuffix(path, "/@latest"):
|
||||
case strings.HasSuffix(path, ".info"):
|
||||
vf, err := parseVersionFile(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
escMod, _ := module.EscapePath(vf.module)
|
||||
vs, infos, err := versions(r.Context(), files, repoName, escMod)
|
||||
if err != nil || infos[vf.version] || !slices.Contains(vs, vf.version) {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
body, err := p.GenerateLocalIndex(r.Context(), files, repoName, path)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return true
|
||||
}
|
||||
w.Header().Set("Content-Type", p.ContentType(path))
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(body)
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
path = strings.TrimPrefix(path, "/")
|
||||
if escMod, ok := strings.CutSuffix(path, "/@v/list"); ok {
|
||||
vs, _, err := versions(ctx, files, repoName, escMod)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, v := range vs {
|
||||
b.WriteString(v + "\n")
|
||||
}
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
if escMod, ok := strings.CutSuffix(path, "/@latest"); ok {
|
||||
vs, _, err := versions(ctx, files, repoName, escMod)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v := Latest(vs)
|
||||
if v == "" {
|
||||
return nil, fmt.Errorf("no versions of %s", escMod)
|
||||
}
|
||||
return infoJSON(v), nil
|
||||
}
|
||||
if strings.HasSuffix(path, ".info") {
|
||||
vf, err := parseVersionFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return infoJSON(vf.version), nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported index path: %q", path)
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package goproxy
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/mod/module"
|
||||
modzip "golang.org/x/mod/zip"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
type fakeFileStore struct{ paths []string }
|
||||
|
||||
func (f *fakeFileStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func (f *fakeFileStore) ListFilesByPrefix(_ context.Context, _, prefix string) ([]provider.FileEntry, error) {
|
||||
var out []provider.FileEntry
|
||||
for _, p := range f.paths {
|
||||
if strings.HasPrefix(p, prefix) {
|
||||
out = append(out, provider.FileEntry{FilePath: p, ContentHash: "sha256:x"})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func TestValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
cases := []struct {
|
||||
in, want string
|
||||
ok bool
|
||||
}{
|
||||
{"example.unkin.net/hello/@v/v0.1.0.zip", "example.unkin.net/hello/@v/v0.1.0.zip", true},
|
||||
{"/example.unkin.net/hello/@v/v0.1.0.mod", "example.unkin.net/hello/@v/v0.1.0.mod", true},
|
||||
{"github.com/!burnt!sushi/toml/@v/v1.2.0.info", "github.com/!burnt!sushi/toml/@v/v1.2.0.info", true},
|
||||
{"example.unkin.net/hello/v2/@v/v2.0.0-rc.1.zip", "example.unkin.net/hello/v2/@v/v2.0.0-rc.1.zip", true},
|
||||
{"github.com/BurntSushi/toml/@v/v1.2.0.zip", "", false},
|
||||
{"example.unkin.net/hello/@v/v0.1.zip", "", false},
|
||||
{"example.unkin.net/hello/@v/v2.0.0.zip", "", false},
|
||||
{"example.unkin.net/hello/@v/v0.1.0.tar.gz", "", false},
|
||||
{"example.unkin.net/hello/v0.1.0.zip", "", false},
|
||||
{"example.unkin.net/hello/@v/list", "", false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
got, ct, err := p.ValidateUpload(c.in)
|
||||
if (err == nil) != c.ok {
|
||||
t.Errorf("%s: err=%v, want ok=%v", c.in, err, c.ok)
|
||||
continue
|
||||
}
|
||||
if c.ok && (got != c.want || ct == "") {
|
||||
t.Errorf("%s: got %q %q, want %q", c.in, got, ct, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeZip(t *testing.T, files map[string]string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "m.zip")
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
zw := zip.NewWriter(f)
|
||||
for name, body := range files {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := w.Write([]byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func writeFile(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "f")
|
||||
if err := os.WriteFile(path, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
func TestValidateContentZip(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp := "example.unkin.net/hello/@v/v0.1.0.zip"
|
||||
|
||||
dir := t.TempDir()
|
||||
os.WriteFile(filepath.Join(dir, "go.mod"), []byte("module example.unkin.net/hello\n"), 0o600)
|
||||
os.WriteFile(filepath.Join(dir, "main.go"), []byte("package main\n"), 0o600)
|
||||
good := filepath.Join(t.TempDir(), "good.zip")
|
||||
f, _ := os.Create(good)
|
||||
if err := modzip.CreateFromDir(f, module.Version{Path: "example.unkin.net/hello", Version: "v0.1.0"}, dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.Close()
|
||||
if err := p.ValidateContent(sp, good); err != nil {
|
||||
t.Errorf("valid zip rejected: %v", err)
|
||||
}
|
||||
|
||||
bad := map[string]map[string]string{
|
||||
"no prefix": {"go.mod": "module example.unkin.net/hello\n"},
|
||||
"wrong version": {"example.unkin.net/hello@v0.2.0/go.mod": "module example.unkin.net/hello\n"},
|
||||
"wrong module": {"example.unkin.net/other@v0.1.0/go.mod": "module example.unkin.net/other\n"},
|
||||
}
|
||||
for name, files := range bad {
|
||||
if err := p.ValidateContent(sp, writeZip(t, files)); err == nil {
|
||||
t.Errorf("%s: expected rejection", name)
|
||||
}
|
||||
}
|
||||
if err := p.ValidateContent(sp, writeFile(t, "not a zip")); err == nil {
|
||||
t.Error("non-zip accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateContentModInfo(t *testing.T) {
|
||||
p := &Provider{}
|
||||
mod := "example.unkin.net/hello/@v/v0.1.0.mod"
|
||||
info := "example.unkin.net/hello/@v/v0.1.0.info"
|
||||
cases := []struct {
|
||||
path, body string
|
||||
ok bool
|
||||
}{
|
||||
{mod, "module example.unkin.net/hello\n\ngo 1.25\n", true},
|
||||
{mod, "module example.unkin.net/other\n", false},
|
||||
{mod, "go 1.25\n", false},
|
||||
{mod, "module (\n", false},
|
||||
{info, `{"Version":"v0.1.0","Time":"2026-01-01T00:00:00Z"}`, true},
|
||||
{info, `{"Version":"v0.2.0"}`, false},
|
||||
{info, `nope`, false},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if err := p.ValidateContent(c.path, writeFile(t, c.body)); (err == nil) != c.ok {
|
||||
t.Errorf("%s %q: err=%v, want ok=%v", c.path, c.body, err, c.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func serve(t *testing.T, fs provider.FileStore, path string) (bool, int, string) {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
handled := (&Provider{}).ServeLocalIndex(rec, httptest.NewRequest("GET", "/"+path, nil), fs, "go-local", path)
|
||||
return handled, rec.Code, rec.Body.String()
|
||||
}
|
||||
|
||||
func TestServeLocalIndex(t *testing.T) {
|
||||
fs := &fakeFileStore{paths: []string{
|
||||
"example.unkin.net/hello/@v/v0.10.0.mod",
|
||||
"example.unkin.net/hello/@v/v0.10.0.zip",
|
||||
"example.unkin.net/hello/@v/v0.2.0.mod",
|
||||
"example.unkin.net/hello/@v/v0.2.0.zip",
|
||||
"example.unkin.net/hello/@v/v0.2.0.info",
|
||||
"example.unkin.net/hello/@v/v0.11.0-rc.1.mod",
|
||||
"example.unkin.net/hello/@v/v0.3.0.zip",
|
||||
"example.unkin.net/hello/v2/@v/v2.0.0.mod",
|
||||
"example.unkin.net/!pre/@v/v1.0.0-beta.mod",
|
||||
}}
|
||||
|
||||
if h, code, body := serve(t, fs, "example.unkin.net/hello/@v/list"); !h || code != 200 || body != "v0.2.0\nv0.10.0\nv0.11.0-rc.1\n" {
|
||||
t.Errorf("list: %v %d %q", h, code, body)
|
||||
}
|
||||
if h, code, body := serve(t, fs, "example.unkin.net/hello/@latest"); !h || code != 200 || body != `{"Version":"v0.10.0"}` {
|
||||
t.Errorf("latest: %v %d %q", h, code, body)
|
||||
}
|
||||
if h, _, body := serve(t, fs, "example.unkin.net/!pre/@latest"); !h || body != `{"Version":"v1.0.0-beta"}` {
|
||||
t.Errorf("prerelease-only latest: %q", body)
|
||||
}
|
||||
if h, code, _ := serve(t, fs, "example.unkin.net/missing/@latest"); !h || code != 404 {
|
||||
t.Errorf("missing latest: %v %d", h, code)
|
||||
}
|
||||
if h, code, body := serve(t, fs, "example.unkin.net/missing/@v/list"); !h || code != 200 || body != "" {
|
||||
t.Errorf("missing list: %v %d %q", h, code, body)
|
||||
}
|
||||
if h, _, body := serve(t, fs, "example.unkin.net/hello/@v/v0.10.0.info"); !h || body != `{"Version":"v0.10.0"}` {
|
||||
t.Errorf("synthesized info: %v %q", h, body)
|
||||
}
|
||||
if h, _, _ := serve(t, fs, "example.unkin.net/hello/@v/v0.2.0.info"); h {
|
||||
t.Error("uploaded .info must be served from storage")
|
||||
}
|
||||
if h, _, _ := serve(t, fs, "example.unkin.net/hello/@v/v0.3.0.info"); h {
|
||||
t.Error(".info without a .mod must not be synthesized")
|
||||
}
|
||||
if h, _, _ := serve(t, fs, "example.unkin.net/hello/@v/v0.10.0.zip"); h {
|
||||
t.Error(".zip must be served from storage")
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,24 @@
|
||||
package helm
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -39,6 +51,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewritesResponse() {}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
@@ -56,3 +70,203 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
// ValidateUpload stores charts flat by filename so index.yaml can reference
|
||||
// them with relative urls, as `helm repo index` does.
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := path.Base(path.Clean("/" + filePath))
|
||||
if !strings.HasSuffix(filename, ".tgz") || filename == ".tgz" {
|
||||
return "", "", fmt.Errorf("file must be a .tgz chart package")
|
||||
}
|
||||
return filename, "application/gzip", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
return map[string]any{
|
||||
"filename": storagePath,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
reader, _, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
|
||||
if err != nil {
|
||||
slog.Error("helm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
|
||||
meta, err := parseChart(reader)
|
||||
if err != nil {
|
||||
slog.Error("helm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
if want := meta.Name + "-" + meta.Version + ".tgz"; storagePath != want {
|
||||
slog.Error("helm metadata: filename does not match chart", "repo", repoName, "path", storagePath, "expected", want)
|
||||
return
|
||||
}
|
||||
meta.RepoName = repoName
|
||||
meta.FilePath = storagePath
|
||||
meta.ContentHash = contentHash
|
||||
|
||||
store, ok := db.(provider.HelmMetadataStore)
|
||||
if !ok {
|
||||
slog.Error("helm metadata: store does not support helm metadata", "repo", repoName)
|
||||
return
|
||||
}
|
||||
if err := store.InsertHelmMetadata(ctx, meta); err != nil {
|
||||
slog.Error("helm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("helm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
deleter, ok := db.(provider.HelmMetadataDeleter)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return deleter.DeleteHelmMetadata(ctx, repoName, storagePath)
|
||||
}
|
||||
|
||||
func isIndexPath(p string) bool {
|
||||
return strings.TrimPrefix(path.Clean("/"+p), "/") == "index.yaml"
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
if !isIndexPath(reqPath) {
|
||||
return false
|
||||
}
|
||||
body, err := p.GenerateLocalIndex(r.Context(), files, repoName, reqPath)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/yaml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write(body)
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, reqPath string) ([]byte, error) {
|
||||
if !isIndexPath(reqPath) {
|
||||
return nil, fmt.Errorf("unsupported index path: %q", reqPath)
|
||||
}
|
||||
reader, ok := files.(provider.HelmMetadataReader)
|
||||
if !ok {
|
||||
return nil, errors.New("helm metadata not available")
|
||||
}
|
||||
metas, err := reader.ListHelmMetadataEntries(ctx, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return generateIndex(metas)
|
||||
}
|
||||
|
||||
// generateIndex builds index.yaml from stored Chart.yaml documents; it is
|
||||
// stamped from the newest upload so every replica serves identical bytes.
|
||||
func generateIndex(metas []provider.HelmMetadata) ([]byte, error) {
|
||||
entries := map[string][]map[string]yaml.Node{}
|
||||
var newest time.Time
|
||||
budget := provider.MaxAliasExpandedNodes
|
||||
for _, m := range metas {
|
||||
var raw map[string]yaml.Node
|
||||
if err := yaml.Unmarshal([]byte(m.ChartYAML), &raw); err != nil {
|
||||
return nil, fmt.Errorf("chart %s: %w", m.FilePath, err)
|
||||
}
|
||||
chart, err := provider.ExpandYAMLAliases(raw, &budget)
|
||||
if err != nil {
|
||||
slog.Warn("helm index: dropping chart", "chart", m.Name, "path", m.FilePath, "error", err)
|
||||
continue
|
||||
}
|
||||
chart["name"] = node(m.Name)
|
||||
chart["version"] = node(m.Version)
|
||||
chart["urls"] = node([]string{url.PathEscape(m.FilePath)})
|
||||
chart["digest"] = node(strings.TrimPrefix(m.ContentHash, "sha256:"))
|
||||
chart["created"] = node(m.CreatedAt.UTC().Format(time.RFC3339Nano))
|
||||
entries[m.Name] = append(entries[m.Name], chart)
|
||||
if m.CreatedAt.After(newest) {
|
||||
newest = m.CreatedAt
|
||||
}
|
||||
}
|
||||
return yaml.Marshal(map[string]any{
|
||||
"apiVersion": "v1",
|
||||
"entries": entries,
|
||||
"generated": newest.UTC().Format(time.RFC3339Nano),
|
||||
})
|
||||
}
|
||||
|
||||
func node(v any) yaml.Node {
|
||||
var n yaml.Node
|
||||
_ = n.Encode(v)
|
||||
return n
|
||||
}
|
||||
|
||||
const (
|
||||
maxChartYAML = 1 << 20
|
||||
maxChartBytes = 50 << 20
|
||||
maxChartEntries = 10000
|
||||
)
|
||||
|
||||
var errChartTooLarge = fmt.Errorf("chart archive exceeds %d decompressed bytes", maxChartBytes)
|
||||
|
||||
type capReader struct {
|
||||
r io.Reader
|
||||
n int64
|
||||
}
|
||||
|
||||
func (c *capReader) Read(p []byte) (int, error) {
|
||||
if c.n <= 0 {
|
||||
return 0, errChartTooLarge
|
||||
}
|
||||
if int64(len(p)) > c.n {
|
||||
p = p[:c.n]
|
||||
}
|
||||
n, err := c.r.Read(p)
|
||||
c.n -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// parseChart reads the top-level <chart>/Chart.yaml from a packaged chart.
|
||||
func parseChart(r io.Reader) (*provider.HelmMetadata, error) {
|
||||
zr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tr := tar.NewReader(&capReader{r: zr, n: maxChartBytes})
|
||||
for entries := 0; ; entries++ {
|
||||
if entries >= maxChartEntries {
|
||||
return nil, fmt.Errorf("chart archive exceeds %d entries", maxChartEntries)
|
||||
}
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
return nil, errors.New("no Chart.yaml in chart archive")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir, file := path.Split(path.Clean(hdr.Name))
|
||||
if file != "Chart.yaml" || strings.Count(dir, "/") != 1 {
|
||||
continue
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(tr, maxChartYAML+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(raw) > maxChartYAML {
|
||||
return nil, fmt.Errorf("parse Chart.yaml: exceeds %d bytes", maxChartYAML)
|
||||
}
|
||||
var chart struct {
|
||||
Name string `yaml:"name"`
|
||||
Version string `yaml:"version"`
|
||||
}
|
||||
if err := yaml.Unmarshal(raw, &chart); err != nil {
|
||||
return nil, fmt.Errorf("parse Chart.yaml: %w", err)
|
||||
}
|
||||
if chart.Name == "" || chart.Version == "" {
|
||||
return nil, errors.New("chart metadata missing name or version")
|
||||
}
|
||||
return &provider.HelmMetadata{Name: chart.Name, Version: chart.Version, ChartYAML: string(raw)}, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,236 @@
|
||||
package helm
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"io"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func chartTgz(t *testing.T, files map[string]string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for name, body := range files {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write([]byte(body)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
type fakeBlobs struct{ data []byte }
|
||||
|
||||
func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type fakeStore struct {
|
||||
inserted *provider.HelmMetadata
|
||||
metas []provider.HelmMetadata
|
||||
}
|
||||
|
||||
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeStore) InsertHelmMetadata(_ context.Context, m *provider.HelmMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
func (f *fakeStore) ListHelmMetadataEntries(context.Context, string) ([]provider.HelmMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f *fakeStore) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for in, want := range map[string]string{
|
||||
"mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
|
||||
"charts/mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
|
||||
"../../mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
|
||||
} {
|
||||
sp, ct, err := p.ValidateUpload(in)
|
||||
if err != nil || sp != want || ct != "application/gzip" {
|
||||
t.Errorf("ValidateUpload(%q) = %q %q %v", in, sp, ct, err)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"index.yaml", "chart.tar.gz", ".tgz", ""} {
|
||||
if _, _, err := p.ValidateUpload(bad); err == nil {
|
||||
t.Errorf("ValidateUpload(%q) accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAfterUploadParsesTopLevelChart(t *testing.T) {
|
||||
data := chartTgz(t, map[string]string{
|
||||
"mychart/charts/sub/Chart.yaml": "apiVersion: v2\nname: sub\nversion: 9.9.9\n",
|
||||
"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0\ndescription: hi\n",
|
||||
"mychart/values.yaml": "a: 1\n",
|
||||
})
|
||||
store := &fakeStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "mychart-1.0.tgz", "sha256:ab", fakeBlobs{data}, store)
|
||||
m := store.inserted
|
||||
if m == nil || m.Name != "mychart" || m.Version != "1.0" || m.FilePath != "mychart-1.0.tgz" || m.ContentHash != "sha256:ab" {
|
||||
t.Fatalf("inserted %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAfterUploadRejectsChartWithoutMetadata(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\n"})
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{data}, store)
|
||||
if store.inserted != nil {
|
||||
t.Fatal("chart without version was recorded")
|
||||
}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{[]byte("not gzip")}, store)
|
||||
if store.inserted != nil {
|
||||
t.Fatal("non-gzip upload was recorded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeLocalIndex(t *testing.T) {
|
||||
t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
t2 := t1.Add(time.Hour)
|
||||
store := &fakeStore{metas: []provider.HelmMetadata{
|
||||
{FilePath: "mychart-1.0.tgz", ContentHash: "sha256:aa", Name: "mychart", Version: "1.0", CreatedAt: t1,
|
||||
ChartYAML: "apiVersion: v2\nname: mychart\nversion: 1.0\nappVersion: \"2\"\ndescription: hi\n"},
|
||||
{FilePath: "other-0.2.0.tgz", ContentHash: "sha256:bb", Name: "other", Version: "0.2.0", CreatedAt: t2,
|
||||
ChartYAML: "apiVersion: v2\nname: other\nversion: 0.2.0\n"},
|
||||
}}
|
||||
p := &Provider{}
|
||||
|
||||
if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), store, "r", "mychart-1.0.tgz") {
|
||||
t.Fatal("chart path handled as index")
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), store, "r", "./index.yaml") {
|
||||
t.Fatal("index.yaml not handled")
|
||||
}
|
||||
if rec.Code != 200 || rec.Header().Get("Content-Type") != "text/yaml" {
|
||||
t.Fatalf("code=%d ct=%q", rec.Code, rec.Header().Get("Content-Type"))
|
||||
}
|
||||
|
||||
var idx struct {
|
||||
APIVersion string `yaml:"apiVersion"`
|
||||
Generated string `yaml:"generated"`
|
||||
Entries map[string][]map[string]any
|
||||
}
|
||||
if err := yaml.Unmarshal(rec.Body.Bytes(), &idx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if idx.APIVersion != "v1" || idx.Generated != t2.Format(time.RFC3339Nano) {
|
||||
t.Errorf("header %+v", idx)
|
||||
}
|
||||
e := idx.Entries["mychart"][0]
|
||||
if e["digest"] != "aa" || e["description"] != "hi" || e["appVersion"] != "2" || e["apiVersion"] != "v2" {
|
||||
t.Errorf("entry %v", e)
|
||||
}
|
||||
if urls, _ := e["urls"].([]any); len(urls) != 1 || urls[0] != "mychart-1.0.tgz" {
|
||||
t.Errorf("urls %v", e["urls"])
|
||||
}
|
||||
if e["version"] != "1.0" || e["name"] != "mychart" {
|
||||
t.Errorf("version must stay a string: %#v", e["version"])
|
||||
}
|
||||
if len(idx.Entries["other"]) != 1 {
|
||||
t.Errorf("other entries %v", idx.Entries["other"])
|
||||
}
|
||||
|
||||
again, err := p.GenerateLocalIndex(context.Background(), store, "r", "index.yaml")
|
||||
if err != nil || !bytes.Equal(again, rec.Body.Bytes()) {
|
||||
t.Errorf("index not deterministic: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAfterUploadRejectsFilenameMismatch(t *testing.T) {
|
||||
store := &fakeStore{}
|
||||
data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0.0\n"})
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "other-1.0.0.tgz", "sha256:ab", fakeBlobs{data}, store)
|
||||
if store.inserted != nil {
|
||||
t.Fatal("chart indexed under a filename that does not match name-version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseChartLimits(t *testing.T) {
|
||||
chart := "apiVersion: v2\nname: c\nversion: 1.0.0\n"
|
||||
for name, tc := range map[string]struct {
|
||||
files map[string]string
|
||||
want string
|
||||
}{
|
||||
"decompressed": {map[string]string{"c/a.bin": strings.Repeat("\x00", maxChartBytes+1)}, "decompressed bytes"},
|
||||
"chart yaml": {map[string]string{"c/Chart.yaml": chart + "#" + strings.Repeat("x", maxChartYAML)}, "Chart.yaml: exceeds"},
|
||||
} {
|
||||
_, err := parseChart(bytes.NewReader(chartTgz(t, tc.files)))
|
||||
if err == nil || !strings.Contains(err.Error(), tc.want) {
|
||||
t.Errorf("%s: err = %v, want %q", name, err, tc.want)
|
||||
}
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for i := 0; i <= maxChartEntries; i++ {
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "c/templates/" + strconv.Itoa(i), Mode: 0o644})
|
||||
}
|
||||
_ = tw.WriteHeader(&tar.Header{Name: "c/Chart.yaml", Mode: 0o644, Size: int64(len(chart))})
|
||||
_, _ = tw.Write([]byte(chart))
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
if _, err := parseChart(&buf); err == nil || !strings.Contains(err.Error(), "entries") {
|
||||
t.Errorf("entries: err = %v", err)
|
||||
}
|
||||
|
||||
if m, err := parseChart(bytes.NewReader(chartTgz(t, map[string]string{"c/Chart.yaml": chart}))); err != nil || m.Name != "c" {
|
||||
t.Errorf("valid chart: %v %v", m, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexEscapesURLs(t *testing.T) {
|
||||
out, err := generateIndex([]provider.HelmMetadata{{FilePath: "my chart-1.0.0.tgz", Name: "my chart", Version: "1.0.0", ChartYAML: "name: x\n"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(out), "my%20chart-1.0.0.tgz") {
|
||||
t.Errorf("url not escaped:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexExpandsAliasesBeforeOverride(t *testing.T) {
|
||||
body, err := generateIndex([]provider.HelmMetadata{{
|
||||
FilePath: "app-1.0.0.tgz", ContentHash: "sha256:aa", Name: "app", Version: "1.0.0",
|
||||
ChartYAML: "apiVersion: v2\nname: app\nversion: &v 1.0.0\nappVersion: *v\n",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var idx struct {
|
||||
Entries map[string][]map[string]any
|
||||
}
|
||||
if err := yaml.Unmarshal(body, &idx); err != nil {
|
||||
t.Fatalf("index does not parse: %v\n%s", err, body)
|
||||
}
|
||||
if e := idx.Entries["app"]; len(e) != 1 || e[0]["appVersion"] != "1.0.0" {
|
||||
t.Errorf("entries %v", idx.Entries)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
|
||||
// Alpine-specific persistence surfaces. They are kept separate from the shared
|
||||
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
|
||||
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
|
||||
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
|
||||
// satisfies all three.
|
||||
type AlpineMetadataStore interface {
|
||||
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
|
||||
}
|
||||
|
||||
type AlpineMetadataDeleter interface {
|
||||
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type AlpineMetadataReader interface {
|
||||
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
|
||||
}
|
||||
|
||||
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
|
||||
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
|
||||
// sha1 of the control gzip stream) and the download/installed sizes.
|
||||
type AlpineMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
|
||||
Name string // P:
|
||||
Version string // V:
|
||||
Arch string // A:
|
||||
DownloadSize int64 // S: on-disk .apk size
|
||||
InstalledSize int64 // I: unpacked size from .PKGINFO
|
||||
Description string // T:
|
||||
URL string // U:
|
||||
License string // L:
|
||||
Origin string // o:
|
||||
Maintainer string // m:
|
||||
BuildTime int64 // t:
|
||||
Commit string // c:
|
||||
ProviderPriority string // k:
|
||||
Depends []string // D:
|
||||
Provides []string // p:
|
||||
InstallIf []string // i:
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DebMetadataReader is the read surface the deb LocalIndexer needs to
|
||||
// regenerate a flat apt repository (Packages/Release) from stored rows.
|
||||
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
|
||||
// mirroring how the rpm provider reaches its RPMMetadataReader.
|
||||
type DebMetadataReader interface {
|
||||
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
|
||||
}
|
||||
|
||||
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
|
||||
// the full raw control stanza so the Packages index can be regenerated
|
||||
// faithfully alongside the computed size/md5/sha256 apt requires.
|
||||
type DebMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Version string
|
||||
Architecture string
|
||||
Control string
|
||||
Size int64
|
||||
MD5 string
|
||||
SHA256 string
|
||||
// CreatedAt is the persisted insert time; the Release Date: is derived from
|
||||
// the newest value so the index is byte-identical across replicas and
|
||||
// regenerations (issue #117) rather than stamped from wall clock.
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HelmMetadataStore interface {
|
||||
InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error
|
||||
}
|
||||
|
||||
type HelmMetadataDeleter interface {
|
||||
DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type HelmMetadataReader interface {
|
||||
ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error)
|
||||
}
|
||||
|
||||
// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml
|
||||
// entries carry every field helm reads.
|
||||
type HelmMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Version string
|
||||
ChartYAML string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type PuppetMetadataStore interface {
|
||||
InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error
|
||||
}
|
||||
|
||||
type PuppetMetadataDeleter interface {
|
||||
DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type PuppetMetadataReader interface {
|
||||
ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error)
|
||||
}
|
||||
|
||||
// PuppetMetadata is a local module release; MetadataJSON is the tarball's
|
||||
// metadata.json verbatim so Forge release objects carry every field.
|
||||
type PuppetMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Slug string
|
||||
Version string
|
||||
MetadataJSON string
|
||||
Size int64
|
||||
MD5 string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RPMMetadataReader interface {
|
||||
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires []RPMDep
|
||||
Provides []RPMDep
|
||||
Conflicts []RPMDep
|
||||
Obsoletes []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
// CreatedAt is the persisted upload timestamp; used as a stable, replica-independent
|
||||
// value for the repodata <time>/<revision> fields so generated indexes are deterministic.
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type RPMDep struct {
|
||||
Name string `json:"name"`
|
||||
Flags string `json:"flags,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Release string `json:"release,omitempty"`
|
||||
}
|
||||
|
||||
type RPMFile struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type RPMChangelog struct {
|
||||
Author string `json:"author"`
|
||||
Date int64 `json:"date"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
|
||||
}
|
||||
|
||||
type MetadataDeleter interface {
|
||||
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
|
||||
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
@@ -37,6 +37,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewritesResponse() {}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" || !json.Valid(body) {
|
||||
return nil, nil
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Package nuget proxies a NuGet v3 feed: the service index, registration pages
|
||||
// and flat container indexes are mutable, .nupkg/.nuspec downloads immutable.
|
||||
// and flat container indexes are mutable; package, symbol, nuspec, readme and
|
||||
// icon blobs are immutable.
|
||||
package nuget
|
||||
|
||||
import (
|
||||
@@ -7,6 +8,7 @@ import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
pathpkg "path"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
@@ -34,10 +36,12 @@ func (p *Provider) Classify(path string) provider.Mutability {
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".nupkg"):
|
||||
case strings.HasSuffix(path, ".nupkg"), strings.HasSuffix(path, ".snupkg"), pathpkg.Base(path) == "icon":
|
||||
return "application/octet-stream"
|
||||
case strings.HasSuffix(path, ".nuspec"):
|
||||
return "application/xml"
|
||||
case pathpkg.Base(path) == "readme":
|
||||
return "text/markdown"
|
||||
}
|
||||
return "application/json"
|
||||
}
|
||||
@@ -64,8 +68,8 @@ func (p *Provider) RewriteServed(body []byte, remote models.Remote, proxyBaseURL
|
||||
if from == "" || !json.Valid(body) {
|
||||
return body
|
||||
}
|
||||
to := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
|
||||
return []byte(strings.ReplaceAll(string(body), from, to))
|
||||
to, _ := json.Marshal(strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name)
|
||||
return []byte(strings.ReplaceAll(string(body), from, string(to[1:len(to)-1])))
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
@@ -80,6 +84,14 @@ func origin(baseURL string) string {
|
||||
return u.Scheme + "://" + u.Host
|
||||
}
|
||||
|
||||
// isPackage matches flat container blobs: {id}.{ver}.(s)nupkg, {id}.nuspec and
|
||||
// the {id}/{ver}/readme and {id}/{ver}/icon files.
|
||||
func isPackage(path string) bool {
|
||||
return strings.HasSuffix(path, ".nupkg") || strings.HasSuffix(path, ".nuspec")
|
||||
for _, ext := range []string{".nupkg", ".snupkg", ".nuspec"} {
|
||||
if strings.HasSuffix(path, ext) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
base := pathpkg.Base(path)
|
||||
return (base == "readme" || base == "icon") && strings.Count(strings.Trim(path, "/"), "/") >= 2
|
||||
}
|
||||
|
||||
@@ -19,7 +19,11 @@ func TestClassify(t *testing.T) {
|
||||
}{
|
||||
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", provider.Immutable},
|
||||
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec", provider.Immutable},
|
||||
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.snupkg", provider.Immutable},
|
||||
{"v3-flatcontainer/newtonsoft.json/13.0.3/readme", provider.Immutable},
|
||||
{"v3-flatcontainer/newtonsoft.json/13.0.3/icon", provider.Immutable},
|
||||
{"v3-flatcontainer/newtonsoft.json/index.json", provider.Mutable},
|
||||
{"readme", provider.Mutable},
|
||||
{"v3/registration5-gz-semver2/newtonsoft.json/index.json", provider.Mutable},
|
||||
{"index.json", provider.Mutable},
|
||||
}
|
||||
@@ -30,6 +34,28 @@ func TestClassify(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"v3-flatcontainer/a/1.0.0/a.1.0.0.snupkg": "application/octet-stream",
|
||||
"v3-flatcontainer/a/1.0.0/icon": "application/octet-stream",
|
||||
"v3-flatcontainer/a/1.0.0/readme": "text/markdown",
|
||||
"v3-flatcontainer/a/index.json": "application/json",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteServedEscapesJSON(t *testing.T) {
|
||||
body := `{"@id":"https://api.nuget.org/v3-flatcontainer/"}`
|
||||
got := (&Provider{}).RewriteServed([]byte(body), nugetOrg, `http://evil"<x>`)
|
||||
if want := `{"@id":"http://evil\"\u003cx\u003e/api/v1/remote/nuget-org/v3-flatcontainer/"}`; string(got) != want {
|
||||
t.Errorf("RewriteServed = %s, want %s", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpstreamURL(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := []struct {
|
||||
|
||||
@@ -3,13 +3,14 @@ package provider
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// RemoteMountPath is the URL path remote proxies are served under.
|
||||
const RemoteMountPath = "/api/v1/remote/"
|
||||
|
||||
type Mutability int
|
||||
|
||||
const (
|
||||
@@ -31,205 +32,6 @@ type FileEntry struct {
|
||||
ContentHash string
|
||||
}
|
||||
|
||||
type FileStore interface {
|
||||
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
|
||||
ListPackages(ctx context.Context, repoName string) ([]string, error)
|
||||
}
|
||||
|
||||
type LocalUploader interface {
|
||||
ValidateUpload(filePath string) (storagePath, contentType string, err error)
|
||||
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
|
||||
}
|
||||
|
||||
type LocalIndexer interface {
|
||||
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
|
||||
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
|
||||
}
|
||||
|
||||
type BlobReader interface {
|
||||
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
|
||||
}
|
||||
|
||||
type PostUploadHook interface {
|
||||
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
|
||||
}
|
||||
|
||||
// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that
|
||||
// feeds generated repodata) after a local file is removed.
|
||||
type PostDeleteHook interface {
|
||||
AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error
|
||||
}
|
||||
|
||||
type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
|
||||
}
|
||||
|
||||
// RemoteServer lets a remote provider fully answer a request itself instead of
|
||||
// going through the byte-proxy engine. It is the remote-side analog of
|
||||
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
|
||||
// repodata from derived per-asset metadata and to redirect package downloads to
|
||||
// a backend remote, without ever precaching the packages. Returning false lets
|
||||
// the normal proxy path take over.
|
||||
type RemoteServer interface {
|
||||
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
|
||||
}
|
||||
|
||||
// ServeRewriter rewrites a cached mutable remote response per request, for
|
||||
// indexes whose URLs must name the requesting host and so cannot be rewritten
|
||||
// before caching.
|
||||
type ServeRewriter interface {
|
||||
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
|
||||
}
|
||||
|
||||
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
|
||||
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
|
||||
type RemoteMetadataStore interface {
|
||||
RPMMetadataReader
|
||||
MetadataStore
|
||||
MetadataDeleter
|
||||
}
|
||||
|
||||
type MetadataDeleter interface {
|
||||
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
|
||||
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type RPMMetadataReader interface {
|
||||
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
|
||||
}
|
||||
|
||||
// DebMetadataReader is the read surface the deb LocalIndexer needs to
|
||||
// regenerate a flat apt repository (Packages/Release) from stored rows.
|
||||
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
|
||||
// mirroring how the rpm provider reaches its RPMMetadataReader.
|
||||
type DebMetadataReader interface {
|
||||
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
|
||||
}
|
||||
|
||||
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
|
||||
// the full raw control stanza so the Packages index can be regenerated
|
||||
// faithfully alongside the computed size/md5/sha256 apt requires.
|
||||
type DebMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Version string
|
||||
Architecture string
|
||||
Control string
|
||||
Size int64
|
||||
MD5 string
|
||||
SHA256 string
|
||||
// CreatedAt is the persisted insert time; the Release Date: is derived from
|
||||
// the newest value so the index is byte-identical across replicas and
|
||||
// regenerations (issue #117) rather than stamped from wall clock.
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
|
||||
// Alpine-specific persistence surfaces. They are kept separate from the shared
|
||||
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
|
||||
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
|
||||
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
|
||||
// satisfies all three.
|
||||
type AlpineMetadataStore interface {
|
||||
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
|
||||
}
|
||||
|
||||
type AlpineMetadataDeleter interface {
|
||||
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type AlpineMetadataReader interface {
|
||||
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
|
||||
}
|
||||
|
||||
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
|
||||
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
|
||||
// sha1 of the control gzip stream) and the download/installed sizes.
|
||||
type AlpineMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
|
||||
Name string // P:
|
||||
Version string // V:
|
||||
Arch string // A:
|
||||
DownloadSize int64 // S: on-disk .apk size
|
||||
InstalledSize int64 // I: unpacked size from .PKGINFO
|
||||
Description string // T:
|
||||
URL string // U:
|
||||
License string // L:
|
||||
Origin string // o:
|
||||
Maintainer string // m:
|
||||
BuildTime int64 // t:
|
||||
Commit string // c:
|
||||
ProviderPriority string // k:
|
||||
Depends []string // D:
|
||||
Provides []string // p:
|
||||
InstallIf []string // i:
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Epoch int
|
||||
Version string
|
||||
Release string
|
||||
Arch string
|
||||
Summary string
|
||||
Description string
|
||||
RPMSize int64
|
||||
InstalledSize int64
|
||||
License string
|
||||
Vendor string
|
||||
Group string
|
||||
BuildHost string
|
||||
SourceRPM string
|
||||
URL string
|
||||
Packager string
|
||||
Requires []RPMDep
|
||||
Provides []RPMDep
|
||||
Conflicts []RPMDep
|
||||
Obsoletes []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
// CreatedAt is the persisted upload timestamp; used as a stable, replica-independent
|
||||
// value for the repodata <time>/<revision> fields so generated indexes are deterministic.
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type RPMDep struct {
|
||||
Name string `json:"name"`
|
||||
Flags string `json:"flags,omitempty"`
|
||||
Epoch string `json:"epoch,omitempty"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Release string `json:"release,omitempty"`
|
||||
}
|
||||
|
||||
type RPMFile struct {
|
||||
Path string `json:"path"`
|
||||
Type string `json:"type,omitempty"`
|
||||
}
|
||||
|
||||
type RPMChangelog struct {
|
||||
Author string `json:"author"`
|
||||
Date int64 `json:"date"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
var registry = map[models.PackageType]Provider{}
|
||||
|
||||
func Register(p Provider) {
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
)
|
||||
|
||||
const filesPrefix = "v3/files/"
|
||||
|
||||
// releaseRe matches <owner>-<name>-<semver>, the Forge release slug and the
|
||||
// tarball basename that `puppet module build` produces.
|
||||
var releaseRe = regexp.MustCompile(`^([a-z0-9]+)-([a-z][a-z0-9_]*)-(\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?)$`)
|
||||
|
||||
func ParseRelease(s string) (slug, version string, ok bool) {
|
||||
m := releaseRe.FindStringSubmatch(strings.ToLower(s))
|
||||
if m == nil {
|
||||
return "", "", false
|
||||
}
|
||||
return m[1] + "-" + m[2], m[3], true
|
||||
}
|
||||
|
||||
var slugRe = regexp.MustCompile(`^[a-z0-9]+-[a-z][a-z0-9_]*$`)
|
||||
|
||||
// ValidSlug reports whether s is a normalized <owner>-<name> module slug.
|
||||
func ValidSlug(s string) bool { return slugRe.MatchString(s) }
|
||||
|
||||
func NormalizeSlug(s string) string {
|
||||
return strings.ToLower(strings.ReplaceAll(s, "/", "-"))
|
||||
}
|
||||
|
||||
// ValidateUpload stores tarballs under v3/files/ so Forge file_uri downloads
|
||||
// resolve straight to the stored file.
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := strings.ToLower(path.Base(path.Clean("/" + filePath)))
|
||||
if _, _, ok := ParseRelease(strings.TrimSuffix(filename, ".tar.gz")); !ok || !strings.HasSuffix(filename, ".tar.gz") {
|
||||
return "", "", fmt.Errorf("file must be named <owner>-<module>-<version>.tar.gz")
|
||||
}
|
||||
return filesPrefix + filename, "application/gzip", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
return map[string]any{
|
||||
"filename": path.Base(storagePath),
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
reader, size, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
|
||||
if err != nil {
|
||||
slog.Error("puppet metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = reader.Close() }()
|
||||
if size > maxModuleTarball {
|
||||
slog.Error("puppet metadata: tarball too large", "repo", repoName, "path", storagePath, "size", size)
|
||||
return
|
||||
}
|
||||
|
||||
hash := md5.New()
|
||||
src := io.TeeReader(io.LimitReader(reader, maxModuleTarball), hash)
|
||||
meta, err := parseModule(src)
|
||||
if err != nil {
|
||||
slog.Error("puppet metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
if _, err := io.Copy(io.Discard, src); err != nil {
|
||||
slog.Error("puppet metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
slug, version, _ := ParseRelease(strings.TrimSuffix(path.Base(storagePath), ".tar.gz"))
|
||||
if meta.Slug != slug || meta.Version != version {
|
||||
slog.Error("puppet metadata: metadata.json does not match filename", "repo", repoName, "path", storagePath, "name", meta.Slug, "version", meta.Version)
|
||||
return
|
||||
}
|
||||
meta.RepoName = repoName
|
||||
meta.FilePath = storagePath
|
||||
meta.ContentHash = contentHash
|
||||
meta.Size = size
|
||||
meta.MD5 = hex.EncodeToString(hash.Sum(nil))
|
||||
|
||||
store, ok := db.(provider.PuppetMetadataStore)
|
||||
if !ok {
|
||||
slog.Error("puppet metadata: store does not support puppet metadata", "repo", repoName)
|
||||
return
|
||||
}
|
||||
if err := store.InsertPuppetMetadata(ctx, meta); err != nil {
|
||||
slog.Error("puppet metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("puppet metadata: parsed", "repo", repoName, "module", meta.Slug, "version", meta.Version)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
deleter, ok := db.(provider.PuppetMetadataDeleter)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return deleter.DeletePuppetMetadata(ctx, repoName, storagePath)
|
||||
}
|
||||
|
||||
const maxMetadataJSON = 1 << 20
|
||||
|
||||
var (
|
||||
maxModuleTarball int64 = 100 << 20
|
||||
maxModuleScan int64 = 200 << 20
|
||||
)
|
||||
|
||||
// parseModule reads the top-level <dir>/metadata.json from a module tarball,
|
||||
// scanning at most maxModuleScan decompressed bytes.
|
||||
func parseModule(r io.Reader) (*provider.PuppetMetadata, error) {
|
||||
zr, err := gzip.NewReader(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scan := &io.LimitedReader{R: zr, N: maxModuleScan}
|
||||
tr := tar.NewReader(scan)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if scan.N <= 0 {
|
||||
return nil, errors.New("module tarball exceeds decompressed scan limit before metadata.json")
|
||||
}
|
||||
if err == io.EOF {
|
||||
return nil, errors.New("no metadata.json in module tarball")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dir, file := path.Split(strings.TrimPrefix(path.Clean(hdr.Name), "./"))
|
||||
if hdr.Typeflag != tar.TypeReg || file != "metadata.json" || strings.Count(dir, "/") != 1 {
|
||||
continue
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(tr, maxMetadataJSON))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var md struct {
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &md); err != nil {
|
||||
return nil, fmt.Errorf("parse metadata.json: %w", err)
|
||||
}
|
||||
if md.Name == "" || md.Version == "" {
|
||||
return nil, errors.New("metadata.json missing name or version")
|
||||
}
|
||||
return &provider.PuppetMetadata{Slug: NormalizeSlug(md.Name), Version: md.Version, MetadataJSON: string(body)}, nil
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
clean := strings.TrimPrefix(path.Clean("/"+reqPath), "/")
|
||||
var list func(slug string) ([]provider.PuppetMetadata, error)
|
||||
if reader, ok := files.(provider.PuppetMetadataReader); ok {
|
||||
list = func(slug string) ([]provider.PuppetMetadata, error) {
|
||||
metas, err := reader.ListPuppetMetadataEntries(r.Context(), repoName, slug)
|
||||
sortReleases(metas)
|
||||
return metas, err
|
||||
}
|
||||
} else {
|
||||
list = func(string) ([]provider.PuppetMetadata, error) {
|
||||
return nil, errors.New("puppet metadata not available")
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case clean == "v3/releases":
|
||||
serveReleaseList(w, r, list)
|
||||
case strings.HasPrefix(clean, "v3/releases/"):
|
||||
slug, version, ok := ParseRelease(strings.TrimPrefix(clean, "v3/releases/"))
|
||||
if !ok {
|
||||
NotFound(w)
|
||||
return true
|
||||
}
|
||||
metas, err := list(slug)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
for _, m := range metas {
|
||||
if m.Version == version {
|
||||
WriteJSON(w, releaseObject(m))
|
||||
return true
|
||||
}
|
||||
}
|
||||
NotFound(w)
|
||||
case strings.HasPrefix(clean, "v3/modules/"):
|
||||
slug := NormalizeSlug(strings.TrimPrefix(clean, "v3/modules/"))
|
||||
metas, err := list(slug)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
if len(metas) == 0 {
|
||||
NotFound(w)
|
||||
return true
|
||||
}
|
||||
WriteJSON(w, moduleObject(metas))
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("puppet local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) ([]provider.PuppetMetadata, error)) {
|
||||
q := r.URL.Query()
|
||||
slug := NormalizeSlug(q.Get("module"))
|
||||
metas, err := list(slug)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
releases := make([]any, len(metas))
|
||||
for i, m := range metas {
|
||||
releases[i] = releaseObject(m)
|
||||
}
|
||||
WriteJSON(w, ReleasesPage(slug, releases, q))
|
||||
}
|
||||
|
||||
// ReleasesPage is one limit/offset page of a /v3/releases listing, with
|
||||
// pagination links relative to the Forge root.
|
||||
func ReleasesPage(slug string, releases []any, q url.Values) map[string]any {
|
||||
limit, err := strconv.Atoi(q.Get("limit"))
|
||||
if err != nil || limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
limit = min(limit, 100)
|
||||
offset, err := strconv.Atoi(q.Get("offset"))
|
||||
if err != nil || offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
offset = min(offset, len(releases))
|
||||
|
||||
page := func(off int) any {
|
||||
if off < 0 || off >= len(releases) && off != 0 {
|
||||
return nil
|
||||
}
|
||||
v := url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}}
|
||||
if slug != "" {
|
||||
v.Set("module", slug)
|
||||
}
|
||||
return "/v3/releases?" + v.Encode()
|
||||
}
|
||||
var previous any
|
||||
if offset > 0 {
|
||||
previous = page(max(offset-limit, 0))
|
||||
}
|
||||
return map[string]any{
|
||||
"pagination": map[string]any{
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
"first": page(0),
|
||||
"previous": previous,
|
||||
"current": page(offset),
|
||||
"next": page(offset + limit),
|
||||
"total": len(releases),
|
||||
},
|
||||
"results": releases[offset : offset+min(limit, len(releases)-offset)],
|
||||
}
|
||||
}
|
||||
|
||||
func owner(slug string) map[string]any {
|
||||
name, _, _ := strings.Cut(slug, "-")
|
||||
return map[string]any{"uri": "/v3/users/" + name, "slug": name, "username": name}
|
||||
}
|
||||
|
||||
func moduleRef(slug string) map[string]any {
|
||||
_, name, _ := strings.Cut(slug, "-")
|
||||
return map[string]any{"uri": "/v3/modules/" + slug, "slug": slug, "name": name, "deprecated_at": nil, "owner": owner(slug)}
|
||||
}
|
||||
|
||||
const forgeTime = "2006-01-02 15:04:05 -0700"
|
||||
|
||||
func releaseObject(m provider.PuppetMetadata) map[string]any {
|
||||
slug := m.Slug + "-" + m.Version
|
||||
return map[string]any{
|
||||
"uri": "/v3/releases/" + slug,
|
||||
"slug": slug,
|
||||
"module": moduleRef(m.Slug),
|
||||
"version": m.Version,
|
||||
"metadata": json.RawMessage(m.MetadataJSON),
|
||||
"tags": []string{},
|
||||
"supported": false,
|
||||
"pdk": false,
|
||||
"file_uri": "/" + m.FilePath,
|
||||
"file_size": m.Size,
|
||||
"file_md5": m.MD5,
|
||||
"file_sha256": strings.TrimPrefix(m.ContentHash, "sha256:"),
|
||||
"downloads": 0,
|
||||
"readme": nil,
|
||||
"changelog": nil,
|
||||
"license": nil,
|
||||
"created_at": m.CreatedAt.Format(forgeTime),
|
||||
"updated_at": m.CreatedAt.Format(forgeTime),
|
||||
"deleted_at": nil,
|
||||
}
|
||||
}
|
||||
|
||||
// moduleObject expects metas sorted newest version first.
|
||||
func moduleObject(metas []provider.PuppetMetadata) map[string]any {
|
||||
obj := moduleRef(metas[0].Slug)
|
||||
releases := make([]map[string]any, 0, len(metas))
|
||||
for _, m := range metas {
|
||||
slug := m.Slug + "-" + m.Version
|
||||
releases = append(releases, map[string]any{
|
||||
"uri": "/v3/releases/" + slug, "slug": slug, "version": m.Version,
|
||||
"file_uri": "/" + m.FilePath, "file_size": m.Size, "supported": false,
|
||||
"created_at": m.CreatedAt.Format(forgeTime), "deleted_at": nil,
|
||||
})
|
||||
}
|
||||
obj["current_release"] = releaseObject(metas[0])
|
||||
obj["releases"] = releases
|
||||
obj["downloads"] = 0
|
||||
obj["created_at"] = metas[len(metas)-1].CreatedAt.Format(forgeTime)
|
||||
obj["updated_at"] = metas[0].CreatedAt.Format(forgeTime)
|
||||
return obj
|
||||
}
|
||||
|
||||
func WriteJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetEscapeHTML(false)
|
||||
_ = enc.Encode(v)
|
||||
}
|
||||
|
||||
func NotFound(w http.ResponseWriter) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, `{"message":"404 Not Found","errors":["The requested resource could not be found"]}`)
|
||||
}
|
||||
|
||||
// sortReleases orders by module slug, then newest semver first.
|
||||
func sortReleases(metas []provider.PuppetMetadata) {
|
||||
slices.SortStableFunc(metas, func(a, b provider.PuppetMetadata) int {
|
||||
if c := strings.Compare(a.Slug, b.Slug); c != 0 {
|
||||
return c
|
||||
}
|
||||
return CompareSemver(b.Version, a.Version)
|
||||
})
|
||||
}
|
||||
|
||||
func CompareSemver(a, b string) int {
|
||||
a, _, _ = strings.Cut(a, "+")
|
||||
b, _, _ = strings.Cut(b, "+")
|
||||
aCore, aPre, aHasPre := strings.Cut(a, "-")
|
||||
bCore, bPre, bHasPre := strings.Cut(b, "-")
|
||||
if c := compareIdents(strings.Split(aCore, "."), strings.Split(bCore, ".")); c != 0 {
|
||||
return c
|
||||
}
|
||||
switch {
|
||||
case aHasPre && !bHasPre:
|
||||
return -1
|
||||
case !aHasPre && bHasPre:
|
||||
return 1
|
||||
}
|
||||
return compareIdents(strings.Split(aPre, "."), strings.Split(bPre, "."))
|
||||
}
|
||||
|
||||
func compareIdents(a, b []string) int {
|
||||
for i := 0; i < len(a) && i < len(b); i++ {
|
||||
an, aErr := strconv.Atoi(a[i])
|
||||
bn, bErr := strconv.Atoi(b[i])
|
||||
var c int
|
||||
switch {
|
||||
case aErr == nil && bErr == nil:
|
||||
c = an - bn
|
||||
case aErr == nil:
|
||||
c = -1
|
||||
case bErr == nil:
|
||||
c = 1
|
||||
default:
|
||||
c = strings.Compare(a[i], b[i])
|
||||
}
|
||||
if c != 0 {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return len(a) - len(b)
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
package puppet
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func moduleTarball(t *testing.T, dir, metadata string) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for name, body := range map[string]string{dir + "/metadata.json": metadata, dir + "/manifests/init.pp": "class hello {}\n"} {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _ = tw.Write([]byte(body))
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
const helloMetadata = `{"name":"unkin-hello","version":"0.1.0","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0"}]}`
|
||||
|
||||
type fakeBlobs struct{ data []byte }
|
||||
|
||||
func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type fakeStore struct{ inserted *provider.PuppetMetadata }
|
||||
|
||||
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeStore) InsertPuppetMetadata(_ context.Context, m *provider.PuppetMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeReader struct{ metas []provider.PuppetMetadata }
|
||||
|
||||
func (f fakeReader) ListPuppetMetadataEntries(_ context.Context, _, slug string) ([]provider.PuppetMetadata, error) {
|
||||
var out []provider.PuppetMetadata
|
||||
for _, m := range f.metas {
|
||||
if slug == "" || m.Slug == slug {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (fakeReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (fakeReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("some/dir/Unkin-hello-0.1.0.tar.gz")
|
||||
if err != nil || sp != "v3/files/unkin-hello-0.1.0.tar.gz" || ct != "application/gzip" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
for _, bad := range []string{"hello-0.1.0.tar.gz", "unkin-hello-0.1.tar.gz", "unkin-hello-0.1.0.zip", "unkin-Hello_x-0.1.0.tgz"} {
|
||||
if _, _, err := p.ValidateUpload(bad); err == nil {
|
||||
t.Errorf("%q accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAfterUploadRecordsMetadata(t *testing.T) {
|
||||
raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata)
|
||||
store := &fakeStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("nothing inserted")
|
||||
}
|
||||
sum := md5.Sum(raw)
|
||||
if m.Slug != "unkin-hello" || m.Version != "0.1.0" || m.MD5 != hex.EncodeToString(sum[:]) || m.Size != int64(len(raw)) || m.MetadataJSON != helloMetadata {
|
||||
t.Errorf("got %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAfterUploadRejectsMismatchedFilename(t *testing.T) {
|
||||
raw := moduleTarball(t, "unkin-hello-0.2.0", `{"name":"unkin-hello","version":"0.2.0"}`)
|
||||
store := &fakeStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("mismatched version indexed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseModuleRequiresTopLevelMetadata(t *testing.T) {
|
||||
if _, err := parseModule(bytes.NewReader(moduleTarball(t, "a/b", helloMetadata))); err == nil {
|
||||
t.Error("nested metadata.json accepted")
|
||||
}
|
||||
if _, err := parseModule(bytes.NewReader(moduleTarball(t, "x", `{"name":"unkin-hello"}`))); err == nil {
|
||||
t.Error("missing version accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func tarballOf(t *testing.T, entries ...*tar.Header) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
for _, h := range entries {
|
||||
if err := tw.WriteHeader(h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if h.Typeflag == tar.TypeReg {
|
||||
body := helloMetadata
|
||||
if !strings.HasSuffix(h.Name, "metadata.json") {
|
||||
body = strings.Repeat("x", int(h.Size))
|
||||
}
|
||||
_, _ = tw.Write([]byte(body))
|
||||
}
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = zw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestParseModuleSkipsNonRegularMetadata(t *testing.T) {
|
||||
raw := tarballOf(t,
|
||||
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"},
|
||||
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json/", Typeflag: tar.TypeDir, Mode: 0o755},
|
||||
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))},
|
||||
)
|
||||
meta, err := parseModule(bytes.NewReader(raw))
|
||||
if err != nil || meta.Slug != "unkin-hello" || meta.Version != "0.1.0" {
|
||||
t.Fatalf("got %v, %v", meta, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseModuleCapsDecompressedScan(t *testing.T) {
|
||||
defer func(n int64) { maxModuleScan = n }(maxModuleScan)
|
||||
maxModuleScan = 64 << 10
|
||||
raw := tarballOf(t,
|
||||
&tar.Header{Name: "unkin-hello-0.1.0/big", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1 << 20},
|
||||
&tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))},
|
||||
)
|
||||
if _, err := parseModule(bytes.NewReader(raw)); err == nil || !strings.Contains(err.Error(), "scan limit") {
|
||||
t.Fatalf("expected scan limit error, got %v", err)
|
||||
}
|
||||
maxModuleScan = 2 << 20
|
||||
if _, err := parseModule(bytes.NewReader(raw)); err != nil {
|
||||
t.Fatalf("under cap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAfterUploadCapsTarballSize(t *testing.T) {
|
||||
defer func(n int64) { maxModuleTarball = n }(maxModuleTarball)
|
||||
raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata)
|
||||
maxModuleTarball = int64(len(raw)) - 1
|
||||
store := &fakeStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("oversized tarball indexed")
|
||||
}
|
||||
maxModuleTarball = int64(len(raw))
|
||||
(&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store)
|
||||
if store.inserted == nil {
|
||||
t.Error("tarball at cap not indexed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseListClampsPagination(t *testing.T) {
|
||||
files := releases("0.1.0", "0.2.0", "0.3.0")
|
||||
for _, q := range []string{"offset=9223372036854775807", "offset=9223372036854775807&limit=9223372036854775807", "offset=-5&limit=-5", "limit=9223372036854775807"} {
|
||||
code, body := get(t, files, "v3/releases?module=unkin-hello&"+q)
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("%s: code %d", q, code)
|
||||
}
|
||||
pg := body["pagination"].(map[string]any)
|
||||
if l := pg["limit"].(float64); l < 1 || l > 100 {
|
||||
t.Errorf("%s: limit %v", q, l)
|
||||
}
|
||||
if o := pg["offset"].(float64); o < 0 || o > 3 {
|
||||
t.Errorf("%s: offset %v", q, o)
|
||||
}
|
||||
}
|
||||
_, body := get(t, files, "v3/releases?module=unkin-hello&offset=9223372036854775807")
|
||||
if len(body["results"].([]any)) != 0 || body["pagination"].(map[string]any)["next"] != nil {
|
||||
t.Errorf("past-end page: %v", body)
|
||||
}
|
||||
}
|
||||
|
||||
func get(t *testing.T, files provider.FileStore, target string) (int, map[string]any) {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x/"+target, nil)
|
||||
path, _, _ := strings.Cut(target, "?")
|
||||
if !(&Provider{}).ServeLocalIndex(rec, req, files, "forge", path) {
|
||||
return 0, nil
|
||||
}
|
||||
var body map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &body)
|
||||
return rec.Code, body
|
||||
}
|
||||
|
||||
func releases(versions ...string) fakeReader {
|
||||
var f fakeReader
|
||||
for i, v := range versions {
|
||||
f.metas = append(f.metas, provider.PuppetMetadata{
|
||||
FilePath: "v3/files/unkin-hello-" + v + ".tar.gz", ContentHash: "sha256:feed", Slug: "unkin-hello",
|
||||
Version: v, MetadataJSON: `{"name":"unkin-hello","version":"` + v + `"}`, MD5: "m" + v,
|
||||
CreatedAt: time.Unix(int64(i), 0),
|
||||
})
|
||||
}
|
||||
f.metas = append(f.metas, provider.PuppetMetadata{Slug: "other-mod", Version: "9.9.9", MetadataJSON: `{}`})
|
||||
return f
|
||||
}
|
||||
|
||||
func TestReleaseListPaginatesNewestFirst(t *testing.T) {
|
||||
files := releases("0.1.0", "0.10.0", "0.2.0", "1.0.0-rc.1", "1.0.0")
|
||||
code, body := get(t, files, "v3/releases?module=unkin/hello&limit=2")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("code %d", code)
|
||||
}
|
||||
results := body["results"].([]any)
|
||||
pg := body["pagination"].(map[string]any)
|
||||
if len(results) != 2 || results[0].(map[string]any)["version"] != "1.0.0" || results[1].(map[string]any)["version"] != "1.0.0-rc.1" {
|
||||
t.Fatalf("page 1: %v", results)
|
||||
}
|
||||
if pg["total"] != float64(5) || pg["next"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" || pg["previous"] != nil {
|
||||
t.Errorf("pagination: %v", pg)
|
||||
}
|
||||
|
||||
_, body = get(t, files, "v3/releases?module=unkin-hello&limit=2&offset=4")
|
||||
results = body["results"].([]any)
|
||||
pg = body["pagination"].(map[string]any)
|
||||
if len(results) != 1 || results[0].(map[string]any)["version"] != "0.1.0" || pg["next"] != nil || pg["previous"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" {
|
||||
t.Errorf("last page: %v %v", results, pg)
|
||||
}
|
||||
|
||||
_, body = get(t, files, "v3/releases?module=nobody-none")
|
||||
if len(body["results"].([]any)) != 0 {
|
||||
t.Error("unknown module returned results")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReleaseObject(t *testing.T) {
|
||||
code, body := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.1.0")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("code %d", code)
|
||||
}
|
||||
if body["file_uri"] != "/v3/files/unkin-hello-0.1.0.tar.gz" || body["file_md5"] != "m0.1.0" || body["file_sha256"] != "feed" {
|
||||
t.Errorf("release: %v", body)
|
||||
}
|
||||
if body["metadata"].(map[string]any)["name"] != "unkin-hello" || body["module"].(map[string]any)["owner"].(map[string]any)["username"] != "unkin" {
|
||||
t.Errorf("release refs: %v", body)
|
||||
}
|
||||
if code, _ := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.2.0"); code != http.StatusNotFound {
|
||||
t.Errorf("missing version: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleObject(t *testing.T) {
|
||||
code, body := get(t, releases("0.9.0", "0.10.0"), "v3/modules/unkin-hello")
|
||||
if code != http.StatusOK {
|
||||
t.Fatalf("code %d", code)
|
||||
}
|
||||
if body["slug"] != "unkin-hello" || body["name"] != "hello" || body["current_release"].(map[string]any)["version"] != "0.10.0" || len(body["releases"].([]any)) != 2 {
|
||||
t.Errorf("module: %v", body)
|
||||
}
|
||||
if code, _ := get(t, releases(), "v3/modules/unkin-missing"); code != http.StatusNotFound {
|
||||
t.Errorf("missing module: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilesFallThrough(t *testing.T) {
|
||||
if code, _ := get(t, releases(), "v3/files/unkin-hello-0.1.0.tar.gz"); code != 0 {
|
||||
t.Error("file downloads must fall through to the stored file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareSemver(t *testing.T) {
|
||||
ordered := []string{"0.1.0", "0.2.0", "0.10.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0", "1.0.1+build"}
|
||||
for i := 1; i < len(ordered); i++ {
|
||||
if CompareSemver(ordered[i-1], ordered[i]) >= 0 || CompareSemver(ordered[i], ordered[i-1]) <= 0 {
|
||||
t.Errorf("%s should sort before %s", ordered[i-1], ordered[i])
|
||||
}
|
||||
}
|
||||
if CompareSemver("1.0.0+a", "1.0.0+b") != 0 {
|
||||
t.Error("build metadata must not affect precedence")
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@ package puppet
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
@@ -39,6 +40,34 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
// ForwardedQuery keeps the module filter and pagination on release listings.
|
||||
func (p *Provider) ForwardedQuery(path string) map[string]provider.QueryValidator {
|
||||
if path != "v3/releases" {
|
||||
return nil
|
||||
}
|
||||
return map[string]provider.QueryValidator{"module": moduleParam, "limit": pageLimit, "offset": pageOffset}
|
||||
}
|
||||
|
||||
func moduleParam(v string) (string, bool) {
|
||||
v = NormalizeSlug(v)
|
||||
return v, ValidSlug(v)
|
||||
}
|
||||
|
||||
func pageLimit(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
if err != nil || n == 0 {
|
||||
return "", false
|
||||
}
|
||||
return strconv.FormatUint(min(n, 100), 10), true
|
||||
}
|
||||
|
||||
func pageOffset(v string) (string, bool) {
|
||||
n, err := strconv.ParseUint(v, 10, 32)
|
||||
return strconv.FormatUint(n, 10), err == nil
|
||||
}
|
||||
|
||||
func (p *Provider) RewritesResponse() {}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
|
||||
@@ -76,3 +76,28 @@ func TestAuthHeaders(t *testing.T) {
|
||||
t.Error("no credentials, no header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestForwardedQuery(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.ForwardedQuery("v3/files/a-b-1.0.0.tar.gz") != nil || p.ForwardedQuery("v3/modules/a-b") != nil {
|
||||
t.Fatal("only release listings forward a query")
|
||||
}
|
||||
f := p.ForwardedQuery("v3/releases")
|
||||
for name, c := range map[string]struct {
|
||||
in, want string
|
||||
ok bool
|
||||
}{
|
||||
"module": {"Puppetlabs/stdlib", "puppetlabs-stdlib", true},
|
||||
"limit": {"500", "100", true},
|
||||
"offset": {"40", "40", true},
|
||||
} {
|
||||
if got, ok := f[name](c.in); got != c.want || ok != c.ok {
|
||||
t.Errorf("%s(%q) = %q %v, want %q %v", name, c.in, got, ok, c.want, c.ok)
|
||||
}
|
||||
}
|
||||
for name, bad := range map[string]string{"module": "../x", "limit": "0", "offset": "-1"} {
|
||||
if _, ok := f[name](bad); ok {
|
||||
t.Errorf("%s(%q) accepted", name, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,6 +52,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewritesResponse() {}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if proxyBaseURL == "" {
|
||||
return nil, nil
|
||||
|
||||
@@ -343,6 +343,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
// A transiently failed asset fails the scan so the ETag is not advanced and
|
||||
// the retry re-derives it; an invalid package is skipped as no retry fixes it.
|
||||
var failed []error
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
@@ -373,10 +376,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
if !errors.Is(err, provider.ErrInvalidPackage) {
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := store.InsertRPMMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
|
||||
continue
|
||||
}
|
||||
slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
@@ -388,7 +395,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
|
||||
_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return newEtag, true, nil
|
||||
return newEtag, true, errors.Join(failed...)
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
@@ -569,12 +576,12 @@ func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote,
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("parse rpm header: %w", perr)
|
||||
return nil, fmt.Errorf("%w: parse rpm header: %w", provider.ErrInvalidPackage, perr)
|
||||
}
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
// response holds the whole object.
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
@@ -596,15 +603,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
return provider.ReadPrefix(resp, n)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
|
||||
@@ -627,7 +628,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"cmp"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -24,15 +26,22 @@ import (
|
||||
// fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
|
||||
// mirroring the (repo_name, file_path) uniqueness of the real table.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]provider.RPMMetadata
|
||||
mu sync.Mutex
|
||||
rows map[string]provider.RPMMetadata
|
||||
insertFail map[string]int // package name -> inserts left to fail
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
|
||||
func newFakeStore() *fakeStore {
|
||||
return &fakeStore{rows: map[string]provider.RPMMetadata{}, insertFail: map[string]int{}}
|
||||
}
|
||||
|
||||
func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.insertFail[m.Name] > 0 {
|
||||
f.insertFail[m.Name]--
|
||||
return errors.New("insert failed")
|
||||
}
|
||||
if _, ok := f.rows[m.FilePath]; ok {
|
||||
return nil // ON CONFLICT DO NOTHING
|
||||
}
|
||||
@@ -79,15 +88,20 @@ type githubFixture struct {
|
||||
releaseAuth string // Authorization header seen on the last releases request
|
||||
assetAuth string // Authorization header seen on the last asset request
|
||||
failStatus int // when set, the releases list answers this status (rate-limit style)
|
||||
assetFail map[string]int // asset filename -> downloads left to answer failCode
|
||||
failCode int // status for assetFail downloads; 0 = 500
|
||||
shortRead map[string]int // asset filename -> ranged GETs left to close early
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
rpmBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
rpmBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
assetFail: map[string]int{},
|
||||
shortRead: map[string]int{},
|
||||
}
|
||||
f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
|
||||
|
||||
@@ -145,6 +159,17 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
if f.assetFail[name] > 0 {
|
||||
f.assetFail[name]--
|
||||
code := cmp.Or(f.failCode, http.StatusInternalServerError)
|
||||
f.mu.Unlock()
|
||||
http.Error(w, "boom", code)
|
||||
return
|
||||
}
|
||||
short := rng != "" && f.shortRead[name] > 0
|
||||
if short {
|
||||
f.shortRead[name]--
|
||||
}
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
@@ -165,6 +190,9 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
end = len(body) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
|
||||
if short {
|
||||
end /= 2
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.Itoa(end+1))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(body[:end+1])
|
||||
|
||||
@@ -399,3 +399,74 @@ func TestSyncerFailedScanRetriesAfterBackoff(t *testing.T) {
|
||||
t.Fatalf("recovery did not refresh metadata: %d rows", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
// A transiently failed asset fails the scan: the ETag is not advanced, a retry
|
||||
// is scheduled, and the retry against the unchanged release list derives it. An
|
||||
// invalid package is skipped instead, since no retry can fix it.
|
||||
func TestSyncerFailedAssetRetry(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
corrupt bool
|
||||
failCode int
|
||||
short bool
|
||||
insert bool
|
||||
wantRetry bool
|
||||
}{
|
||||
{name: "transient", wantRetry: true},
|
||||
{name: "rate limited", failCode: http.StatusForbidden, wantRetry: true},
|
||||
{name: "short read", short: true, wantRetry: true},
|
||||
{name: "insert failure", insert: true, wantRetry: true},
|
||||
{name: "invalid package", corrupt: true},
|
||||
{name: "asset not found", failCode: http.StatusNotFound},
|
||||
{name: "asset gone", failCode: http.StatusGone},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
const other = "other-9-9.aarch64.rpm"
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
fx.rpmBytes[other] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
|
||||
store := newFakeSyncStore()
|
||||
switch {
|
||||
case tc.corrupt:
|
||||
fx.rpmBytes[other] = []byte("not a package")
|
||||
case tc.short:
|
||||
fx.shortRead[other] = 1
|
||||
case tc.insert:
|
||||
store.insertFail["other"] = 1
|
||||
case tc.failCode != 0 && !tc.wantRetry:
|
||||
fx.failCode, fx.assetFail[other] = tc.failCode, 1<<30
|
||||
default:
|
||||
fx.failCode, fx.assetFail[other] = tc.failCode, 1
|
||||
}
|
||||
s := newSyncer(store, newTestProvider(), testSyncConfig())
|
||||
remote := fx.remote()
|
||||
bg := context.Background()
|
||||
|
||||
s.process(bg, syncJob{remote: remote, prime: true})
|
||||
_, pending := store.retryAt[remote.Name]
|
||||
if pending != tc.wantRetry {
|
||||
t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
|
||||
}
|
||||
if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
|
||||
t.Fatalf("etag saved=%v after first scan", saved)
|
||||
}
|
||||
if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("healthy asset not served: %d rows", len(rows))
|
||||
}
|
||||
if !tc.wantRetry {
|
||||
return
|
||||
}
|
||||
|
||||
store.mu.Lock()
|
||||
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
|
||||
store.mu.Unlock()
|
||||
s.process(bg, syncJob{remote: remote})
|
||||
if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
|
||||
t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
|
||||
}
|
||||
if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 2 {
|
||||
t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,8 +3,10 @@ package provider
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -43,6 +45,49 @@ func NewUpstreamStatusError(url string, resp *http.Response) *UpstreamStatusErro
|
||||
return e
|
||||
}
|
||||
|
||||
// ErrInvalidPackage marks an asset whose bytes are not a valid package. It is
|
||||
// permanent, so a scan skips the asset instead of failing and retrying.
|
||||
var ErrInvalidPackage = errors.New("invalid package")
|
||||
|
||||
// AssetStatusError classifies a non-success asset download. A missing or
|
||||
// unsatisfiable asset is permanent; anything else (403/429 rate limits, 5xx) is
|
||||
// transient.
|
||||
func AssetStatusError(url string, resp *http.Response) error {
|
||||
switch resp.StatusCode {
|
||||
case http.StatusNotFound, http.StatusGone, http.StatusRequestedRangeNotSatisfiable:
|
||||
return fmt.Errorf("%w: %s: status %d", ErrInvalidPackage, url, resp.StatusCode)
|
||||
}
|
||||
return NewUpstreamStatusError(url, resp)
|
||||
}
|
||||
|
||||
// ReadPrefix reads up to n bytes of a 200 or 206 asset response. full reports
|
||||
// that the body is the whole object (per Content-Length or the Content-Range
|
||||
// total, or a clean EOF before n when the total is unknown); a body ending before n bytes that is not the whole object is a
|
||||
// transient short read.
|
||||
func ReadPrefix(resp *http.Response, n int64) (body []byte, full bool, err error) {
|
||||
body, err = io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
total := resp.ContentLength
|
||||
unknown := resp.StatusCode == http.StatusOK && total == -1
|
||||
if resp.StatusCode == http.StatusPartialContent {
|
||||
total = -1
|
||||
if _, t, ok := strings.Cut(resp.Header.Get("Content-Range"), "/"); ok {
|
||||
if v, perr := strconv.ParseInt(t, 10, 64); perr == nil {
|
||||
total = v
|
||||
}
|
||||
unknown = t == "*"
|
||||
}
|
||||
}
|
||||
// net/http errors on a premature close, so a clean EOF before n with an unknown total is the whole object.
|
||||
full = total == int64(len(body)) || (unknown && int64(len(body)) < n)
|
||||
if !full && int64(len(body)) < n {
|
||||
return nil, false, fmt.Errorf("short read: got %d of %d bytes", len(body), n)
|
||||
}
|
||||
return body, full, nil
|
||||
}
|
||||
|
||||
// SyncResult is a background scan's outcome, recorded when its sync lease is
|
||||
// released. A failed scan keeps the prior sync time and ETag and schedules a
|
||||
// retry after Backoff, doubled per consecutive failure up to MaxBackoff, and
|
||||
|
||||
@@ -3,7 +3,9 @@ package provider
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -61,3 +63,76 @@ func TestNewSyncResult(t *testing.T) {
|
||||
t.Fatalf("wrapped hint not clamped to ttl: retry in %v", until)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssetStatusErrorClassification(t *testing.T) {
|
||||
for status, permanent := range map[int]bool{
|
||||
http.StatusNotFound: true, http.StatusGone: true, http.StatusRequestedRangeNotSatisfiable: true,
|
||||
http.StatusForbidden: false, http.StatusTooManyRequests: false, http.StatusInternalServerError: false, http.StatusBadGateway: false,
|
||||
} {
|
||||
err := AssetStatusError("u", &http.Response{StatusCode: status, Header: http.Header{}})
|
||||
if errors.Is(err, ErrInvalidPackage) != permanent {
|
||||
t.Errorf("status %d: permanent=%v, want %v", status, !permanent, permanent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadPrefix(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
status int
|
||||
cl int64
|
||||
crange string
|
||||
body string
|
||||
n int64
|
||||
wantFull bool
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "200 whole object", status: 200, cl: 5, body: "hello", n: 32, wantFull: true},
|
||||
{name: "200 larger than n", status: 200, cl: 5, body: "hello", n: 4},
|
||||
{name: "200 chunked shorter than n", status: 200, cl: -1, body: "hello", n: 32, wantFull: true},
|
||||
{name: "200 chunked larger than n", status: 200, cl: -1, body: "hello", n: 4},
|
||||
{name: "206 whole object", status: 206, crange: "bytes 0-4/5", body: "hello", n: 32, wantFull: true},
|
||||
{name: "206 prefix", status: 206, crange: "bytes 0-3/100", body: "hell", n: 4},
|
||||
{name: "206 short read", status: 206, crange: "bytes 0-31/100", body: "hello", n: 32, wantErr: true},
|
||||
{name: "206 no content-range short", status: 206, body: "hello", n: 32, wantErr: true},
|
||||
{name: "206 unknown total shorter than n", status: 206, cl: -1, crange: "bytes 0-4/*", body: "hello", n: 32, wantFull: true},
|
||||
{name: "206 unknown total prefix", status: 206, cl: -1, crange: "bytes 0-3/*", body: "hell", n: 4},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
if tc.cl > 0 {
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(tc.cl, 10))
|
||||
}
|
||||
if tc.crange != "" {
|
||||
w.Header().Set("Content-Range", tc.crange)
|
||||
}
|
||||
w.WriteHeader(tc.status)
|
||||
_, _ = io.WriteString(w, tc.body)
|
||||
w.(http.Flusher).Flush()
|
||||
}))
|
||||
defer srv.Close()
|
||||
resp, err := http.Get(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if tc.cl == -1 && resp.ContentLength != -1 {
|
||||
t.Fatalf("response not chunked: ContentLength=%d", resp.ContentLength)
|
||||
}
|
||||
body, full, err := ReadPrefix(resp, tc.n)
|
||||
if (err != nil) != tc.wantErr {
|
||||
t.Fatalf("err=%v, wantErr=%v", err, tc.wantErr)
|
||||
}
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrInvalidPackage) {
|
||||
t.Fatalf("short read must be transient: %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if full != tc.wantFull || int64(len(body)) > tc.n {
|
||||
t.Fatalf("full=%v len=%d, want full=%v", full, len(body), tc.wantFull)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,6 +73,8 @@ func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewritesResponse() {}
|
||||
|
||||
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
|
||||
if remote.ReleasesRemote == "" {
|
||||
return nil, nil
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
package provider
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
const MaxAliasExpandedNodes = 100000
|
||||
|
||||
// ExpandYAMLAliases inlines alias targets and drops anchors so a node map
|
||||
// stays valid when copied away from the document that held its anchors;
|
||||
// budget caps nodes reached through aliases against alias bombs.
|
||||
func ExpandYAMLAliases(m map[string]yaml.Node, budget *int) (map[string]yaml.Node, error) {
|
||||
var expand func(n *yaml.Node, viaAlias bool) (*yaml.Node, error)
|
||||
expand = func(n *yaml.Node, viaAlias bool) (*yaml.Node, error) {
|
||||
if viaAlias {
|
||||
*budget--
|
||||
if *budget < 0 {
|
||||
return nil, errors.New("yaml aliases expand to too many nodes")
|
||||
}
|
||||
}
|
||||
if n.Kind == yaml.AliasNode {
|
||||
return expand(n.Alias, true)
|
||||
}
|
||||
c := *n
|
||||
c.Anchor = ""
|
||||
c.Content = make([]*yaml.Node, len(n.Content))
|
||||
for i, ch := range n.Content {
|
||||
var err error
|
||||
if c.Content[i], err = expand(ch, viaAlias); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
out := make(map[string]yaml.Node, len(m))
|
||||
for k, v := range m {
|
||||
e, err := expand(&v, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[k] = *e
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package proxy
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
@@ -37,7 +38,9 @@ func NewClassifier(p provider.Provider) *Classifier {
|
||||
return &Classifier{provider: p}
|
||||
}
|
||||
|
||||
// Classify judges the bare path; a forwarded query must not change the verdict.
|
||||
func (c *Classifier) Classify(remote models.Remote, path string) Classification {
|
||||
path, _, _ = strings.Cut(path, "?")
|
||||
if matchesAny(path, compilePatterns(remote.Blocklist)) {
|
||||
return ClassDenied
|
||||
}
|
||||
|
||||
@@ -38,6 +38,24 @@ func TestClassifierBranches(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassifierIgnoresForwardedQuery(t *testing.T) {
|
||||
gp, err := provider.Get(models.PackageGeneric)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := NewClassifier(gp)
|
||||
if got := c.Classify(models.Remote{Blocklist: []string{`\.exe$`}}, "x.exe?limit=1"); got != ClassDenied {
|
||||
t.Errorf("blocklist bypassed by query: %v", got)
|
||||
}
|
||||
allow := models.Remote{Patterns: []string{`/versions/$`}}
|
||||
if got := c.Classify(allow, "a/versions/?limit=1&offset=100"); got != ClassImmutable {
|
||||
t.Errorf("allowlisted paginated path = %v, want immutable", got)
|
||||
}
|
||||
if got := c.Classify(models.Remote{MutablePatterns: []string{`/versions/$`}}, "a/versions/?offset=100"); got != ClassMutable {
|
||||
t.Errorf("mutable pattern on paginated path = %v, want mutable", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClassificationString(t *testing.T) {
|
||||
for c, want := range map[Classification]string{
|
||||
ClassImmutable: "immutable",
|
||||
|
||||
+52
-25
@@ -134,10 +134,11 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
|
||||
slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path)
|
||||
}
|
||||
|
||||
locked, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
|
||||
token, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
|
||||
if err != nil {
|
||||
slog.Warn("lock acquire failed", "error", err)
|
||||
}
|
||||
locked := token != ""
|
||||
|
||||
if !locked {
|
||||
// Another request holds the fetch lock. Poll the store until the leader
|
||||
@@ -151,7 +152,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
|
||||
}
|
||||
|
||||
if locked {
|
||||
defer e.cache.ReleaseLock(ctx, remote.Name, path)
|
||||
defer func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remote.Name, path, token) }()
|
||||
}
|
||||
|
||||
if class == ClassMutable && remote.CheckMutable {
|
||||
@@ -160,7 +161,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
|
||||
notModified, err := e.checkUpstream(ctx, remote, path, etag, prov)
|
||||
if err == nil && notModified {
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, etagTTL(ttl))
|
||||
result, err := e.serveFromStore(ctx, remote, path)
|
||||
if err == nil {
|
||||
result.Source = "cache"
|
||||
@@ -216,7 +217,8 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
|
||||
|
||||
// Evict drops path from every cache layer (artifact row, index object, Redis
|
||||
// freshness and ETag keys) so the next request refetches from upstream. A
|
||||
// trailing "/*" evicts every path under that directory.
|
||||
// trailing "/*" evicts every path under that directory; a plain path also
|
||||
// drops its forwarded-query variants ("path?...").
|
||||
func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
|
||||
prefix, wildcard := strings.CutSuffix(path, "*")
|
||||
if wildcard && !strings.HasSuffix(prefix, "/") {
|
||||
@@ -228,21 +230,34 @@ func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
|
||||
return fmt.Errorf("get remote: %w", err)
|
||||
}
|
||||
if !wildcard {
|
||||
if err := e.waitForLock(ctx, remoteName, path); err != nil {
|
||||
release, err := e.Lock(ctx, remoteName, path, e.evictLockWait, fetchLockTTL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remoteName, path) }()
|
||||
defer release()
|
||||
if err := e.db.DeleteArtifact(ctx, remoteName, path); err != nil {
|
||||
return fmt.Errorf("delete artifact: %w", err)
|
||||
}
|
||||
if err := e.store.Delete(ctx, storage.IndexKey(remoteName, path)); err != nil {
|
||||
return fmt.Errorf("delete index: %w", err)
|
||||
}
|
||||
return e.cache.ForgetPath(ctx, remoteName, path)
|
||||
if err := e.cache.ForgetPath(ctx, remoteName, path); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.Contains(path, "?") {
|
||||
return nil
|
||||
}
|
||||
return e.evictPrefix(ctx, remoteName, path+"?")
|
||||
}
|
||||
// ponytail: no lock for wildcards; a Fetch already in flight under the
|
||||
// prefix can re-cache its path after the evict. Per-path locks over a
|
||||
// directory would close it if that ever matters.
|
||||
return e.evictPrefix(ctx, remoteName, prefix)
|
||||
}
|
||||
|
||||
// evictPrefix drops every cached path starting with prefix, including the
|
||||
// forwarded-query variants ("path?...") of a single path.
|
||||
func (e *Engine) evictPrefix(ctx context.Context, remoteName, prefix string) error {
|
||||
if err := e.db.DeleteArtifactsByPrefix(ctx, remoteName, prefix); err != nil {
|
||||
return fmt.Errorf("delete artifacts: %w", err)
|
||||
}
|
||||
@@ -252,28 +267,36 @@ func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
|
||||
return e.cache.ForgetPrefix(ctx, remoteName, prefix)
|
||||
}
|
||||
|
||||
// waitForLock takes the per-path fetch lock so an in-flight Fetch cannot
|
||||
// re-set TTL/ETag keys after an evict. It fails with a 503 when the lock
|
||||
// cannot be taken within evictLockWait or Redis errors.
|
||||
func (e *Engine) waitForLock(ctx context.Context, remoteName, path string) error {
|
||||
deadline := time.Now().Add(e.evictLockWait)
|
||||
// Lock takes the per-path Redis lock (the one Fetch holds) and returns its
|
||||
// release. It fails with a 503 when the lock cannot be taken within wait or
|
||||
// Redis errors.
|
||||
func (e *Engine) Lock(ctx context.Context, remoteName, path string, wait, ttl time.Duration) (func(), error) {
|
||||
token, err := e.waitForLock(ctx, remoteName, path, wait, ttl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remoteName, path, token) }, nil
|
||||
}
|
||||
|
||||
func (e *Engine) waitForLock(ctx context.Context, remoteName, path string, wait, ttl time.Duration) (string, error) {
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
ok, err := e.cache.AcquireLock(ctx, remoteName, path, fetchLockTTL)
|
||||
if ok {
|
||||
return nil
|
||||
token, err := e.cache.AcquireLock(ctx, remoteName, path, ttl)
|
||||
if token != "" {
|
||||
return token, nil
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
return "", ctx.Err()
|
||||
}
|
||||
if err != nil {
|
||||
return &ProxyError{Status: http.StatusServiceUnavailable, Message: fmt.Sprintf("fetch lock: %v", err)}
|
||||
return "", &ProxyError{Status: http.StatusServiceUnavailable, Message: fmt.Sprintf("lock: %v", err)}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return &ProxyError{Status: http.StatusServiceUnavailable, Message: "fetch in progress, retry evict"}
|
||||
return "", &ProxyError{Status: http.StatusServiceUnavailable, Message: "resource busy, retry later"}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
return "", ctx.Err()
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
@@ -353,7 +376,7 @@ func (e *Engine) headUpstreamOnce(ctx context.Context, remote models.Remote, pat
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
}
|
||||
return http.DefaultClient.Do(req)
|
||||
return clientForRemote(remote).Do(req)
|
||||
}
|
||||
|
||||
resp, err := doHead(nil)
|
||||
@@ -491,8 +514,9 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
|
||||
}
|
||||
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
if etag := resp.Header.Get("ETag"); etag != "" {
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
// A zero TTL never expires, so it never needs revalidating.
|
||||
if remote.CheckMutable && ttl > 0 {
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, resp.Header.Get("ETag"), etagTTL(ttl))
|
||||
}
|
||||
|
||||
return &FetchResult{
|
||||
@@ -520,9 +544,6 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
|
||||
}
|
||||
|
||||
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
|
||||
if etag := resp.Header.Get("ETag"); etag != "" {
|
||||
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
|
||||
}
|
||||
|
||||
reader, info, err := e.store.Download(ctx, casResult.S3Key)
|
||||
if err != nil {
|
||||
@@ -642,6 +663,12 @@ func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, pa
|
||||
return resp.StatusCode == http.StatusNotModified, nil
|
||||
}
|
||||
|
||||
// etagTTL keeps a mutable path's ETag well past its freshness key so an expired
|
||||
// path revalidates with a conditional HEAD, while idle paths still age out.
|
||||
func etagTTL(ttl time.Duration) time.Duration {
|
||||
return max(10*ttl, 7*24*time.Hour)
|
||||
}
|
||||
|
||||
func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration {
|
||||
switch class {
|
||||
case ClassImmutable:
|
||||
|
||||
+173
-17
@@ -17,12 +17,14 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
goredis "github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
var (
|
||||
testEngine *Engine
|
||||
testCache *cache.Redis
|
||||
testDB *database.DB
|
||||
testRedis string
|
||||
upstream *httptest.Server
|
||||
)
|
||||
|
||||
@@ -64,6 +66,7 @@ func TestMain(m *testing.M) {
|
||||
}
|
||||
|
||||
testCache = redis
|
||||
testRedis = redisURL
|
||||
testDB = db
|
||||
testEngine = NewEngine(db, redis, s3)
|
||||
upstream = httptest.NewServer(http.HandlerFunc(mockUpstream))
|
||||
@@ -216,6 +219,20 @@ func TestHead(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeadUpstreamUsesRemoteTimeout(t *testing.T) {
|
||||
slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(3 * time.Second)
|
||||
}))
|
||||
defer slow.Close()
|
||||
r := models.Remote{Name: "slow-head", PackageType: models.PackageGeneric, BaseURL: slow.URL, UpstreamResponseHeaderTimeout: 1}
|
||||
|
||||
start := time.Now()
|
||||
_, err := (&Engine{}).headUpstreamOnce(context.Background(), r, "blob.bin", prov(t, models.PackageGeneric))
|
||||
if err == nil || time.Since(start) > 2*time.Second {
|
||||
t.Fatalf("HEAD ignored the remote's response header timeout: err=%v after %s", err, time.Since(start))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleOnError(t *testing.T) {
|
||||
requireStack(t)
|
||||
ctx := context.Background()
|
||||
@@ -260,30 +277,169 @@ func TestCircuitOpenServesStale(t *testing.T) {
|
||||
res.Reader.Close()
|
||||
}
|
||||
|
||||
// expire drops path's freshness key so the next fetch revalidates.
|
||||
func expire(t *testing.T, remote, path string) {
|
||||
t.Helper()
|
||||
if err := testCache.SetTTL(context.Background(), remote, path, time.Millisecond); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
func mutableFetch(t *testing.T, r models.Remote, path string) (string, string) {
|
||||
t.Helper()
|
||||
res, err := testEngine.Fetch(context.Background(), r, path, prov(t, models.PackageRPM))
|
||||
if err != nil {
|
||||
t.Fatalf("fetch: %v", err)
|
||||
}
|
||||
return readAll(t, res), res.Source
|
||||
}
|
||||
|
||||
const repomd = "8/Everything/x86_64/repodata/repomd.xml"
|
||||
|
||||
func TestMutableRevalidation(t *testing.T) {
|
||||
requireStack(t)
|
||||
srv, rev, conditional := changingUpstream(t)
|
||||
r := seed(t, models.Remote{Name: "eng-reval", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
|
||||
|
||||
if body, _ := mutableFetch(t, r, repomd); body != "rev1:/"+repomd {
|
||||
t.Fatalf("initial fetch = %q", body)
|
||||
}
|
||||
expire(t, r.Name, repomd)
|
||||
if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" {
|
||||
t.Errorf("after expiry = %q from %s, want cached rev1", body, src)
|
||||
}
|
||||
if n := conditional.Load(); n != 1 {
|
||||
t.Errorf("conditional requests = %d, want 1 revalidation", n)
|
||||
}
|
||||
if fresh, _ := testCache.CheckTTL(context.Background(), r.Name, repomd); !fresh {
|
||||
t.Error("304 should re-arm the freshness key")
|
||||
}
|
||||
|
||||
rev.Store("rev2")
|
||||
expire(t, r.Name, repomd)
|
||||
if body, src := mutableFetch(t, r, repomd); body != "rev2:/"+repomd || src != "remote" {
|
||||
t.Errorf("after upstream change = %q from %s, want remote rev2", body, src)
|
||||
}
|
||||
}
|
||||
|
||||
func etagKeyTTL(t *testing.T, remote, path string) time.Duration {
|
||||
t.Helper()
|
||||
opts, err := goredis.ParseURL(testRedis)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
c := goredis.NewClient(opts)
|
||||
defer func() { _ = c.Close() }()
|
||||
d, err := c.PTTL(context.Background(), "etag:"+remote+":"+path).Result()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// The ETag is only kept when it can be used, and never without an expiry.
|
||||
func TestMutableETagStorage(t *testing.T) {
|
||||
requireStack(t)
|
||||
srv, _, _ := changingUpstream(t)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
check bool
|
||||
ttl int
|
||||
want time.Duration
|
||||
}{
|
||||
{"eng-etag-off", false, 60, -2},
|
||||
{"eng-etag-never", true, 0, -2},
|
||||
{"eng-etag-short", true, 60, 7 * 24 * time.Hour},
|
||||
{"eng-etag-long", true, 86400, 10 * 24 * time.Hour},
|
||||
} {
|
||||
r := seed(t, models.Remote{Name: tc.name, PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: tc.check, MutableTTL: tc.ttl})
|
||||
mutableFetch(t, r, repomd)
|
||||
got := etagKeyTTL(t, tc.name, repomd)
|
||||
if tc.want < 0 {
|
||||
if got != tc.want {
|
||||
t.Errorf("%s: etag key ttl = %v, want no key", tc.name, got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if got <= tc.want-time.Minute || got > tc.want {
|
||||
t.Errorf("%s: etag key ttl = %v, want ~%v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Immutable paths are never revalidated, so their upstream ETag is not kept.
|
||||
func TestImmutableFetchStoresNoETag(t *testing.T) {
|
||||
requireStack(t)
|
||||
srv, _, _ := changingUpstream(t)
|
||||
r := seed(t, models.Remote{Name: "eng-etag-imm", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
|
||||
const rpm = "Packages/e2e-1.0-1.noarch.rpm"
|
||||
if body, src := mutableFetch(t, r, rpm); body != "rev1:/"+rpm || src != "remote" {
|
||||
t.Fatalf("fetch = %q from %s", body, src)
|
||||
}
|
||||
if got := etagKeyTTL(t, r.Name, rpm); got != -2 {
|
||||
t.Errorf("etag key ttl = %v, want no key", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A 304 for a path whose S3 body is gone falls back to a full refetch.
|
||||
func TestNotModifiedWithEvictedBodyRefetches(t *testing.T) {
|
||||
requireStack(t)
|
||||
srv, _, conditional := changingUpstream(t)
|
||||
r := seed(t, models.Remote{Name: "eng-304-gone", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60})
|
||||
|
||||
mutableFetch(t, r, repomd)
|
||||
if err := testEngine.store.Delete(context.Background(), storage.IndexKey(r.Name, repomd)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expire(t, r.Name, repomd)
|
||||
if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "remote" {
|
||||
t.Errorf("got %q from %s, want refetched rev1 from remote", body, src)
|
||||
}
|
||||
if n := conditional.Load(); n != 1 {
|
||||
t.Errorf("conditional requests = %d, want the 304 HEAD first", n)
|
||||
}
|
||||
}
|
||||
|
||||
// A failing revalidation HEAD still serves the stored copy with stale_on_error.
|
||||
func TestStaleOnErrorWhenRevalidationFails(t *testing.T) {
|
||||
requireStack(t)
|
||||
srv, _, _ := changingUpstream(t)
|
||||
r := seed(t, models.Remote{Name: "eng-stale-head", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60, StaleOnError: true})
|
||||
|
||||
mutableFetch(t, r, repomd)
|
||||
srv.Close()
|
||||
expire(t, r.Name, repomd)
|
||||
if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" {
|
||||
t.Errorf("got %q from %s, want stale rev1 from cache", body, src)
|
||||
}
|
||||
}
|
||||
|
||||
// After a remote is deleted and flushed, a re-created remote of the same name
|
||||
// fetches from its own upstream instead of revalidating the old ETag.
|
||||
func TestRecreatedRemoteHasNoStaleState(t *testing.T) {
|
||||
requireStack(t)
|
||||
ctx := context.Background()
|
||||
r := seed(t, models.Remote{Name: "eng-npm", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true})
|
||||
p := prov(t, models.PackageNPM)
|
||||
oldSrv, _, _ := changingUpstream(t)
|
||||
r := seed(t, models.Remote{Name: "eng-recreate", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: oldSrv.URL, CheckMutable: true, MutableTTL: 60})
|
||||
mutableFetch(t, r, repomd)
|
||||
|
||||
res, err := testEngine.Fetch(ctx, r, "pkg", p)
|
||||
if err != nil {
|
||||
t.Fatalf("initial mutable fetch: %v", err)
|
||||
if err := testDB.DeleteRemote(ctx, r.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
res.Reader.Close()
|
||||
|
||||
// Expire only the freshness marker; the ETag persists, forcing a
|
||||
// conditional revalidation that the upstream answers with 304.
|
||||
testCache.SetTTL(ctx, "eng-npm", "pkg", time.Millisecond)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
res, err = testEngine.Fetch(ctx, r, "pkg", p)
|
||||
if err != nil {
|
||||
t.Fatalf("revalidation fetch: %v", err)
|
||||
if err := testCache.FlushRemote(ctx, r.Name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if res.Source != "cache" {
|
||||
t.Errorf("revalidated response should come from cache, got %s", res.Source)
|
||||
newSrv, rev, conditional := changingUpstream(t)
|
||||
rev.Store("rev9")
|
||||
r.BaseURL = newSrv.URL
|
||||
seed(t, r)
|
||||
if body, src := mutableFetch(t, r, repomd); body != "rev9:/"+repomd || src != "remote" {
|
||||
t.Errorf("got %q from %s, want rev9 from the new upstream", body, src)
|
||||
}
|
||||
if n := conditional.Load(); n != 0 {
|
||||
t.Errorf("conditional requests = %d, want none after flush", n)
|
||||
}
|
||||
res.Reader.Close()
|
||||
}
|
||||
|
||||
func TestBearerTokenFlow(t *testing.T) {
|
||||
|
||||
@@ -59,7 +59,7 @@ type cached struct{ row, index, ttl, etag bool }
|
||||
// Mutable indexes live in the S3 index; immutable blobs get an artifact row.
|
||||
var (
|
||||
indexCached = cached{index: true, ttl: true, etag: true}
|
||||
blobCached = cached{row: true, ttl: true, etag: true}
|
||||
blobCached = cached{row: true, ttl: true}
|
||||
)
|
||||
|
||||
func layers(t *testing.T, remote, path string) cached {
|
||||
@@ -152,13 +152,15 @@ func TestEvictUnknownRemote(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func holdLock(t *testing.T, remote, path string) {
|
||||
func holdLock(t *testing.T, remote, path string) string {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
if ok, err := testCache.AcquireLock(ctx, remote, path, time.Minute); !ok || err != nil {
|
||||
t.Fatalf("acquire: %v %v", ok, err)
|
||||
token, err := testCache.AcquireLock(ctx, remote, path, time.Minute)
|
||||
if token == "" || err != nil {
|
||||
t.Fatalf("acquire: %q %v", token, err)
|
||||
}
|
||||
t.Cleanup(func() { _ = testCache.ReleaseLock(ctx, remote, path) })
|
||||
t.Cleanup(func() { _ = testCache.ReleaseLock(ctx, remote, path, token) })
|
||||
return token
|
||||
}
|
||||
|
||||
func TestEvictWaitsForFetchLock(t *testing.T) {
|
||||
@@ -167,7 +169,7 @@ func TestEvictWaitsForFetchLock(t *testing.T) {
|
||||
r := rpmRemote(t, "evict-lock", srv.URL)
|
||||
ctx := context.Background()
|
||||
const path = "8/Everything/x86_64/repodata/repomd.xml"
|
||||
holdLock(t, r.Name, path)
|
||||
token := holdLock(t, r.Name, path)
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- testEngine.Evict(ctx, r.Name, path) }()
|
||||
select {
|
||||
@@ -175,13 +177,13 @@ func TestEvictWaitsForFetchLock(t *testing.T) {
|
||||
t.Fatalf("evict returned while fetch lock held: %v", err)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
_ = testCache.ReleaseLock(ctx, r.Name, path)
|
||||
_ = testCache.ReleaseLock(ctx, r.Name, path, token)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("evict: %v", err)
|
||||
}
|
||||
ok, err := testCache.AcquireLock(ctx, r.Name, path, time.Second)
|
||||
if !ok || err != nil {
|
||||
t.Fatalf("lock still held after evict: %v %v", ok, err)
|
||||
token, err := testCache.AcquireLock(ctx, r.Name, path, time.Second)
|
||||
if token == "" || err != nil {
|
||||
t.Fatalf("lock still held after evict: %q %v", token, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,3 +271,28 @@ func TestEvictWildcardClearsPrefixOnly(t *testing.T) {
|
||||
t.Errorf("path outside prefix = %q, want cached rev1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvictPathDropsQueryVariants(t *testing.T) {
|
||||
requireStack(t)
|
||||
srv, _, _ := changingUpstream(t)
|
||||
r := rpmRemote(t, "evict-query", srv.URL)
|
||||
const (
|
||||
path = "8/Everything/x86_64/repodata/repomd.xml"
|
||||
variant = path + "?limit=10"
|
||||
sibling = "8/Everything/x86_64/repodata/repomd.xml.asc"
|
||||
)
|
||||
for _, p := range []string{path, variant, sibling} {
|
||||
fetchBody(t, r, p)
|
||||
}
|
||||
if err := testEngine.Evict(context.Background(), r.Name, path); err != nil {
|
||||
t.Fatalf("evict: %v", err)
|
||||
}
|
||||
for _, p := range []string{path, variant} {
|
||||
if c := layers(t, r.Name, p); c != (cached{}) {
|
||||
t.Errorf("%s after evict = %+v, want every layer gone", p, c)
|
||||
}
|
||||
}
|
||||
if c := layers(t, r.Name, sibling); c == (cached{}) {
|
||||
t.Errorf("%s sharing the name prefix was evicted", sibling)
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user