From 48c127e5a0bd53ca0175f3a831d0ccad34fc6231 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 24 Oct 2026 16:31:02 +1100 Subject: [PATCH 1/4] Add vcs remote type for git hosting archives and release assets --- README.md | 1 + docs/common/caching.md | 2 +- docs/vcs/remote.md | 29 ++ e2e-docker/caching_vcs_test.go | 7 + .../octo/repo/archive/refs/tags/v1.0.0.tar.gz | Bin 0 -> 154 bytes internal/provider/all/vcs.go | 3 + internal/provider/vcs/vcs.go | 263 ++++++++++++++++++ internal/provider/vcs/vcs_test.go | 230 +++++++++++++++ pkg/models/package_type_vcs.go | 5 + ui/src/packagetypes/defs/vcs.ts | 26 ++ 10 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 docs/vcs/remote.md create mode 100644 e2e-docker/caching_vcs_test.go create mode 100644 e2e-docker/fixtures/octo/repo/archive/refs/tags/v1.0.0.tar.gz create mode 100644 internal/provider/all/vcs.go create mode 100644 internal/provider/vcs/vcs.go create mode 100644 internal/provider/vcs/vcs_test.go create mode 100644 pkg/models/package_type_vcs.go create mode 100644 ui/src/packagetypes/defs/vcs.ts diff --git a/README.md b/README.md index bc7262d..616dbb5 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/< - `pypi`: [remote](docs/pypi/remote.md) - `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md) - `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md) +- `vcs`: [remote](docs/vcs/remote.md) ### GitHub authentication diff --git a/docs/common/caching.md b/docs/common/caching.md index 223cc75..3b32604 100644 --- a/docs/common/caching.md +++ b/docs/common/caching.md @@ -98,4 +98,4 @@ Each type's page lists its auto-detected mutable and immutable paths: [docker](../docker/remote.md), [generic](../generic/remote.md), [gems](../gems/remote.md), [goproxy](../goproxy/remote.md), [helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md), [puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md), -[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md). +[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md), [vcs](../vcs/remote.md). diff --git a/docs/vcs/remote.md b/docs/vcs/remote.md new file mode 100644 index 0000000..501510f --- /dev/null +++ b/docs/vcs/remote.md @@ -0,0 +1,29 @@ +# `vcs` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +Caches source archives and release assets from GitHub (`base_url = "https://github.com"`) or any Gitea (any other `base_url`). Only the paths below are served; anything else is a 404. + +| Path | Behaviour | +|---|---| +| `//archive/refs/tags/.(tar.gz\|zip)` | immutable | +| `//archive/<40-char sha>.(tar.gz\|zip)` | immutable | +| `//releases/download//` | immutable | +| `//releases/latest/download/` | mutable | +| `//archive/refs/heads/.(tar.gz\|zip)` | `302` to the archive of the branch's current commit, resolved via the provider API and reused for `mutable_ttl` | + +Branch archives redirect, so `curl` needs `-L`; pip, npm and go follow redirects. The archive's top-level directory is `-`. `patterns` and `blocklist` match both the branch path and the commit path it redirects to, so allowlist repos by prefix (`^cli/cli/`). GitHub API calls use the [server GitHub credential](../../README.md#github-authentication); Gitea API calls use the remote's `username`/`password`. + +```hcl +resource "artifactapi_remote_vcs" "github" { + name = "github-vcs" + base_url = "https://github.com" + mutable_ttl = 600 + patterns = ["^cli/cli/", "^psf/requests/"] +} +``` + +```bash +curl -fsSLO https://artifactapi.example/api/v1/remote/github-vcs/cli/cli/archive/refs/tags/v2.60.0.tar.gz +pip install https://artifactapi.example/api/v1/remote/github-vcs/psf/requests/archive/refs/tags/v2.32.3.tar.gz +``` diff --git a/e2e-docker/caching_vcs_test.go b/e2e-docker/caching_vcs_test.go new file mode 100644 index 0000000..a3aed3f --- /dev/null +++ b/e2e-docker/caching_vcs_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"vcs", "octo/repo/archive/refs/tags/v1.0.0.tar.gz", "octo/repo/archive/refs/tags/v1.0.0.tar.gz"}) +} diff --git a/e2e-docker/fixtures/octo/repo/archive/refs/tags/v1.0.0.tar.gz b/e2e-docker/fixtures/octo/repo/archive/refs/tags/v1.0.0.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..252af2ff420d98f282b81acfe9937195b1d25c63 GIT binary patch literal 154 zcmb2|=3oE==C>Cexta|GS|6^lb=6xSt7!PzDR$RKwpb3m7LO~x&EGC_bzCUm_3^)% z@q`056$_*)FVm@za*s(WX;@AEz_nR_}CDP3AUj2RE=JR!PnIQH<$OGnb5^ukKWYAz>003iL BL(KpH literal 0 HcmV?d00001 diff --git a/internal/provider/all/vcs.go b/internal/provider/all/vcs.go new file mode 100644 index 0000000..b746a08 --- /dev/null +++ b/internal/provider/all/vcs.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/vcs" diff --git a/internal/provider/vcs/vcs.go b/internal/provider/vcs/vcs.go new file mode 100644 index 0000000..4807b65 --- /dev/null +++ b/internal/provider/vcs/vcs.go @@ -0,0 +1,263 @@ +// Package vcs caches source archives and release assets from GitHub or Gitea, +// keyed by ref: tags, commits and release assets are immutable, and branch +// archives redirect to the archive of the commit the branch currently points at. +package vcs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "sync" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/provider/generic" + "git.unkin.net/unkin/artifactapi/internal/proxy" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(New()) +} + +type kind int + +const ( + tagArchive kind = iota + 1 + branchArchive + commitArchive + releaseAsset + latestReleaseAsset +) + +// target is a parsed vcs request path. +type target struct { + Kind kind + Owner, Repo string + Name string // tag, branch, commit or asset + Ext string +} + +var ( + segmentRe = regexp.MustCompile(`^[A-Za-z0-9._+-]+$`) + commitRe = regexp.MustCompile(`^[0-9a-f]{40}$`) +) + +var errNotFound = errors.New("ref not found") + +// Provider embeds generic for upstream URLs, content types and GitHub/Basic auth. +type Provider struct { + generic.Provider + + client *http.Client + githubAPI string + + mu sync.Mutex + refs map[string]resolved +} + +type resolved struct { + commit string + at time.Time +} + +func New() *Provider { + return &Provider{ + client: &http.Client{Timeout: 30 * time.Second}, + githubAPI: "https://api.github.com", + refs: map[string]resolved{}, + } +} + +func (p *Provider) Type() models.PackageType { return models.PackageVCS } + +func (p *Provider) Classify(path string) provider.Mutability { + if ref, ok := parsePath(path); ok && (ref.Kind == branchArchive || ref.Kind == latestReleaseAsset) { + return provider.Mutable + } + return provider.Immutable +} + +// parsePath accepts only the archive and release-download shapes, rejecting +// any segment that could escape owner/repo or a ref. +func parsePath(path string) (target, bool) { + parts := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 3) + if len(parts) != 3 || !validSegment(parts[0]) || !validSegment(parts[1]) { + return target{}, false + } + ref := target{Owner: parts[0], Repo: parts[1]} + rest := parts[2] + switch { + case strings.HasPrefix(rest, "archive/"): + name, ext, ok := archiveName(strings.TrimPrefix(rest, "archive/")) + if !ok { + return target{}, false + } + ref.Ext = ext + switch { + case strings.HasPrefix(name, "refs/tags/"): + ref.Kind, ref.Name = tagArchive, strings.TrimPrefix(name, "refs/tags/") + case strings.HasPrefix(name, "refs/heads/"): + ref.Kind, ref.Name = branchArchive, strings.TrimPrefix(name, "refs/heads/") + case commitRe.MatchString(name): + return target{Kind: commitArchive, Owner: ref.Owner, Repo: ref.Repo, Name: name, Ext: ext}, true + default: + return target{}, false + } + return ref, validRefName(ref.Name) + case strings.HasPrefix(rest, "releases/latest/download/"): + ref.Kind, ref.Name = latestReleaseAsset, strings.TrimPrefix(rest, "releases/latest/download/") + return ref, validSegment(ref.Name) + case strings.HasPrefix(rest, "releases/download/"): + tagAsset := strings.TrimPrefix(rest, "releases/download/") + i := strings.LastIndex(tagAsset, "/") + if i < 0 { + return target{}, false + } + ref.Kind, ref.Name = releaseAsset, tagAsset[i+1:] + return ref, validRefName(tagAsset[:i]) && validSegment(ref.Name) + } + return target{}, false +} + +func archiveName(s string) (string, string, bool) { + for _, ext := range []string{".tar.gz", ".zip"} { + if name, ok := strings.CutSuffix(s, ext); ok && name != "" { + return name, ext, true + } + } + return "", "", false +} + +func validSegment(s string) bool { + return segmentRe.MatchString(s) && s != "." && s != ".." && !strings.Contains(s, "..") +} + +func validRefName(s string) bool { + for _, seg := range strings.Split(s, "/") { + if !validSegment(seg) || strings.HasPrefix(seg, ".") { + return false + } + } + return true +} + +// ServeRemote rejects unsupported paths and redirects branch archives to the +// immutable archive of the branch's current commit; everything else is proxied. +func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool { + ref, ok := parsePath(path) + if !ok { + http.Error(w, "unsupported vcs path", http.StatusNotFound) + return true + } + if ref.Kind != branchArchive { + return false + } + if proxy.NewClassifier(p).Classify(remote, path) == proxy.ClassDenied { + http.Error(w, "access denied", http.StatusForbidden) + return true + } + commit, err := p.resolve(r.Context(), remote, ref) + if errors.Is(err, errNotFound) { + http.Error(w, "branch not found", http.StatusNotFound) + return true + } + if err != nil { + http.Error(w, "bad gateway", http.StatusBadGateway) + return true + } + loc := strings.TrimRight(proxyBaseURL, "/") + provider.RemoteMountPath + remote.Name + "/" + + ref.Owner + "/" + ref.Repo + "/archive/" + commit + ref.Ext + http.Redirect(w, r, loc, http.StatusFound) + return true +} + +// resolve maps a branch to its commit, reusing the answer for mutable_ttl and, +// with stale_on_error, past an upstream failure. +func (p *Provider) resolve(ctx context.Context, remote models.Remote, ref target) (string, error) { + key := remote.Name + "\x00" + ref.Owner + "/" + ref.Repo + "\x00" + ref.Name + p.mu.Lock() + prev, ok := p.refs[key] + p.mu.Unlock() + if ok && time.Since(prev.at) < time.Duration(remote.MutableTTL)*time.Second { + return prev.commit, nil + } + commit, err := p.lookup(ctx, remote, ref) + if err != nil { + if ok && remote.StaleOnError && !errors.Is(err, errNotFound) { + return prev.commit, nil + } + return "", err + } + p.mu.Lock() + p.refs[key] = resolved{commit: commit, at: time.Now()} + p.mu.Unlock() + return commit, nil +} + +func (p *Provider) lookup(ctx context.Context, remote models.Remote, ref target) (string, error) { + segs := strings.Split(ref.Name, "/") + for i, s := range segs { + segs[i] = url.PathEscape(s) + } + branch := strings.Join(segs, "/") + github := isGitHub(remote.BaseURL) + api := strings.TrimRight(remote.BaseURL, "/") + "/api/v1/repos/" + ref.Owner + "/" + ref.Repo + "/branches/" + branch + if github { + api = p.githubAPI + "/repos/" + ref.Owner + "/" + ref.Repo + "/commits/" + branch + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, api, nil) + if err != nil { + return "", err + } + h, err := p.AuthHeaders(ctx, remote) + if err != nil { + return "", err + } + req.Header = h + if github { + req.Header.Set("Accept", "application/vnd.github.sha") + } + resp, err := p.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusUnprocessableEntity { + return "", errNotFound + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("branch lookup %s: %s", api, resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if err != nil { + return "", err + } + commit := strings.TrimSpace(string(body)) + if !github { + var b struct { + Commit struct { + ID string `json:"id"` + } `json:"commit"` + } + if err := json.Unmarshal(body, &b); err != nil { + return "", err + } + commit = b.Commit.ID + } + if !commitRe.MatchString(commit) { + return "", fmt.Errorf("branch lookup %s: unexpected commit %q", api, commit) + } + return commit, nil +} + +func isGitHub(baseURL string) bool { + u, err := url.Parse(baseURL) + return err == nil && strings.EqualFold(u.Hostname(), "github.com") +} diff --git a/internal/provider/vcs/vcs_test.go b/internal/provider/vcs/vcs_test.go new file mode 100644 index 0000000..c3d12d3 --- /dev/null +++ b/internal/provider/vcs/vcs_test.go @@ -0,0 +1,230 @@ +package vcs + +import ( + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +const ( + shaA = "4f2cfe92e3499f8581bf39db77e0a1431fa7b55c" + shaB = "7fa5434dbc083c853eef435d1915020345219d2f" +) + +func TestParsePath(t *testing.T) { + tests := []struct { + path string + ok bool + kind kind + name string + }{ + {"cli/cli/archive/refs/tags/v2.60.0.tar.gz", true, tagArchive, "v2.60.0"}, + {"/cli/cli/archive/refs/tags/v2.60.0.zip", true, tagArchive, "v2.60.0"}, + {"o/r/archive/refs/tags/release/1.0.tar.gz", true, tagArchive, "release/1.0"}, + {"o/r/archive/refs/heads/feature/x.tar.gz", true, branchArchive, "feature/x"}, + {"o/r/archive/" + shaA + ".tar.gz", true, commitArchive, shaA}, + {"o/r/releases/download/v1.0/tool_linux_amd64.tar.gz", true, releaseAsset, "tool_linux_amd64.tar.gz"}, + {"o/r/releases/latest/download/tool.zip", true, latestReleaseAsset, "tool.zip"}, + {"o/r/archive/main.tar.gz", false, 0, ""}, + {"o/r/archive/" + shaA[:7] + ".tar.gz", false, 0, ""}, + {"o/r/archive/refs/tags/v1.tar.bz2", false, 0, ""}, + {"o/r/archive/refs/tags/.tar.gz", false, 0, ""}, + {"../r/archive/refs/tags/v1.tar.gz", false, 0, ""}, + {"o/../archive/refs/tags/v1.tar.gz", false, 0, ""}, + {"o/r/archive/refs/tags/../../../etc/passwd.tar.gz", false, 0, ""}, + {"o/r/archive/refs/heads/a//b.tar.gz", false, 0, ""}, + {"o/r/archive/refs/heads/.hidden.tar.gz", false, 0, ""}, + {"o/r/archive/refs/heads/a..b.tar.gz", false, 0, ""}, + {"o/r/archive/refs/tags/v1%2f.tar.gz", false, 0, ""}, + {"o/r/releases/download/v1/../x", false, 0, ""}, + {"o/r/releases/download/asset", false, 0, ""}, + {"o/r/raw/main/README.md", false, 0, ""}, + {"o/r", false, 0, ""}, + } + for _, tt := range tests { + ref, ok := parsePath(tt.path) + if ok != tt.ok || (ok && (ref.Kind != tt.kind || ref.Name != tt.name)) { + t.Errorf("parsePath(%q) = %+v, %v; want kind %v name %q ok %v", tt.path, ref, ok, tt.kind, tt.name, tt.ok) + } + } +} + +func TestClassify(t *testing.T) { + p := New() + for path, want := range map[string]provider.Mutability{ + "o/r/archive/refs/tags/v1.tar.gz": provider.Immutable, + "o/r/archive/" + shaA + ".zip": provider.Immutable, + "o/r/releases/download/v1/a.bin": provider.Immutable, + "o/r/archive/refs/heads/main.tar.gz": provider.Mutable, + "o/r/releases/latest/download/webui.zip": provider.Mutable, + } { + if got := p.Classify(path); got != want { + t.Errorf("Classify(%q) = %v, want %v", path, got, want) + } + } +} + +type upstream struct { + srv *httptest.Server + hits atomic.Int32 + commit atomic.Value + status atomic.Int32 + accept atomic.Value + path atomic.Value +} + +func newUpstream(t *testing.T, gitea bool) *upstream { + u := &upstream{} + u.commit.Store(shaA) + u.status.Store(http.StatusOK) + u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.hits.Add(1) + u.accept.Store(r.Header.Get("Accept")) + u.path.Store(r.URL.EscapedPath()) + if s := int(u.status.Load()); s != http.StatusOK { + w.WriteHeader(s) + return + } + if gitea { + w.Write([]byte(`{"name":"main","commit":{"id":"` + u.commit.Load().(string) + `"}}`)) + return + } + w.Write([]byte(u.commit.Load().(string))) + })) + t.Cleanup(u.srv.Close) + return u +} + +func serve(p *Provider, remote models.Remote, path string) (*httptest.ResponseRecorder, bool) { + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/api/v1/remote/"+remote.Name+"/"+path, nil) + return w, p.ServeRemote(w, r, remote, path, "http://proxy", nil) +} + +func TestServeRemoteGiteaBranchRedirectsToCommit(t *testing.T) { + u := newUpstream(t, true) + p := New() + remote := models.Remote{Name: "gitea", BaseURL: u.srv.URL, MutableTTL: 3600} + + w, handled := serve(p, remote, "unkin/app/archive/refs/heads/feature/x.zip") + if !handled || w.Code != http.StatusFound { + t.Fatalf("handled=%v code=%d", handled, w.Code) + } + if got, want := w.Header().Get("Location"), "http://proxy/api/v1/remote/gitea/unkin/app/archive/"+shaA+".zip"; got != want { + t.Errorf("Location = %q, want %q", got, want) + } + if got := u.path.Load(); got != "/api/v1/repos/unkin/app/branches/feature/x" { + t.Errorf("api path = %q", got) + } +} + +func TestServeRemoteGitHubBranchUsesAPI(t *testing.T) { + u := newUpstream(t, false) + p := New() + p.githubAPI = u.srv.URL + remote := models.Remote{Name: "gh", BaseURL: "https://github.com", MutableTTL: 3600} + + w, _ := serve(p, remote, "cli/cli/archive/refs/heads/trunk.tar.gz") + if !strings.HasSuffix(w.Header().Get("Location"), "/cli/cli/archive/"+shaA+".tar.gz") { + t.Fatalf("Location = %q", w.Header().Get("Location")) + } + if u.path.Load() != "/repos/cli/cli/commits/trunk" || u.accept.Load() != "application/vnd.github.sha" { + t.Errorf("api path %q accept %q", u.path.Load(), u.accept.Load()) + } +} + +func TestServeRemoteBranchResolutionHonoursMutableTTL(t *testing.T) { + u := newUpstream(t, true) + p := New() + cached := models.Remote{Name: "cached", BaseURL: u.srv.URL, MutableTTL: 3600} + path := "o/r/archive/refs/heads/main.tar.gz" + + serve(p, cached, path) + u.commit.Store(shaB) + w, _ := serve(p, cached, path) + if u.hits.Load() != 1 || !strings.Contains(w.Header().Get("Location"), shaA) { + t.Fatalf("within ttl: hits=%d location=%q", u.hits.Load(), w.Header().Get("Location")) + } + + expired := cached + expired.Name = "expired" + expired.MutableTTL = 0 + serve(p, expired, path) + w, _ = serve(p, expired, path) + if u.hits.Load() != 3 || !strings.Contains(w.Header().Get("Location"), shaB) { + t.Fatalf("past ttl: hits=%d location=%q", u.hits.Load(), w.Header().Get("Location")) + } +} + +func TestServeRemoteBranchUpstreamFailure(t *testing.T) { + path := "o/r/archive/refs/heads/main.tar.gz" + for _, tc := range []struct { + name string + stale bool + want int + }{ + {"stale", true, http.StatusFound}, + {"strict", false, http.StatusBadGateway}, + } { + t.Run(tc.name, func(t *testing.T) { + u := newUpstream(t, true) + p := New() + remote := models.Remote{Name: tc.name, BaseURL: u.srv.URL, StaleOnError: tc.stale} + serve(p, remote, path) + u.status.Store(http.StatusInternalServerError) + if w, _ := serve(p, remote, path); w.Code != tc.want { + t.Fatalf("code = %d, want %d", w.Code, tc.want) + } + }) + } +} + +func TestServeRemoteBranchNotFound(t *testing.T) { + u := newUpstream(t, false) + u.status.Store(http.StatusUnprocessableEntity) + p := New() + p.githubAPI = u.srv.URL + w, _ := serve(p, models.Remote{Name: "gh", BaseURL: "https://github.com"}, "o/r/archive/refs/heads/nope.tar.gz") + if w.Code != http.StatusNotFound { + t.Fatalf("code = %d, want 404", w.Code) + } +} + +func TestServeRemoteRejectsBadCommit(t *testing.T) { + u := newUpstream(t, true) + u.commit.Store("../../etc") + w, _ := serve(New(), models.Remote{Name: "g", BaseURL: u.srv.URL}, "o/r/archive/refs/heads/main.tar.gz") + if w.Code != http.StatusBadGateway { + t.Fatalf("code = %d, want 502", w.Code) + } +} + +func TestServeRemoteAllowlist(t *testing.T) { + u := newUpstream(t, true) + remote := models.Remote{Name: "g", BaseURL: u.srv.URL, Patterns: []string{"^allowed/repo/"}} + if w, _ := serve(New(), remote, "other/repo/archive/refs/heads/main.tar.gz"); w.Code != http.StatusForbidden { + t.Fatalf("code = %d, want 403", w.Code) + } + if u.hits.Load() != 0 { + t.Fatalf("denied branch must not hit the API") + } +} + +func TestServeRemotePassesThroughAndRejects(t *testing.T) { + p := New() + remote := models.Remote{Name: "g", BaseURL: "https://github.com"} + if _, handled := serve(p, remote, "o/r/archive/refs/tags/v1.tar.gz"); handled { + t.Error("tag archives must go through the proxy engine") + } + if _, handled := serve(p, remote, "o/r/releases/download/v1/a.bin"); handled { + t.Error("release assets must go through the proxy engine") + } + if w, handled := serve(p, remote, "o/r/raw/main/x"); !handled || w.Code != http.StatusNotFound { + t.Errorf("unsupported path: handled=%v code=%d", handled, w.Code) + } +} diff --git a/pkg/models/package_type_vcs.go b/pkg/models/package_type_vcs.go new file mode 100644 index 0000000..eb20281 --- /dev/null +++ b/pkg/models/package_type_vcs.go @@ -0,0 +1,5 @@ +package models + +const PackageVCS PackageType = "vcs" + +func init() { registerPackageType(PackageVCS) } diff --git a/ui/src/packagetypes/defs/vcs.ts b/ui/src/packagetypes/defs/vcs.ts new file mode 100644 index 0000000..658dbc8 --- /dev/null +++ b/ui/src/packagetypes/defs/vcs.ts @@ -0,0 +1,26 @@ +import type { PackageTypeUI } from '../types'; + +const def: PackageTypeUI = { + color: 'default', + snippets: ({ proxy }) => { + return [ + { + title: 'Download a tag archive', + language: 'bash', + code: `curl -fsSLO ${proxy}///archive/refs/tags/.tar.gz`, + }, + { + title: 'Download a branch archive (redirects to its current commit)', + language: 'bash', + code: `curl -fsSL -o src.tar.gz ${proxy}///archive/refs/heads/.tar.gz`, + }, + { + title: 'Download a release asset', + language: 'bash', + code: `curl -fsSLO ${proxy}///releases/download//`, + }, + ]; + }, +}; + +export default def; -- 2.47.3 From 31b5faed6f121fd7ebff6591b76ee150b7e8a3c9 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 24 Oct 2026 16:36:09 +1100 Subject: [PATCH 2/4] Note commit archive naming in vcs docs --- docs/vcs/remote.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/vcs/remote.md b/docs/vcs/remote.md index 501510f..31ea1f9 100644 --- a/docs/vcs/remote.md +++ b/docs/vcs/remote.md @@ -12,7 +12,7 @@ Caches source archives and release assets from GitHub (`base_url = "https://gith | `//releases/latest/download/` | mutable | | `//archive/refs/heads/.(tar.gz\|zip)` | `302` to the archive of the branch's current commit, resolved via the provider API and reused for `mutable_ttl` | -Branch archives redirect, so `curl` needs `-L`; pip, npm and go follow redirects. The archive's top-level directory is `-`. `patterns` and `blocklist` match both the branch path and the commit path it redirects to, so allowlist repos by prefix (`^cli/cli/`). GitHub API calls use the [server GitHub credential](../../README.md#github-authentication); Gitea API calls use the remote's `username`/`password`. +Branch archives redirect, so `curl` needs `-L`; pip, npm and go follow redirects. The archive is the host's commit archive (GitHub names its top-level directory `-`). `patterns` and `blocklist` match both the branch path and the commit path it redirects to, so allowlist repos by prefix (`^cli/cli/`). GitHub API calls use the [server GitHub credential](../../README.md#github-authentication); Gitea API calls use the remote's `username`/`password`. ```hcl resource "artifactapi_remote_vcs" "github" { -- 2.47.3 From d15364cbd67ec8ecf6304a216d29e9fb89e67aef Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 24 Oct 2026 16:36:21 +1100 Subject: [PATCH 3/4] Name only redirect-following clients verified in vcs docs --- docs/vcs/remote.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/vcs/remote.md b/docs/vcs/remote.md index 31ea1f9..14cca60 100644 --- a/docs/vcs/remote.md +++ b/docs/vcs/remote.md @@ -12,7 +12,7 @@ Caches source archives and release assets from GitHub (`base_url = "https://gith | `//releases/latest/download/` | mutable | | `//archive/refs/heads/.(tar.gz\|zip)` | `302` to the archive of the branch's current commit, resolved via the provider API and reused for `mutable_ttl` | -Branch archives redirect, so `curl` needs `-L`; pip, npm and go follow redirects. The archive is the host's commit archive (GitHub names its top-level directory `-`). `patterns` and `blocklist` match both the branch path and the commit path it redirects to, so allowlist repos by prefix (`^cli/cli/`). GitHub API calls use the [server GitHub credential](../../README.md#github-authentication); Gitea API calls use the remote's `username`/`password`. +Branch archives redirect, so `curl` needs `-L`; pip and npm follow redirects. The archive is the host's commit archive (GitHub names its top-level directory `-`). `patterns` and `blocklist` match both the branch path and the commit path it redirects to, so allowlist repos by prefix (`^cli/cli/`). GitHub API calls use the [server GitHub credential](../../README.md#github-authentication); Gitea API calls use the remote's `username`/`password`. ```hcl resource "artifactapi_remote_vcs" "github" { -- 2.47.3 From cff459abda32efa1210ce3f0c4097080d81ae438 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 24 Oct 2026 16:49:02 +1100 Subject: [PATCH 4/4] Cache missing branches, bound and dedupe vcs branch lookups, map rate limits to 503 --- docs/vcs/remote.md | 6 +- internal/provider/generic/generic.go | 6 +- internal/provider/vcs/vcs.go | 132 ++++++++++++------ internal/provider/vcs/vcs_test.go | 196 +++++++++++++++++++++++---- 4 files changed, 267 insertions(+), 73 deletions(-) diff --git a/docs/vcs/remote.md b/docs/vcs/remote.md index 14cca60..00273e0 100644 --- a/docs/vcs/remote.md +++ b/docs/vcs/remote.md @@ -2,7 +2,7 @@ Shared classification, TTL and eviction rules: [caching](../common/caching.md). -Caches source archives and release assets from GitHub (`base_url = "https://github.com"`) or any Gitea (any other `base_url`). Only the paths below are served; anything else is a 404. +Caches source archives and release assets from GitHub (`base_url = "https://github.com"`) or Gitea (any other `base_url`). GitHub Enterprise is not supported: a non-github.com host is treated as Gitea. Only the paths below are served; anything else is a 404. | Path | Behaviour | |---|---| @@ -10,9 +10,9 @@ Caches source archives and release assets from GitHub (`base_url = "https://gith | `//archive/<40-char sha>.(tar.gz\|zip)` | immutable | | `//releases/download//` | immutable | | `//releases/latest/download/` | mutable | -| `//archive/refs/heads/.(tar.gz\|zip)` | `302` to the archive of the branch's current commit, resolved via the provider API and reused for `mutable_ttl` | +| `//archive/refs/heads/.(tar.gz\|zip)` | `302` to the archive of the branch's current commit, resolved via the provider's branch API and reused for `mutable_ttl` | -Branch archives redirect, so `curl` needs `-L`; pip and npm follow redirects. The archive is the host's commit archive (GitHub names its top-level directory `-`). `patterns` and `blocklist` match both the branch path and the commit path it redirects to, so allowlist repos by prefix (`^cli/cli/`). GitHub API calls use the [server GitHub credential](../../README.md#github-authentication); Gitea API calls use the remote's `username`/`password`. +Branch archives redirect, so `curl` needs `-L`; pip and npm follow redirects. The archive is the host's commit archive (GitHub names its top-level directory `-`). `patterns` and `blocklist` match both the branch path and the commit path it redirects to, so allowlist repos by prefix (`^cli/cli/`); a branch whose commit path is denied is a `403`. A missing branch is a `404`, cached for `mutable_ttl` (at least 60s). An API rate limit (`429`, or `403` with rate-limit headers) is a `503` with `Retry-After`, unless `stale_on_error` serves the last known commit. Each remote caches up to 4096 branch lookups. GitHub API calls use the [server GitHub credential](../../README.md#github-authentication); Gitea API calls use the remote's `username`/`password`. ```hcl resource "artifactapi_remote_vcs" "github" { diff --git a/internal/provider/generic/generic.go b/internal/provider/generic/generic.go index c909bf2..24281b5 100644 --- a/internal/provider/generic/generic.go +++ b/internal/provider/generic/generic.go @@ -71,7 +71,7 @@ func (p *Provider) AuthHeaders(ctx context.Context, remote models.Remote) (http. h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password))) return h, nil } - if isGitHubHost(remote.BaseURL) { + if IsGitHubHost(remote.BaseURL) { if c := githubauth.Server(); c != nil { tok, err := c.Token(ctx) if err != nil { @@ -85,11 +85,11 @@ func (p *Provider) AuthHeaders(ctx context.Context, remote models.Remote) (http. return h, nil } -// isGitHubHost reports whether rawURL targets a GitHub API/download host that +// IsGitHubHost reports whether rawURL targets a GitHub API/download host that // accepts the server credential. objects.githubusercontent.com is deliberately // excluded: release-asset downloads 302-redirect there with a pre-signed URL // that must not carry an Authorization header. -func isGitHubHost(rawURL string) bool { +func IsGitHubHost(rawURL string) bool { u, err := url.Parse(rawURL) if err != nil { return false diff --git a/internal/provider/vcs/vcs.go b/internal/provider/vcs/vcs.go index 4807b65..9abb9de 100644 --- a/internal/provider/vcs/vcs.go +++ b/internal/provider/vcs/vcs.go @@ -12,10 +12,13 @@ import ( "net/http" "net/url" "regexp" + "strconv" "strings" "sync" "time" + "golang.org/x/sync/singleflight" + "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/internal/provider/generic" "git.unkin.net/unkin/artifactapi/internal/proxy" @@ -51,6 +54,11 @@ var ( var errNotFound = errors.New("ref not found") +const ( + maxRefs = 4096 + negativeTTL = time.Minute +) + // Provider embeds generic for upstream URLs, content types and GitHub/Basic auth. type Provider struct { generic.Provider @@ -58,10 +66,12 @@ type Provider struct { client *http.Client githubAPI string - mu sync.Mutex - refs map[string]resolved + group singleflight.Group + mu sync.Mutex + refs map[string]map[string]resolved } +// resolved is a branch lookup; an empty commit records a missing branch. type resolved struct { commit string at time.Time @@ -71,7 +81,7 @@ func New() *Provider { return &Provider{ client: &http.Client{Timeout: 30 * time.Second}, githubAPI: "https://api.github.com", - refs: map[string]resolved{}, + refs: map[string]map[string]resolved{}, } } @@ -159,46 +169,88 @@ func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote mo if ref.Kind != branchArchive { return false } - if proxy.NewClassifier(p).Classify(remote, path) == proxy.ClassDenied { + classifier := proxy.NewClassifier(p) + if classifier.Classify(remote, path) == proxy.ClassDenied { http.Error(w, "access denied", http.StatusForbidden) return true } commit, err := p.resolve(r.Context(), remote, ref) - if errors.Is(err, errNotFound) { + var se *provider.UpstreamStatusError + switch { + case errors.Is(err, errNotFound): http.Error(w, "branch not found", http.StatusNotFound) return true - } - if err != nil { + case errors.As(err, &se) && (se.Status == http.StatusTooManyRequests || se.Status == http.StatusForbidden && !se.RetryAt.IsZero()): + secs := 60 + if !se.RetryAt.IsZero() { + secs = max(1, int(time.Until(se.RetryAt).Seconds()+1)) + } + w.Header().Set("Retry-After", strconv.Itoa(secs)) + http.Error(w, "upstream rate limited", http.StatusServiceUnavailable) + return true + case err != nil: http.Error(w, "bad gateway", http.StatusBadGateway) return true } - loc := strings.TrimRight(proxyBaseURL, "/") + provider.RemoteMountPath + remote.Name + "/" + - ref.Owner + "/" + ref.Repo + "/archive/" + commit + ref.Ext - http.Redirect(w, r, loc, http.StatusFound) + target := ref.Owner + "/" + ref.Repo + "/archive/" + commit + ref.Ext + if classifier.Classify(remote, target) == proxy.ClassDenied { + http.Error(w, "access denied", http.StatusForbidden) + return true + } + http.Redirect(w, r, strings.TrimRight(proxyBaseURL, "/")+provider.RemoteMountPath+remote.Name+"/"+target, http.StatusFound) return true } -// resolve maps a branch to its commit, reusing the answer for mutable_ttl and, -// with stale_on_error, past an upstream failure. +// resolve maps a branch to its commit, reusing the answer for mutable_ttl (a +// missing branch for at least negativeTTL) and, with stale_on_error, past an +// upstream failure. Concurrent lookups of one branch share a single API call. func (p *Provider) resolve(ctx context.Context, remote models.Remote, ref target) (string, error) { - key := remote.Name + "\x00" + ref.Owner + "/" + ref.Repo + "\x00" + ref.Name + key := ref.Owner + "/" + ref.Repo + "\x00" + ref.Name + ttl := time.Duration(remote.MutableTTL) * time.Second p.mu.Lock() - prev, ok := p.refs[key] + prev, ok := p.refs[remote.Name][key] p.mu.Unlock() - if ok && time.Since(prev.at) < time.Duration(remote.MutableTTL)*time.Second { + if ok && prev.commit != "" && time.Since(prev.at) < ttl { return prev.commit, nil } - commit, err := p.lookup(ctx, remote, ref) + if ok && prev.commit == "" && time.Since(prev.at) < max(ttl, negativeTTL) { + return "", errNotFound + } + v, err, _ := p.group.Do(remote.Name+"\x00"+key, func() (any, error) { + commit, err := p.lookup(context.WithoutCancel(ctx), remote, ref) + if err == nil || errors.Is(err, errNotFound) { + p.store(remote.Name, key, commit) + } + return commit, err + }) if err != nil { - if ok && remote.StaleOnError && !errors.Is(err, errNotFound) { + if ok && prev.commit != "" && remote.StaleOnError && !errors.Is(err, errNotFound) { return prev.commit, nil } return "", err } + return v.(string), nil +} + +// store records a lookup, evicting the remote's oldest entry once it holds maxRefs. +func (p *Provider) store(remote, key, commit string) { p.mu.Lock() - p.refs[key] = resolved{commit: commit, at: time.Now()} - p.mu.Unlock() - return commit, nil + defer p.mu.Unlock() + m := p.refs[remote] + if m == nil { + m = map[string]resolved{} + p.refs[remote] = m + } + if _, ok := m[key]; !ok && len(m) >= maxRefs { + var oldest string + for k, e := range m { + if oldest == "" || e.at.Before(m[oldest].at) { + oldest = k + } + } + delete(m, oldest) + } + m[key] = resolved{commit: commit, at: time.Now()} } func (p *Provider) lookup(ctx context.Context, remote models.Remote, ref target) (string, error) { @@ -207,10 +259,10 @@ func (p *Provider) lookup(ctx context.Context, remote models.Remote, ref target) segs[i] = url.PathEscape(s) } branch := strings.Join(segs, "/") - github := isGitHub(remote.BaseURL) + github := generic.IsGitHubHost(remote.BaseURL) api := strings.TrimRight(remote.BaseURL, "/") + "/api/v1/repos/" + ref.Owner + "/" + ref.Repo + "/branches/" + branch if github { - api = p.githubAPI + "/repos/" + ref.Owner + "/" + ref.Repo + "/commits/" + branch + api = p.githubAPI + "/repos/" + ref.Owner + "/" + ref.Repo + "/git/ref/heads/" + branch } req, err := http.NewRequestWithContext(ctx, http.MethodGet, api, nil) if err != nil { @@ -222,42 +274,36 @@ func (p *Provider) lookup(ctx context.Context, remote models.Remote, ref target) } req.Header = h if github { - req.Header.Set("Accept", "application/vnd.github.sha") + req.Header.Set("Accept", "application/vnd.github+json") } resp, err := p.client.Do(req) if err != nil { return "", err } - defer resp.Body.Close() + defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusUnprocessableEntity { return "", errNotFound } if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("branch lookup %s: %s", api, resp.Status) + return "", provider.NewUpstreamStatusError(api, resp) } - body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) - if err != nil { + var b struct { + Commit struct { + ID string `json:"id"` + } `json:"commit"` + Object struct { + SHA string `json:"sha"` + } `json:"object"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&b); err != nil { return "", err } - commit := strings.TrimSpace(string(body)) - if !github { - var b struct { - Commit struct { - ID string `json:"id"` - } `json:"commit"` - } - if err := json.Unmarshal(body, &b); err != nil { - return "", err - } - commit = b.Commit.ID + commit := b.Commit.ID + if github { + commit = b.Object.SHA } if !commitRe.MatchString(commit) { return "", fmt.Errorf("branch lookup %s: unexpected commit %q", api, commit) } return commit, nil } - -func isGitHub(baseURL string) bool { - u, err := url.Parse(baseURL) - return err == nil && strings.EqualFold(u.Hostname(), "github.com") -} diff --git a/internal/provider/vcs/vcs_test.go b/internal/provider/vcs/vcs_test.go index c3d12d3..a6f061e 100644 --- a/internal/provider/vcs/vcs_test.go +++ b/internal/provider/vcs/vcs_test.go @@ -1,12 +1,18 @@ package vcs import ( + "context" + "fmt" "net/http" "net/http/httptest" + "strconv" "strings" + "sync" "sync/atomic" "testing" + "time" + "git.unkin.net/unkin/artifactapi/internal/githubauth" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -74,27 +80,38 @@ type upstream struct { hits atomic.Int32 commit atomic.Value status atomic.Int32 + header atomic.Value accept atomic.Value + auth atomic.Value path atomic.Value + gate chan struct{} } func newUpstream(t *testing.T, gitea bool) *upstream { u := &upstream{} u.commit.Store(shaA) u.status.Store(http.StatusOK) + u.header.Store(http.Header{}) u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { u.hits.Add(1) + if u.gate != nil { + <-u.gate + } u.accept.Store(r.Header.Get("Accept")) + u.auth.Store(r.Header.Get("Authorization")) u.path.Store(r.URL.EscapedPath()) + for k, v := range u.header.Load().(http.Header) { + w.Header()[k] = v + } if s := int(u.status.Load()); s != http.StatusOK { w.WriteHeader(s) return } if gitea { - w.Write([]byte(`{"name":"main","commit":{"id":"` + u.commit.Load().(string) + `"}}`)) + _, _ = w.Write([]byte(`{"name":"main","commit":{"id":"` + u.commit.Load().(string) + `"}}`)) return } - w.Write([]byte(u.commit.Load().(string))) + _, _ = w.Write([]byte(`{"ref":"refs/heads/main","object":{"sha":"` + u.commit.Load().(string) + `","type":"commit"}}`)) })) t.Cleanup(u.srv.Close) return u @@ -106,10 +123,24 @@ func serve(p *Provider, remote models.Remote, path string) (*httptest.ResponseRe return w, p.ServeRemote(w, r, remote, path, "http://proxy", nil) } +// age backdates every cached lookup of a remote by d. +func age(p *Provider, remote string, d time.Duration) { + p.mu.Lock() + defer p.mu.Unlock() + for k, e := range p.refs[remote] { + e.at = e.at.Add(-d) + p.refs[remote][k] = e + } +} + +type staticCred string + +func (s staticCred) Token(context.Context) (string, error) { return string(s), nil } + func TestServeRemoteGiteaBranchRedirectsToCommit(t *testing.T) { u := newUpstream(t, true) p := New() - remote := models.Remote{Name: "gitea", BaseURL: u.srv.URL, MutableTTL: 3600} + remote := models.Remote{Name: "gitea", BaseURL: u.srv.URL, MutableTTL: 3600, Username: "bot", Password: "pw"} w, handled := serve(p, remote, "unkin/app/archive/refs/heads/feature/x.zip") if !handled || w.Code != http.StatusFound { @@ -121,42 +152,50 @@ func TestServeRemoteGiteaBranchRedirectsToCommit(t *testing.T) { if got := u.path.Load(); got != "/api/v1/repos/unkin/app/branches/feature/x" { t.Errorf("api path = %q", got) } + if got := u.auth.Load(); got != "Basic Ym90OnB3" { + t.Errorf("Authorization = %q, want remote basic auth", got) + } } func TestServeRemoteGitHubBranchUsesAPI(t *testing.T) { + githubauth.SetServer(staticCred("ghs_server")) + t.Cleanup(func() { githubauth.SetServer(nil) }) u := newUpstream(t, false) p := New() p.githubAPI = u.srv.URL - remote := models.Remote{Name: "gh", BaseURL: "https://github.com", MutableTTL: 3600} - w, _ := serve(p, remote, "cli/cli/archive/refs/heads/trunk.tar.gz") - if !strings.HasSuffix(w.Header().Get("Location"), "/cli/cli/archive/"+shaA+".tar.gz") { - t.Fatalf("Location = %q", w.Header().Get("Location")) - } - if u.path.Load() != "/repos/cli/cli/commits/trunk" || u.accept.Load() != "application/vnd.github.sha" { - t.Errorf("api path %q accept %q", u.path.Load(), u.accept.Load()) + for _, base := range []string{"https://github.com", "https://www.github.com/", "https://api.github.com"} { + remote := models.Remote{Name: base, BaseURL: base, MutableTTL: 3600} + w, _ := serve(p, remote, "cli/cli/archive/refs/heads/feature/x.tar.gz") + if !strings.HasSuffix(w.Header().Get("Location"), "/cli/cli/archive/"+shaA+".tar.gz") { + t.Fatalf("%s: Location = %q", base, w.Header().Get("Location")) + } + if u.path.Load() != "/repos/cli/cli/git/ref/heads/feature/x" || u.accept.Load() != "application/vnd.github+json" { + t.Errorf("%s: api path %q accept %q", base, u.path.Load(), u.accept.Load()) + } + if got := u.auth.Load(); got != "Bearer ghs_server" { + t.Errorf("%s: Authorization = %q, want server credential", base, got) + } } } func TestServeRemoteBranchResolutionHonoursMutableTTL(t *testing.T) { u := newUpstream(t, true) p := New() - cached := models.Remote{Name: "cached", BaseURL: u.srv.URL, MutableTTL: 3600} + remote := models.Remote{Name: "g", BaseURL: u.srv.URL, MutableTTL: 600} path := "o/r/archive/refs/heads/main.tar.gz" - serve(p, cached, path) + serve(p, remote, path) u.commit.Store(shaB) - w, _ := serve(p, cached, path) + age(p, remote.Name, 599*time.Second) + w, _ := serve(p, remote, path) if u.hits.Load() != 1 || !strings.Contains(w.Header().Get("Location"), shaA) { t.Fatalf("within ttl: hits=%d location=%q", u.hits.Load(), w.Header().Get("Location")) } - expired := cached - expired.Name = "expired" - expired.MutableTTL = 0 - serve(p, expired, path) - w, _ = serve(p, expired, path) - if u.hits.Load() != 3 || !strings.Contains(w.Header().Get("Location"), shaB) { + age(p, remote.Name, 2*time.Second) + w, _ = serve(p, remote, path) + if u.hits.Load() != 2 || !strings.Contains(w.Header().Get("Location"), shaB) { t.Fatalf("past ttl: hits=%d location=%q", u.hits.Load(), w.Header().Get("Location")) } } @@ -184,14 +223,114 @@ func TestServeRemoteBranchUpstreamFailure(t *testing.T) { } } -func TestServeRemoteBranchNotFound(t *testing.T) { +func TestServeRemoteBranchRateLimited(t *testing.T) { + reset := strconv.FormatInt(time.Now().Add(120*time.Second).Unix(), 10) + for _, tc := range []struct { + name string + status int + header http.Header + want int + retry [2]int + }{ + {"primary 403", http.StatusForbidden, http.Header{"X-Ratelimit-Remaining": {"0"}, "X-Ratelimit-Reset": {reset}}, http.StatusServiceUnavailable, [2]int{100, 122}}, + {"secondary 403", http.StatusForbidden, http.Header{"Retry-After": {"30"}}, http.StatusServiceUnavailable, [2]int{30, 31}}, + {"429", http.StatusTooManyRequests, http.Header{"Retry-After": {"5"}}, http.StatusServiceUnavailable, [2]int{5, 6}}, + {"429 no hint", http.StatusTooManyRequests, http.Header{}, http.StatusServiceUnavailable, [2]int{60, 60}}, + {"plain 403", http.StatusForbidden, http.Header{}, http.StatusBadGateway, [2]int{}}, + } { + t.Run(tc.name, func(t *testing.T) { + u := newUpstream(t, false) + u.status.Store(int32(tc.status)) + u.header.Store(tc.header) + p := New() + p.githubAPI = u.srv.URL + w, _ := serve(p, models.Remote{Name: "gh", BaseURL: "https://github.com"}, "o/r/archive/refs/heads/main.tar.gz") + if w.Code != tc.want { + t.Fatalf("code = %d, want %d", w.Code, tc.want) + } + ra := w.Header().Get("Retry-After") + if tc.want != http.StatusServiceUnavailable { + if ra != "" { + t.Fatalf("Retry-After = %q on %d", ra, w.Code) + } + return + } + if n, err := strconv.Atoi(ra); err != nil || n < tc.retry[0] || n > tc.retry[1] { + t.Fatalf("Retry-After = %q, want %d..%d", ra, tc.retry[0], tc.retry[1]) + } + }) + } +} + +func TestServeRemoteBranchNotFoundIsCached(t *testing.T) { u := newUpstream(t, false) - u.status.Store(http.StatusUnprocessableEntity) + u.status.Store(http.StatusNotFound) p := New() p.githubAPI = u.srv.URL - w, _ := serve(p, models.Remote{Name: "gh", BaseURL: "https://github.com"}, "o/r/archive/refs/heads/nope.tar.gz") - if w.Code != http.StatusNotFound { - t.Fatalf("code = %d, want 404", w.Code) + remote := models.Remote{Name: "gh", BaseURL: "https://github.com"} + path := "o/r/archive/refs/heads/nope.tar.gz" + + for range 3 { + if w, _ := serve(p, remote, path); w.Code != http.StatusNotFound { + t.Fatalf("code = %d, want 404", w.Code) + } + } + if u.hits.Load() != 1 { + t.Fatalf("hits = %d, want 1 within negative ttl", u.hits.Load()) + } + u.status.Store(http.StatusOK) + age(p, remote.Name, negativeTTL) + if w, _ := serve(p, remote, path); w.Code != http.StatusFound || u.hits.Load() != 2 { + t.Fatalf("after negative ttl: code=%d hits=%d", w.Code, u.hits.Load()) + } +} + +func TestServeRemoteConcurrentLookupsShareOneCall(t *testing.T) { + u := newUpstream(t, true) + u.gate = make(chan struct{}) + p := New() + remote := models.Remote{Name: "g", BaseURL: u.srv.URL, MutableTTL: 600} + + var wg sync.WaitGroup + codes := make([]int, 20) + for i := range codes { + wg.Add(1) + go func() { + defer wg.Done() + w, _ := serve(p, remote, "o/r/archive/refs/heads/main.tar.gz") + codes[i] = w.Code + }() + } + for u.hits.Load() == 0 { + time.Sleep(time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + close(u.gate) + wg.Wait() + if u.hits.Load() != 1 { + t.Fatalf("hits = %d, want 1", u.hits.Load()) + } + for i, c := range codes { + if c != http.StatusFound { + t.Fatalf("request %d: code %d", i, c) + } + } +} + +func TestStoreBoundsEntriesPerRemote(t *testing.T) { + p := New() + for i := range maxRefs + 10 { + p.store("a", fmt.Sprint(i), "") + } + p.store("b", "x", shaA) + if n := len(p.refs["a"]); n != maxRefs { + t.Fatalf("remote a holds %d entries, want %d", n, maxRefs) + } + if len(p.refs["b"]) != 1 { + t.Fatalf("remote b must keep its own budget") + } + if _, ok := p.refs["a"][fmt.Sprint(maxRefs+9)]; !ok { + t.Fatalf("newest entry evicted") } } @@ -215,6 +354,15 @@ func TestServeRemoteAllowlist(t *testing.T) { } } +func TestServeRemoteBlockedRedirectTarget(t *testing.T) { + u := newUpstream(t, true) + remote := models.Remote{Name: "g", BaseURL: u.srv.URL, Blocklist: []string{"/archive/" + shaA}} + w, _ := serve(New(), remote, "o/r/archive/refs/heads/main.tar.gz") + if w.Code != http.StatusForbidden || w.Header().Get("Location") != "" { + t.Fatalf("code = %d location %q, want 403 without redirect", w.Code, w.Header().Get("Location")) + } +} + func TestServeRemotePassesThroughAndRejects(t *testing.T) { p := New() remote := models.Remote{Name: "g", BaseURL: "https://github.com"} -- 2.47.3