artifactapi/internal/provider/vcs/vcs_test.go

package vcs

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"git.unkin.net/unkin/artifactapi/internal/githubauth"
	"git.unkin.net/unkin/artifactapi/internal/provider"
	"git.unkin.net/unkin/artifactapi/pkg/models"
)

const (
	shaA = "4f2cfe92e3499f8581bf39db77e0a1431fa7b55c"
	shaB = "7fa5434dbc083c853eef435d1915020345219d2f"
)

func TestParsePath(t *testing.T) {
	tests := []struct {
		path string
		ok   bool
		kind kind
		name string
	}{
		{"cli/cli/archive/refs/tags/v2.60.0.tar.gz", true, tagArchive, "v2.60.0"},
		{"/cli/cli/archive/refs/tags/v2.60.0.zip", true, tagArchive, "v2.60.0"},
		{"o/r/archive/refs/tags/release/1.0.tar.gz", true, tagArchive, "release/1.0"},
		{"o/r/archive/refs/heads/feature/x.tar.gz", true, branchArchive, "feature/x"},
		{"o/r/archive/" + shaA + ".tar.gz", true, commitArchive, shaA},
		{"o/r/releases/download/v1.0/tool_linux_amd64.tar.gz", true, releaseAsset, "tool_linux_amd64.tar.gz"},
		{"o/r/releases/latest/download/tool.zip", true, latestReleaseAsset, "tool.zip"},
		{"o/r/archive/main.tar.gz", false, 0, ""},
		{"o/r/archive/" + shaA[:7] + ".tar.gz", false, 0, ""},
		{"o/r/archive/refs/tags/v1.tar.bz2", false, 0, ""},
		{"o/r/archive/refs/tags/.tar.gz", false, 0, ""},
		{"../r/archive/refs/tags/v1.tar.gz", false, 0, ""},
		{"o/../archive/refs/tags/v1.tar.gz", false, 0, ""},
		{"o/r/archive/refs/tags/../../../etc/passwd.tar.gz", false, 0, ""},
		{"o/r/archive/refs/heads/a//b.tar.gz", false, 0, ""},
		{"o/r/archive/refs/heads/.hidden.tar.gz", false, 0, ""},
		{"o/r/archive/refs/heads/a..b.tar.gz", false, 0, ""},
		{"o/r/archive/refs/tags/v1%2f.tar.gz", false, 0, ""},
		{"o/r/releases/download/v1/../x", false, 0, ""},
		{"o/r/releases/download/asset", false, 0, ""},
		{"o/r/raw/main/README.md", false, 0, ""},
		{"o/r", false, 0, ""},
	}
	for _, tt := range tests {
		ref, ok := parsePath(tt.path)
		if ok != tt.ok || (ok && (ref.Kind != tt.kind || ref.Name != tt.name)) {
			t.Errorf("parsePath(%q) = %+v, %v; want kind %v name %q ok %v", tt.path, ref, ok, tt.kind, tt.name, tt.ok)
		}
	}
}

func TestClassify(t *testing.T) {
	p := New()
	for path, want := range map[string]provider.Mutability{
		"o/r/archive/refs/tags/v1.tar.gz":        provider.Immutable,
		"o/r/archive/" + shaA + ".zip":           provider.Immutable,
		"o/r/releases/download/v1/a.bin":         provider.Immutable,
		"o/r/archive/refs/heads/main.tar.gz":     provider.Mutable,
		"o/r/releases/latest/download/webui.zip": provider.Mutable,
	} {
		if got := p.Classify(path); got != want {
			t.Errorf("Classify(%q) = %v, want %v", path, got, want)
		}
	}
}

type upstream struct {
	srv    *httptest.Server
	hits   atomic.Int32
	commit atomic.Value
	status atomic.Int32
	header atomic.Value
	accept atomic.Value
	auth   atomic.Value
	path   atomic.Value
	gate   chan struct{}
}

func newUpstream(t *testing.T, gitea bool) *upstream {
	u := &upstream{}
	u.commit.Store(shaA)
	u.status.Store(http.StatusOK)
	u.header.Store(http.Header{})
	u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		u.hits.Add(1)
		if u.gate != nil {
			<-u.gate
		}
		u.accept.Store(r.Header.Get("Accept"))
		u.auth.Store(r.Header.Get("Authorization"))
		u.path.Store(r.URL.EscapedPath())
		for k, v := range u.header.Load().(http.Header) {
			w.Header()[k] = v
		}
		if s := int(u.status.Load()); s != http.StatusOK {
			w.WriteHeader(s)
			return
		}
		if gitea {
			_, _ = w.Write([]byte(`{"name":"main","commit":{"id":"` + u.commit.Load().(string) + `"}}`))
			return
		}
		_, _ = w.Write([]byte(`{"ref":"refs/heads/main","object":{"sha":"` + u.commit.Load().(string) + `","type":"commit"}}`))
	}))
	t.Cleanup(u.srv.Close)
	return u
}

func serve(p *Provider, remote models.Remote, path string) (*httptest.ResponseRecorder, bool) {
	w := httptest.NewRecorder()
	r := httptest.NewRequest(http.MethodGet, "/api/v1/remote/"+remote.Name+"/"+path, nil)
	return w, p.ServeRemote(w, r, remote, path, "http://proxy", nil)
}

// age backdates every cached lookup of a remote by d.
func age(p *Provider, remote string, d time.Duration) {
	p.mu.Lock()
	defer p.mu.Unlock()
	for k, e := range p.refs[remote] {
		e.at = e.at.Add(-d)
		p.refs[remote][k] = e
	}
}

type staticCred string

func (s staticCred) Token(context.Context) (string, error) { return string(s), nil }

func TestServeRemoteGiteaBranchRedirectsToCommit(t *testing.T) {
	u := newUpstream(t, true)
	p := New()
	remote := models.Remote{Name: "gitea", BaseURL: u.srv.URL, MutableTTL: 3600, Username: "bot", Password: "pw"}

	w, handled := serve(p, remote, "unkin/app/archive/refs/heads/feature/x.zip")
	if !handled || w.Code != http.StatusFound {
		t.Fatalf("handled=%v code=%d", handled, w.Code)
	}
	if got, want := w.Header().Get("Location"), "http://proxy/api/v1/remote/gitea/unkin/app/archive/"+shaA+".zip"; got != want {
		t.Errorf("Location = %q, want %q", got, want)
	}
	if got := u.path.Load(); got != "/api/v1/repos/unkin/app/branches/feature/x" {
		t.Errorf("api path = %q", got)
	}
	if got := u.auth.Load(); got != "Basic Ym90OnB3" {
		t.Errorf("Authorization = %q, want remote basic auth", got)
	}
}

func TestServeRemoteGitHubBranchUsesAPI(t *testing.T) {
	githubauth.SetServer(staticCred("ghs_server"))
	t.Cleanup(func() { githubauth.SetServer(nil) })
	u := newUpstream(t, false)
	p := New()
	p.githubAPI = u.srv.URL

	for _, base := range []string{"https://github.com", "https://www.github.com/", "https://api.github.com"} {
		remote := models.Remote{Name: base, BaseURL: base, MutableTTL: 3600}
		w, _ := serve(p, remote, "cli/cli/archive/refs/heads/feature/x.tar.gz")
		if !strings.HasSuffix(w.Header().Get("Location"), "/cli/cli/archive/"+shaA+".tar.gz") {
			t.Fatalf("%s: Location = %q", base, w.Header().Get("Location"))
		}
		if u.path.Load() != "/repos/cli/cli/git/ref/heads/feature/x" || u.accept.Load() != "application/vnd.github+json" {
			t.Errorf("%s: api path %q accept %q", base, u.path.Load(), u.accept.Load())
		}
		if got := u.auth.Load(); got != "Bearer ghs_server" {
			t.Errorf("%s: Authorization = %q, want server credential", base, got)
		}
	}
}

func TestServeRemoteBranchResolutionHonoursMutableTTL(t *testing.T) {
	u := newUpstream(t, true)
	p := New()
	remote := models.Remote{Name: "g", BaseURL: u.srv.URL, MutableTTL: 600}
	path := "o/r/archive/refs/heads/main.tar.gz"

	serve(p, remote, path)
	u.commit.Store(shaB)
	age(p, remote.Name, 599*time.Second)
	w, _ := serve(p, remote, path)
	if u.hits.Load() != 1 || !strings.Contains(w.Header().Get("Location"), shaA) {
		t.Fatalf("within ttl: hits=%d location=%q", u.hits.Load(), w.Header().Get("Location"))
	}

	age(p, remote.Name, 2*time.Second)
	w, _ = serve(p, remote, path)
	if u.hits.Load() != 2 || !strings.Contains(w.Header().Get("Location"), shaB) {
		t.Fatalf("past ttl: hits=%d location=%q", u.hits.Load(), w.Header().Get("Location"))
	}
}

func TestServeRemoteBranchUpstreamFailure(t *testing.T) {
	path := "o/r/archive/refs/heads/main.tar.gz"
	for _, tc := range []struct {
		name  string
		stale bool
		want  int
	}{
		{"stale", true, http.StatusFound},
		{"strict", false, http.StatusBadGateway},
	} {
		t.Run(tc.name, func(t *testing.T) {
			u := newUpstream(t, true)
			p := New()
			remote := models.Remote{Name: tc.name, BaseURL: u.srv.URL, StaleOnError: tc.stale}
			serve(p, remote, path)
			u.status.Store(http.StatusInternalServerError)
			if w, _ := serve(p, remote, path); w.Code != tc.want {
				t.Fatalf("code = %d, want %d", w.Code, tc.want)
			}
		})
	}
}

func TestServeRemoteBranchRateLimited(t *testing.T) {
	reset := strconv.FormatInt(time.Now().Add(120*time.Second).Unix(), 10)
	for _, tc := range []struct {
		name   string
		status int
		header http.Header
		want   int
		retry  [2]int
	}{
		{"primary 403", http.StatusForbidden, http.Header{"X-Ratelimit-Remaining": {"0"}, "X-Ratelimit-Reset": {reset}}, http.StatusServiceUnavailable, [2]int{100, 122}},
		{"secondary 403", http.StatusForbidden, http.Header{"Retry-After": {"30"}}, http.StatusServiceUnavailable, [2]int{30, 31}},
		{"429", http.StatusTooManyRequests, http.Header{"Retry-After": {"5"}}, http.StatusServiceUnavailable, [2]int{5, 6}},
		{"429 no hint", http.StatusTooManyRequests, http.Header{}, http.StatusServiceUnavailable, [2]int{60, 60}},
		{"plain 403", http.StatusForbidden, http.Header{}, http.StatusBadGateway, [2]int{}},
	} {
		t.Run(tc.name, func(t *testing.T) {
			u := newUpstream(t, false)
			u.status.Store(int32(tc.status))
			u.header.Store(tc.header)
			p := New()
			p.githubAPI = u.srv.URL
			w, _ := serve(p, models.Remote{Name: "gh", BaseURL: "https://github.com"}, "o/r/archive/refs/heads/main.tar.gz")
			if w.Code != tc.want {
				t.Fatalf("code = %d, want %d", w.Code, tc.want)
			}
			ra := w.Header().Get("Retry-After")
			if tc.want != http.StatusServiceUnavailable {
				if ra != "" {
					t.Fatalf("Retry-After = %q on %d", ra, w.Code)
				}
				return
			}
			if n, err := strconv.Atoi(ra); err != nil || n < tc.retry[0] || n > tc.retry[1] {
				t.Fatalf("Retry-After = %q, want %d..%d", ra, tc.retry[0], tc.retry[1])
			}
		})
	}
}

func TestServeRemoteBranchNotFoundIsCached(t *testing.T) {
	u := newUpstream(t, false)
	u.status.Store(http.StatusNotFound)
	p := New()
	p.githubAPI = u.srv.URL
	remote := models.Remote{Name: "gh", BaseURL: "https://github.com"}
	path := "o/r/archive/refs/heads/nope.tar.gz"

	for range 3 {
		if w, _ := serve(p, remote, path); w.Code != http.StatusNotFound {
			t.Fatalf("code = %d, want 404", w.Code)
		}
	}
	if u.hits.Load() != 1 {
		t.Fatalf("hits = %d, want 1 within negative ttl", u.hits.Load())
	}
	u.status.Store(http.StatusOK)
	age(p, remote.Name, negativeTTL)
	if w, _ := serve(p, remote, path); w.Code != http.StatusFound || u.hits.Load() != 2 {
		t.Fatalf("after negative ttl: code=%d hits=%d", w.Code, u.hits.Load())
	}
}

func TestServeRemoteConcurrentLookupsShareOneCall(t *testing.T) {
	u := newUpstream(t, true)
	u.gate = make(chan struct{})
	p := New()
	remote := models.Remote{Name: "g", BaseURL: u.srv.URL, MutableTTL: 600}

	var wg sync.WaitGroup
	codes := make([]int, 20)
	for i := range codes {
		wg.Add(1)
		go func() {
			defer wg.Done()
			w, _ := serve(p, remote, "o/r/archive/refs/heads/main.tar.gz")
			codes[i] = w.Code
		}()
	}
	for u.hits.Load() == 0 {
		time.Sleep(time.Millisecond)
	}
	time.Sleep(50 * time.Millisecond)
	close(u.gate)
	wg.Wait()
	if u.hits.Load() != 1 {
		t.Fatalf("hits = %d, want 1", u.hits.Load())
	}
	for i, c := range codes {
		if c != http.StatusFound {
			t.Fatalf("request %d: code %d", i, c)
		}
	}
}

func TestStoreBoundsEntriesPerRemote(t *testing.T) {
	p := New()
	for i := range maxRefs + 10 {
		p.store("a", fmt.Sprint(i), "")
	}
	p.store("b", "x", shaA)
	if n := len(p.refs["a"]); n != maxRefs {
		t.Fatalf("remote a holds %d entries, want %d", n, maxRefs)
	}
	if len(p.refs["b"]) != 1 {
		t.Fatalf("remote b must keep its own budget")
	}
	if _, ok := p.refs["a"][fmt.Sprint(maxRefs+9)]; !ok {
		t.Fatalf("newest entry evicted")
	}
}

func TestServeRemoteRejectsBadCommit(t *testing.T) {
	u := newUpstream(t, true)
	u.commit.Store("../../etc")
	w, _ := serve(New(), models.Remote{Name: "g", BaseURL: u.srv.URL}, "o/r/archive/refs/heads/main.tar.gz")
	if w.Code != http.StatusBadGateway {
		t.Fatalf("code = %d, want 502", w.Code)
	}
}

func TestServeRemoteAllowlist(t *testing.T) {
	u := newUpstream(t, true)
	remote := models.Remote{Name: "g", BaseURL: u.srv.URL, Patterns: []string{"^allowed/repo/"}}
	if w, _ := serve(New(), remote, "other/repo/archive/refs/heads/main.tar.gz"); w.Code != http.StatusForbidden {
		t.Fatalf("code = %d, want 403", w.Code)
	}
	if u.hits.Load() != 0 {
		t.Fatalf("denied branch must not hit the API")
	}
}

func TestServeRemoteBlockedRedirectTarget(t *testing.T) {
	u := newUpstream(t, true)
	remote := models.Remote{Name: "g", BaseURL: u.srv.URL, Blocklist: []string{"/archive/" + shaA}}
	w, _ := serve(New(), remote, "o/r/archive/refs/heads/main.tar.gz")
	if w.Code != http.StatusForbidden || w.Header().Get("Location") != "" {
		t.Fatalf("code = %d location %q, want 403 without redirect", w.Code, w.Header().Get("Location"))
	}
}

func TestServeRemotePassesThroughAndRejects(t *testing.T) {
	p := New()
	remote := models.Remote{Name: "g", BaseURL: "https://github.com"}
	if _, handled := serve(p, remote, "o/r/archive/refs/tags/v1.tar.gz"); handled {
		t.Error("tag archives must go through the proxy engine")
	}
	if _, handled := serve(p, remote, "o/r/releases/download/v1/a.bin"); handled {
		t.Error("release assets must go through the proxy engine")
	}
	if w, handled := serve(p, remote, "o/r/raw/main/x"); !handled || w.Code != http.StatusNotFound {
		t.Errorf("unsupported path: handled=%v code=%d", handled, w.Code)
	}
}