310 lines
8.9 KiB
Go
310 lines
8.9 KiB
Go
// Package vcs caches source archives and release assets from GitHub or Gitea,
|
|
// keyed by ref: tags, commits and release assets are immutable, and branch
|
|
// archives redirect to the archive of the commit the branch currently points at.
|
|
package vcs
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"golang.org/x/sync/singleflight"
|
|
|
|
"git.unkin.net/unkin/artifactapi/internal/provider"
|
|
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
|
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
func init() {
|
|
provider.Register(New())
|
|
}
|
|
|
|
type kind int
|
|
|
|
const (
|
|
tagArchive kind = iota + 1
|
|
branchArchive
|
|
commitArchive
|
|
releaseAsset
|
|
latestReleaseAsset
|
|
)
|
|
|
|
// target is a parsed vcs request path.
|
|
type target struct {
|
|
Kind kind
|
|
Owner, Repo string
|
|
Name string // tag, branch, commit or asset
|
|
Ext string
|
|
}
|
|
|
|
var (
|
|
segmentRe = regexp.MustCompile(`^[A-Za-z0-9._+-]+$`)
|
|
commitRe = regexp.MustCompile(`^[0-9a-f]{40}$`)
|
|
)
|
|
|
|
var errNotFound = errors.New("ref not found")
|
|
|
|
const (
|
|
maxRefs = 4096
|
|
negativeTTL = time.Minute
|
|
)
|
|
|
|
// Provider embeds generic for upstream URLs, content types and GitHub/Basic auth.
|
|
type Provider struct {
|
|
generic.Provider
|
|
|
|
client *http.Client
|
|
githubAPI string
|
|
|
|
group singleflight.Group
|
|
mu sync.Mutex
|
|
refs map[string]map[string]resolved
|
|
}
|
|
|
|
// resolved is a branch lookup; an empty commit records a missing branch.
|
|
type resolved struct {
|
|
commit string
|
|
at time.Time
|
|
}
|
|
|
|
func New() *Provider {
|
|
return &Provider{
|
|
client: &http.Client{Timeout: 30 * time.Second},
|
|
githubAPI: "https://api.github.com",
|
|
refs: map[string]map[string]resolved{},
|
|
}
|
|
}
|
|
|
|
func (p *Provider) Type() models.PackageType { return models.PackageVCS }
|
|
|
|
func (p *Provider) Classify(path string) provider.Mutability {
|
|
if ref, ok := parsePath(path); ok && (ref.Kind == branchArchive || ref.Kind == latestReleaseAsset) {
|
|
return provider.Mutable
|
|
}
|
|
return provider.Immutable
|
|
}
|
|
|
|
// parsePath accepts only the archive and release-download shapes, rejecting
|
|
// any segment that could escape owner/repo or a ref.
|
|
func parsePath(path string) (target, bool) {
|
|
parts := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 3)
|
|
if len(parts) != 3 || !validSegment(parts[0]) || !validSegment(parts[1]) {
|
|
return target{}, false
|
|
}
|
|
ref := target{Owner: parts[0], Repo: parts[1]}
|
|
rest := parts[2]
|
|
switch {
|
|
case strings.HasPrefix(rest, "archive/"):
|
|
name, ext, ok := archiveName(strings.TrimPrefix(rest, "archive/"))
|
|
if !ok {
|
|
return target{}, false
|
|
}
|
|
ref.Ext = ext
|
|
switch {
|
|
case strings.HasPrefix(name, "refs/tags/"):
|
|
ref.Kind, ref.Name = tagArchive, strings.TrimPrefix(name, "refs/tags/")
|
|
case strings.HasPrefix(name, "refs/heads/"):
|
|
ref.Kind, ref.Name = branchArchive, strings.TrimPrefix(name, "refs/heads/")
|
|
case commitRe.MatchString(name):
|
|
return target{Kind: commitArchive, Owner: ref.Owner, Repo: ref.Repo, Name: name, Ext: ext}, true
|
|
default:
|
|
return target{}, false
|
|
}
|
|
return ref, validRefName(ref.Name)
|
|
case strings.HasPrefix(rest, "releases/latest/download/"):
|
|
ref.Kind, ref.Name = latestReleaseAsset, strings.TrimPrefix(rest, "releases/latest/download/")
|
|
return ref, validSegment(ref.Name)
|
|
case strings.HasPrefix(rest, "releases/download/"):
|
|
tagAsset := strings.TrimPrefix(rest, "releases/download/")
|
|
i := strings.LastIndex(tagAsset, "/")
|
|
if i < 0 {
|
|
return target{}, false
|
|
}
|
|
ref.Kind, ref.Name = releaseAsset, tagAsset[i+1:]
|
|
return ref, validRefName(tagAsset[:i]) && validSegment(ref.Name)
|
|
}
|
|
return target{}, false
|
|
}
|
|
|
|
func archiveName(s string) (string, string, bool) {
|
|
for _, ext := range []string{".tar.gz", ".zip"} {
|
|
if name, ok := strings.CutSuffix(s, ext); ok && name != "" {
|
|
return name, ext, true
|
|
}
|
|
}
|
|
return "", "", false
|
|
}
|
|
|
|
func validSegment(s string) bool {
|
|
return segmentRe.MatchString(s) && s != "." && s != ".." && !strings.Contains(s, "..")
|
|
}
|
|
|
|
func validRefName(s string) bool {
|
|
for _, seg := range strings.Split(s, "/") {
|
|
if !validSegment(seg) || strings.HasPrefix(seg, ".") {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// ServeRemote rejects unsupported paths and redirects branch archives to the
|
|
// immutable archive of the branch's current commit; everything else is proxied.
|
|
func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool {
|
|
ref, ok := parsePath(path)
|
|
if !ok {
|
|
http.Error(w, "unsupported vcs path", http.StatusNotFound)
|
|
return true
|
|
}
|
|
if ref.Kind != branchArchive {
|
|
return false
|
|
}
|
|
classifier := proxy.NewClassifier(p)
|
|
if classifier.Classify(remote, path) == proxy.ClassDenied {
|
|
http.Error(w, "access denied", http.StatusForbidden)
|
|
return true
|
|
}
|
|
commit, err := p.resolve(r.Context(), remote, ref)
|
|
var se *provider.UpstreamStatusError
|
|
switch {
|
|
case errors.Is(err, errNotFound):
|
|
http.Error(w, "branch not found", http.StatusNotFound)
|
|
return true
|
|
case errors.As(err, &se) && (se.Status == http.StatusTooManyRequests || se.Status == http.StatusForbidden && !se.RetryAt.IsZero()):
|
|
secs := 60
|
|
if !se.RetryAt.IsZero() {
|
|
secs = max(1, int(time.Until(se.RetryAt).Seconds()+1))
|
|
}
|
|
w.Header().Set("Retry-After", strconv.Itoa(secs))
|
|
http.Error(w, "upstream rate limited", http.StatusServiceUnavailable)
|
|
return true
|
|
case err != nil:
|
|
http.Error(w, "bad gateway", http.StatusBadGateway)
|
|
return true
|
|
}
|
|
target := ref.Owner + "/" + ref.Repo + "/archive/" + commit + ref.Ext
|
|
if classifier.Classify(remote, target) == proxy.ClassDenied {
|
|
http.Error(w, "access denied", http.StatusForbidden)
|
|
return true
|
|
}
|
|
http.Redirect(w, r, strings.TrimRight(proxyBaseURL, "/")+provider.RemoteMountPath+remote.Name+"/"+target, http.StatusFound)
|
|
return true
|
|
}
|
|
|
|
// resolve maps a branch to its commit, reusing the answer for mutable_ttl (a
|
|
// missing branch for at least negativeTTL) and, with stale_on_error, past an
|
|
// upstream failure. Concurrent lookups of one branch share a single API call.
|
|
func (p *Provider) resolve(ctx context.Context, remote models.Remote, ref target) (string, error) {
|
|
key := ref.Owner + "/" + ref.Repo + "\x00" + ref.Name
|
|
ttl := time.Duration(remote.MutableTTL) * time.Second
|
|
p.mu.Lock()
|
|
prev, ok := p.refs[remote.Name][key]
|
|
p.mu.Unlock()
|
|
if ok && prev.commit != "" && time.Since(prev.at) < ttl {
|
|
return prev.commit, nil
|
|
}
|
|
if ok && prev.commit == "" && time.Since(prev.at) < max(ttl, negativeTTL) {
|
|
return "", errNotFound
|
|
}
|
|
v, err, _ := p.group.Do(remote.Name+"\x00"+key, func() (any, error) {
|
|
commit, err := p.lookup(context.WithoutCancel(ctx), remote, ref)
|
|
if err == nil || errors.Is(err, errNotFound) {
|
|
p.store(remote.Name, key, commit)
|
|
}
|
|
return commit, err
|
|
})
|
|
if err != nil {
|
|
if ok && prev.commit != "" && remote.StaleOnError && !errors.Is(err, errNotFound) {
|
|
return prev.commit, nil
|
|
}
|
|
return "", err
|
|
}
|
|
return v.(string), nil
|
|
}
|
|
|
|
// store records a lookup, evicting the remote's oldest entry once it holds maxRefs.
|
|
func (p *Provider) store(remote, key, commit string) {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
m := p.refs[remote]
|
|
if m == nil {
|
|
m = map[string]resolved{}
|
|
p.refs[remote] = m
|
|
}
|
|
if _, ok := m[key]; !ok && len(m) >= maxRefs {
|
|
var oldest string
|
|
for k, e := range m {
|
|
if oldest == "" || e.at.Before(m[oldest].at) {
|
|
oldest = k
|
|
}
|
|
}
|
|
delete(m, oldest)
|
|
}
|
|
m[key] = resolved{commit: commit, at: time.Now()}
|
|
}
|
|
|
|
func (p *Provider) lookup(ctx context.Context, remote models.Remote, ref target) (string, error) {
|
|
segs := strings.Split(ref.Name, "/")
|
|
for i, s := range segs {
|
|
segs[i] = url.PathEscape(s)
|
|
}
|
|
branch := strings.Join(segs, "/")
|
|
github := generic.IsGitHubHost(remote.BaseURL)
|
|
api := strings.TrimRight(remote.BaseURL, "/") + "/api/v1/repos/" + ref.Owner + "/" + ref.Repo + "/branches/" + branch
|
|
if github {
|
|
api = p.githubAPI + "/repos/" + ref.Owner + "/" + ref.Repo + "/git/ref/heads/" + branch
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, api, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
h, err := p.AuthHeaders(ctx, remote)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header = h
|
|
if github {
|
|
req.Header.Set("Accept", "application/vnd.github+json")
|
|
}
|
|
resp, err := p.client.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer func() { _ = resp.Body.Close() }()
|
|
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusUnprocessableEntity {
|
|
return "", errNotFound
|
|
}
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", provider.NewUpstreamStatusError(api, resp)
|
|
}
|
|
var b struct {
|
|
Commit struct {
|
|
ID string `json:"id"`
|
|
} `json:"commit"`
|
|
Object struct {
|
|
SHA string `json:"sha"`
|
|
} `json:"object"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&b); err != nil {
|
|
return "", err
|
|
}
|
|
commit := b.Commit.ID
|
|
if github {
|
|
commit = b.Object.SHA
|
|
}
|
|
if !commitRe.MatchString(commit) {
|
|
return "", fmt.Errorf("branch lookup %s: unexpected commit %q", api, commit)
|
|
}
|
|
return commit, nil
|
|
}
|