// Package vcs caches source archives and release assets from GitHub or Gitea, // keyed by ref: tags, commits and release assets are immutable, and branch // archives redirect to the archive of the commit the branch currently points at. package vcs import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "regexp" "strconv" "strings" "sync" "time" "golang.org/x/sync/singleflight" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/internal/provider/generic" "git.unkin.net/unkin/artifactapi/internal/proxy" "git.unkin.net/unkin/artifactapi/pkg/models" ) func init() { provider.Register(New()) } type kind int const ( tagArchive kind = iota + 1 branchArchive commitArchive releaseAsset latestReleaseAsset ) // target is a parsed vcs request path. type target struct { Kind kind Owner, Repo string Name string // tag, branch, commit or asset Ext string } var ( segmentRe = regexp.MustCompile(`^[A-Za-z0-9._+-]+$`) commitRe = regexp.MustCompile(`^[0-9a-f]{40}$`) ) var errNotFound = errors.New("ref not found") const ( maxRefs = 4096 negativeTTL = time.Minute ) // Provider embeds generic for upstream URLs, content types and GitHub/Basic auth. type Provider struct { generic.Provider client *http.Client githubAPI string group singleflight.Group mu sync.Mutex refs map[string]map[string]resolved } // resolved is a branch lookup; an empty commit records a missing branch. type resolved struct { commit string at time.Time } func New() *Provider { return &Provider{ client: &http.Client{Timeout: 30 * time.Second}, githubAPI: "https://api.github.com", refs: map[string]map[string]resolved{}, } } func (p *Provider) Type() models.PackageType { return models.PackageVCS } func (p *Provider) Classify(path string) provider.Mutability { if ref, ok := parsePath(path); ok && (ref.Kind == branchArchive || ref.Kind == latestReleaseAsset) { return provider.Mutable } return provider.Immutable } // parsePath accepts only the archive and release-download shapes, rejecting // any segment that could escape owner/repo or a ref. func parsePath(path string) (target, bool) { parts := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 3) if len(parts) != 3 || !validSegment(parts[0]) || !validSegment(parts[1]) { return target{}, false } ref := target{Owner: parts[0], Repo: parts[1]} rest := parts[2] switch { case strings.HasPrefix(rest, "archive/"): name, ext, ok := archiveName(strings.TrimPrefix(rest, "archive/")) if !ok { return target{}, false } ref.Ext = ext switch { case strings.HasPrefix(name, "refs/tags/"): ref.Kind, ref.Name = tagArchive, strings.TrimPrefix(name, "refs/tags/") case strings.HasPrefix(name, "refs/heads/"): ref.Kind, ref.Name = branchArchive, strings.TrimPrefix(name, "refs/heads/") case commitRe.MatchString(name): return target{Kind: commitArchive, Owner: ref.Owner, Repo: ref.Repo, Name: name, Ext: ext}, true default: return target{}, false } return ref, validRefName(ref.Name) case strings.HasPrefix(rest, "releases/latest/download/"): ref.Kind, ref.Name = latestReleaseAsset, strings.TrimPrefix(rest, "releases/latest/download/") return ref, validSegment(ref.Name) case strings.HasPrefix(rest, "releases/download/"): tagAsset := strings.TrimPrefix(rest, "releases/download/") i := strings.LastIndex(tagAsset, "/") if i < 0 { return target{}, false } ref.Kind, ref.Name = releaseAsset, tagAsset[i+1:] return ref, validRefName(tagAsset[:i]) && validSegment(ref.Name) } return target{}, false } func archiveName(s string) (string, string, bool) { for _, ext := range []string{".tar.gz", ".zip"} { if name, ok := strings.CutSuffix(s, ext); ok && name != "" { return name, ext, true } } return "", "", false } func validSegment(s string) bool { return segmentRe.MatchString(s) && s != "." && s != ".." && !strings.Contains(s, "..") } func validRefName(s string) bool { for _, seg := range strings.Split(s, "/") { if !validSegment(seg) || strings.HasPrefix(seg, ".") { return false } } return true } // ServeRemote rejects unsupported paths and redirects branch archives to the // immutable archive of the branch's current commit; everything else is proxied. func (p *Provider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, _ provider.RemoteMetadataStore) bool { ref, ok := parsePath(path) if !ok { http.Error(w, "unsupported vcs path", http.StatusNotFound) return true } if ref.Kind != branchArchive { return false } classifier := proxy.NewClassifier(p) if classifier.Classify(remote, path) == proxy.ClassDenied { http.Error(w, "access denied", http.StatusForbidden) return true } commit, err := p.resolve(r.Context(), remote, ref) var se *provider.UpstreamStatusError switch { case errors.Is(err, errNotFound): http.Error(w, "branch not found", http.StatusNotFound) return true case errors.As(err, &se) && (se.Status == http.StatusTooManyRequests || se.Status == http.StatusForbidden && !se.RetryAt.IsZero()): secs := 60 if !se.RetryAt.IsZero() { secs = max(1, int(time.Until(se.RetryAt).Seconds()+1)) } w.Header().Set("Retry-After", strconv.Itoa(secs)) http.Error(w, "upstream rate limited", http.StatusServiceUnavailable) return true case err != nil: http.Error(w, "bad gateway", http.StatusBadGateway) return true } target := ref.Owner + "/" + ref.Repo + "/archive/" + commit + ref.Ext if classifier.Classify(remote, target) == proxy.ClassDenied { http.Error(w, "access denied", http.StatusForbidden) return true } http.Redirect(w, r, strings.TrimRight(proxyBaseURL, "/")+provider.RemoteMountPath+remote.Name+"/"+target, http.StatusFound) return true } // resolve maps a branch to its commit, reusing the answer for mutable_ttl (a // missing branch for at least negativeTTL) and, with stale_on_error, past an // upstream failure. Concurrent lookups of one branch share a single API call. func (p *Provider) resolve(ctx context.Context, remote models.Remote, ref target) (string, error) { key := ref.Owner + "/" + ref.Repo + "\x00" + ref.Name ttl := time.Duration(remote.MutableTTL) * time.Second p.mu.Lock() prev, ok := p.refs[remote.Name][key] p.mu.Unlock() if ok && prev.commit != "" && time.Since(prev.at) < ttl { return prev.commit, nil } if ok && prev.commit == "" && time.Since(prev.at) < max(ttl, negativeTTL) { return "", errNotFound } v, err, _ := p.group.Do(remote.Name+"\x00"+key, func() (any, error) { commit, err := p.lookup(context.WithoutCancel(ctx), remote, ref) if err == nil || errors.Is(err, errNotFound) { p.store(remote.Name, key, commit) } return commit, err }) if err != nil { if ok && prev.commit != "" && remote.StaleOnError && !errors.Is(err, errNotFound) { return prev.commit, nil } return "", err } return v.(string), nil } // store records a lookup, evicting the remote's oldest entry once it holds maxRefs. func (p *Provider) store(remote, key, commit string) { p.mu.Lock() defer p.mu.Unlock() m := p.refs[remote] if m == nil { m = map[string]resolved{} p.refs[remote] = m } if _, ok := m[key]; !ok && len(m) >= maxRefs { var oldest string for k, e := range m { if oldest == "" || e.at.Before(m[oldest].at) { oldest = k } } delete(m, oldest) } m[key] = resolved{commit: commit, at: time.Now()} } func (p *Provider) lookup(ctx context.Context, remote models.Remote, ref target) (string, error) { segs := strings.Split(ref.Name, "/") for i, s := range segs { segs[i] = url.PathEscape(s) } branch := strings.Join(segs, "/") github := generic.IsGitHubHost(remote.BaseURL) api := strings.TrimRight(remote.BaseURL, "/") + "/api/v1/repos/" + ref.Owner + "/" + ref.Repo + "/branches/" + branch if github { api = p.githubAPI + "/repos/" + ref.Owner + "/" + ref.Repo + "/git/ref/heads/" + branch } req, err := http.NewRequestWithContext(ctx, http.MethodGet, api, nil) if err != nil { return "", err } h, err := p.AuthHeaders(ctx, remote) if err != nil { return "", err } req.Header = h if github { req.Header.Set("Accept", "application/vnd.github+json") } resp, err := p.client.Do(req) if err != nil { return "", err } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusUnprocessableEntity { return "", errNotFound } if resp.StatusCode != http.StatusOK { return "", provider.NewUpstreamStatusError(api, resp) } var b struct { Commit struct { ID string `json:"id"` } `json:"commit"` Object struct { SHA string `json:"sha"` } `json:"object"` } if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&b); err != nil { return "", err } commit := b.Commit.ID if github { commit = b.Object.SHA } if !commitRe.MatchString(commit) { return "", fmt.Errorf("branch lookup %s: unexpected commit %q", api, commit) } return commit, nil }