unkinben
109ba2ce27
ci/woodpecker/tag/docker Pipeline was successful
## Why Anonymous GitHub is capped at 60 requests/hour and cannot read private repositories. A machine credential usable by a free (non-enterprise) account is needed to lift the request budget to ~5000/hr and to read private-repo release assets. Builds on the background syncer (#108, now merged to `master`); this diff is the auth changes only. ## How - Add `internal/githubauth`: a process-wide GitHub credential delivered via env/secret, applied by default to every outbound GitHub request (releases scan, ranged asset-header GETs, and the generic-github byte proxy for private assets). - Support two modes: - **PAT** — `GITHUB_TOKEN` sent as `Authorization: Bearer <token>`. - **GitHub App** — `GITHUB_APP_ID` + `GITHUB_APP_INSTALLATION_ID` + private key (`GITHUB_APP_PRIVATE_KEY` inline PEM or `GITHUB_APP_PRIVATE_KEY_PATH`). Mint a short-lived RS256 JWT with stdlib `crypto/rsa` (no new dependency), exchange it at `POST /app/installations/{id}/access_tokens` for a ~1h installation token, cache it, and single-flight a refresh a few minutes before expiry. - Inject at the two GitHub call paths: the rpm github provider header builder (releases + ranged fetches) and the generic provider `AuthHeaders` (byte proxy, github.com hosts only; the pre-signed `objects.githubusercontent.com` redirect deliberately gets no Authorization). - Honor precedence: a remote's own `username`/`password` overrides the server credential; no credential configured stays anonymous (current behavior). - Fail closed at startup on partial App configuration (e.g. App id without a private key); a token-and-App conflict is also rejected. - Never persist the credential to the DB, return it from an API, or log it (token-exchange failures never echo the response body). - Read config via the existing `getenv` convention; document PAT vs App setup, the free-account fine-grained PAT scopes (Contents:read + Metadata:read), precedence, and the rate-limit implication. ## Rate limit Authenticated requests share the syncer's single global limiter — no second limiter is added. A token raises the effective GitHub ceiling (~5000/hr vs ~60/hr), so the limiter defaults stay safe. ## Tests `internal/githubauth` and `internal/provider/{rpm,generic}`: - PAT attaches the correct `Authorization` header to releases + asset-header requests. - App mints a valid RS256 JWT (verified against the app public key), exchanges it at a mocked endpoint, reuses the cached token without re-exchanging, refreshes near expiry, and single-flights concurrent callers. - Per-remote credential overrides the server credential (rpm + generic). - No credential → no `Authorization` header, requests still succeed anonymously. - ETag/304 flow still works with auth attached. - The credential does not appear in a remote's serialized JSON. - Config validation: no-config is anonymous; partial App config and token/App conflict both error. Verified fail-before/pass-after for the injection tests. `gofmt -l`, `go build ./...`, `go vet ./...`, `go test ./...` all clean (26 packages). Reviewed-on: #109 Co-authored-by: Ben Vincent <ben@unkin.net> Co-committed-by: Ben Vincent <ben@unkin.net>
103 lines
2.8 KiB
Go
103 lines
2.8 KiB
Go
package generic
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"net/http"
|
|
"net/url"
|
|
"path"
|
|
"strings"
|
|
|
|
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
|
"git.unkin.net/unkin/artifactapi/internal/provider"
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
func init() {
|
|
provider.Register(&Provider{})
|
|
}
|
|
|
|
type Provider struct{}
|
|
|
|
func (p *Provider) Type() models.PackageType { return models.PackageGeneric }
|
|
|
|
func (p *Provider) Classify(_ string) provider.Mutability {
|
|
return provider.Immutable
|
|
}
|
|
|
|
var contentTypeMap = map[string]string{
|
|
".tar.gz": "application/gzip",
|
|
".tgz": "application/gzip",
|
|
".gz": "application/gzip",
|
|
".zip": "application/zip",
|
|
".whl": "application/zip",
|
|
".exe": "application/x-msdownload",
|
|
".rpm": "application/x-rpm",
|
|
".xml": "application/xml",
|
|
".yaml": "text/yaml",
|
|
".yml": "text/yaml",
|
|
".json": "application/json",
|
|
".sig": "application/octet-stream",
|
|
}
|
|
|
|
func (p *Provider) ContentType(filePath string) string {
|
|
lower := strings.ToLower(filePath)
|
|
if strings.HasSuffix(lower, ".tar.gz") {
|
|
return "application/gzip"
|
|
}
|
|
ext := path.Ext(lower)
|
|
if ct, ok := contentTypeMap[ext]; ok {
|
|
return ct
|
|
}
|
|
return "application/octet-stream"
|
|
}
|
|
|
|
func (p *Provider) UpstreamURL(remote models.Remote, reqPath string) string {
|
|
base := strings.TrimRight(remote.BaseURL, "/")
|
|
return base + "/" + strings.TrimLeft(reqPath, "/")
|
|
}
|
|
|
|
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
// AuthHeaders authenticates outbound requests. A per-remote username/password
|
|
// (Basic auth) takes precedence. Otherwise, when the remote points at a GitHub
|
|
// host (e.g. a releases_remote proxying private release assets), the process-wide
|
|
// GitHub credential is attached as a bearer token so private downloads work.
|
|
func (p *Provider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) {
|
|
h := http.Header{}
|
|
if remote.Username != "" {
|
|
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
|
|
return h, nil
|
|
}
|
|
if isGitHubHost(remote.BaseURL) {
|
|
if c := githubauth.Server(); c != nil {
|
|
tok, err := c.Token(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tok != "" {
|
|
h.Set("Authorization", "Bearer "+tok)
|
|
}
|
|
}
|
|
}
|
|
return h, nil
|
|
}
|
|
|
|
// isGitHubHost reports whether rawURL targets a GitHub API/download host that
|
|
// accepts the server credential. objects.githubusercontent.com is deliberately
|
|
// excluded: release-asset downloads 302-redirect there with a pre-signed URL
|
|
// that must not carry an Authorization header.
|
|
func isGitHubHost(rawURL string) bool {
|
|
u, err := url.Parse(rawURL)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
switch strings.ToLower(u.Hostname()) {
|
|
case "github.com", "www.github.com", "api.github.com", "codeload.github.com", "uploads.github.com":
|
|
return true
|
|
}
|
|
return false
|
|
}
|