diff --git a/README.md b/README.md index 0f5edde..734b71a 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,71 @@ resource "artifactapi_remote_github_rpm" "acme-tools" { The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages from the synthesized metadata. +### GitHub authentication + +Anonymous GitHub is capped at **60 requests/hour** and cannot read private +repositories. Configure a **server-level GitHub credential** to raise the ceiling +to roughly **5000 requests/hour** and to read private-repo release assets. The +credential is a process-wide machine identity applied by default to *every* +outbound GitHub request — the releases scan, the ranged asset-header fetches, and +the generic-github byte proxy that streams private release assets. + +The credential is read from the environment (deliver it from a Vault or +Kubernetes secret). It is **never** stored per-remote in the database, **never** +returned by any API, and **never** logged. Configure **exactly one** mode. + +**Precedence.** A remote's own `username`/`password` credential still wins for +that remote's requests; the server credential is the default for everything else. +With no credential configured at all, requests stay anonymous (current behavior). +Partial configuration (e.g. an App id with no private key) is a **startup error** +— artifactapi fails closed rather than silently falling back to anonymous. + +Both modes share the syncer's single global rate limiter, so a token simply +raises the effective GitHub ceiling; the default limiter settings stay safe. + +#### Mode 1 — Personal Access Token (minimum viable, recommended for free accounts) + +Set `GITHUB_TOKEN`. It is sent as `Authorization: Bearer `. + +Recommended free-account setup — a **fine-grained PAT** scoped to just the target +repositories: + +1. GitHub → *Settings → Developer settings → Personal access tokens → + Fine-grained tokens → Generate new token*. +2. Limit *Repository access* to the specific repo(s) serving releases. +3. Grant repository permissions **Contents: Read-only** and **Metadata: + Read-only** (Metadata is mandatory and auto-selected). + +A classic PAT with the `repo` scope also works but is broader than necessary. + +```bash +GITHUB_TOKEN=github_pat_xxxxxxxx +``` + +#### Mode 2 — GitHub App installation token (proper machine identity) + +A GitHub App is not tied to a personal account and can be created and installed on +free personal repos. artifactapi mints a short-lived RS256 **JWT** from the app +private key, exchanges it at `POST /app/installations/{id}/access_tokens` for a +~1-hour **installation access token**, caches that token, and refreshes it a few +minutes before expiry (thread-safe, single-flighted). + +1. GitHub → *Settings → Developer settings → GitHub Apps → New GitHub App*. +2. Under *Permissions → Repository permissions* grant **Contents: Read-only** + (Metadata: Read-only is implied). +3. Generate a **private key** (downloads a PEM) and note the **App ID**. +4. *Install* the App on the account and select the target repositories, then read + the **Installation ID** from the installation URL + (`.../settings/installations/`). + +```bash +GITHUB_APP_ID=123456 +GITHUB_APP_INSTALLATION_ID=7654321 +GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem +# or inline PEM (e.g. mounted from a secret): +# GITHUB_APP_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\n...\n-----END RSA PRIVATE KEY-----" +``` + ## Terraform Remotes and virtuals are managed by Terraform. Each package type has its own resource: @@ -277,6 +342,11 @@ S3 client supports MinIO, Ceph RGW, and AWS S3 (via minio-go). | `GITHUB_SYNC_BURST` | `5` | Token-bucket burst for the shared limiter | | `GITHUB_SYNC_WORKERS` | `3` | Concurrent `github_rpm` scan workers | | `GITHUB_SYNC_POLL_INTERVAL` | `60` | Base scheduler tick in seconds; per-remote cadence is its `mutable_ttl`, enforced by the DB lease | +| `GITHUB_TOKEN` | | Server-level GitHub PAT (fine-grained or classic), sent as `Authorization: Bearer`. Applies to every GitHub request; per-remote creds override it. See [GitHub authentication](#github-authentication) | +| `GITHUB_APP_ID` | | GitHub App id (App auth mode; mutually exclusive with `GITHUB_TOKEN`) | +| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id | +| `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM | +| `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) | ## Development diff --git a/internal/config/env.go b/internal/config/env.go index 1addad3..80ac39f 100644 --- a/internal/config/env.go +++ b/internal/config/env.go @@ -44,6 +44,18 @@ type Config struct { GitHubSyncBurst int GitHubSyncWorkers int GitHubSyncPollInterval int + + // Server-level GitHub machine credential, applied by default to every + // outbound GitHub request (releases scan, ranged asset fetches, and the + // generic-github byte proxy for private assets). Delivered via env/secret + // only — never stored per-remote, never returned by an API, never logged. + // Configure exactly one mode: a Personal Access Token, or a GitHub App + // (id + installation id + private key). Partial App config fails at startup. + GitHubToken string + GitHubAppID string + GitHubAppInstallationID string + GitHubAppPrivateKey string + GitHubAppPrivateKeyPath string } func (c *Config) DatabaseDSN() string { @@ -105,6 +117,12 @@ func Load() (*Config, error) { GitHubSyncBurst: syncBurst, GitHubSyncWorkers: syncWorkers, GitHubSyncPollInterval: syncPoll, + + GitHubToken: getenv("GITHUB_TOKEN", ""), + GitHubAppID: getenv("GITHUB_APP_ID", ""), + GitHubAppInstallationID: getenv("GITHUB_APP_INSTALLATION_ID", ""), + GitHubAppPrivateKey: getenv("GITHUB_APP_PRIVATE_KEY", ""), + GitHubAppPrivateKeyPath: getenv("GITHUB_APP_PRIVATE_KEY_PATH", ""), } return cfg, nil diff --git a/internal/githubauth/app.go b/internal/githubauth/app.go new file mode 100644 index 0000000..707d586 --- /dev/null +++ b/internal/githubauth/app.go @@ -0,0 +1,199 @@ +package githubauth + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" +) + +const ( + defaultAPIBase = "https://api.github.com" + + // jwtLifetime is how long the app JWT is valid. GitHub caps it at 10 minutes; + // 9 leaves headroom for clock skew. + jwtLifetime = 9 * time.Minute + // jwtBackdate backdates iat to tolerate the app server's clock running behind + // GitHub's, which otherwise rejects the JWT. + jwtBackdate = 60 * time.Second + // refreshSkew refreshes the installation token this long before it expires so + // a request never races an expiry. + refreshSkew = 5 * time.Minute +) + +type httpDoer interface { + Do(*http.Request) (*http.Response, error) +} + +// appCredential mints installation access tokens for a GitHub App. It signs a +// short-lived RS256 JWT with the app private key, exchanges it for a ~1h +// installation token, caches that token, and refreshes it shortly before expiry. +// Refreshes are single-flighted by holding the mutex across the exchange, so +// concurrent callers coalesce onto one HTTP request and reuse the cached token. +type appCredential struct { + appID string + installationID string + key *rsa.PrivateKey + apiBase string + client httpDoer + + mu sync.Mutex + token string + expiry time.Time +} + +func newAppCredential(opts Options) (*appCredential, error) { + if opts.AppID == "" { + return nil, errors.New("github app: GITHUB_APP_ID is required") + } + if opts.InstallationID == "" { + return nil, errors.New("github app: GITHUB_APP_INSTALLATION_ID is required") + } + pemBytes, err := loadPrivateKeyPEM(opts) + if err != nil { + return nil, err + } + key, err := parseRSAPrivateKey(pemBytes) + if err != nil { + return nil, err + } + + apiBase := opts.apiBaseURL + if apiBase == "" { + apiBase = defaultAPIBase + } + client := opts.httpClient + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + + return &appCredential{ + appID: opts.AppID, + installationID: opts.InstallationID, + key: key, + apiBase: strings.TrimRight(apiBase, "/"), + client: client, + }, nil +} + +// Token returns a cached installation token, refreshing it under a single-flight +// lock when it is missing or within refreshSkew of expiry. +func (a *appCredential) Token(ctx context.Context) (string, error) { + a.mu.Lock() + defer a.mu.Unlock() + if a.token != "" && time.Now().Before(a.expiry.Add(-refreshSkew)) { + return a.token, nil + } + if err := a.refreshLocked(ctx); err != nil { + return "", err + } + return a.token, nil +} + +func (a *appCredential) refreshLocked(ctx context.Context) error { + jwt, err := mintJWT(a.appID, a.key, time.Now()) + if err != nil { + return fmt.Errorf("github app: mint jwt: %w", err) + } + + u := fmt.Sprintf("%s/app/installations/%s/access_tokens", a.apiBase, a.installationID) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u, nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+jwt) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + + resp, err := a.client.Do(req) + if err != nil { + return fmt.Errorf("github app: token exchange: %w", err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + // Never echo the body verbatim — it can contain sensitive material. + return fmt.Errorf("github app: token exchange status %d", resp.StatusCode) + } + + var out struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + if err := json.Unmarshal(body, &out); err != nil { + return fmt.Errorf("github app: decode token response: %w", err) + } + if out.Token == "" { + return errors.New("github app: token exchange returned an empty token") + } + a.token = out.Token + a.expiry = out.ExpiresAt + if a.expiry.IsZero() { + // Defensive: assume the documented ~1h lifetime if GitHub omits it. + a.expiry = time.Now().Add(time.Hour) + } + return nil +} + +// mintJWT builds and RS256-signs a GitHub App JWT (iss=app id, backdated iat, +// ≤10m exp) using stdlib crypto — no third-party JWT dependency. +func mintJWT(appID string, key *rsa.PrivateKey, now time.Time) (string, error) { + header := map[string]string{"alg": "RS256", "typ": "JWT"} + claims := map[string]any{ + "iat": now.Add(-jwtBackdate).Unix(), + "exp": now.Add(jwtLifetime).Unix(), + "iss": appID, + } + hb, err := json.Marshal(header) + if err != nil { + return "", err + } + cb, err := json.Marshal(claims) + if err != nil { + return "", err + } + signingInput := b64url(hb) + "." + b64url(cb) + digest := sha256.Sum256([]byte(signingInput)) + sig, err := rsa.SignPKCS1v15(rand.Reader, key, crypto.SHA256, digest[:]) + if err != nil { + return "", err + } + return signingInput + "." + b64url(sig), nil +} + +func b64url(b []byte) string { + return base64.RawURLEncoding.EncodeToString(b) +} + +// parseRSAPrivateKey accepts PKCS#1 ("RSA PRIVATE KEY") and PKCS#8 ("PRIVATE +// KEY") PEM, covering both GitHub App key export formats. +func parseRSAPrivateKey(pemBytes []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(pemBytes) + if block == nil { + return nil, errors.New("github app: private key is not valid PEM") + } + if key, err := x509.ParsePKCS1PrivateKey(block.Bytes); err == nil { + return key, nil + } + keyAny, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.New("github app: private key is not a supported RSA PKCS#1/PKCS#8 key") + } + rsaKey, ok := keyAny.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("github app: private key is not an RSA key") + } + return rsaKey, nil +} diff --git a/internal/githubauth/app_test.go b/internal/githubauth/app_test.go new file mode 100644 index 0000000..0fae448 --- /dev/null +++ b/internal/githubauth/app_test.go @@ -0,0 +1,207 @@ +package githubauth + +import ( + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +func testRSAKeyPEM(t *testing.T) string { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("generate key: %v", err) + } + der := x509.MarshalPKCS1PrivateKey(key) + return string(pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: der})) +} + +// appFixture serves the installation-token exchange endpoint, records requests, +// verifies the presented JWT against the app public key, and returns tokens with +// a controllable expiry. +type appFixture struct { + srv *httptest.Server + pub *rsa.PublicKey + mu sync.Mutex + exchanges int + lastJWT string + expiresAt func() time.Time + tokenSeq int +} + +func newAppFixture(t *testing.T, pemKey string) *appFixture { + t.Helper() + block, _ := pem.Decode([]byte(pemKey)) + key, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + t.Fatalf("parse test key: %v", err) + } + f := &appFixture{ + pub: &key.PublicKey, + expiresAt: func() time.Time { return time.Now().Add(time.Hour) }, + } + mux := http.NewServeMux() + mux.HandleFunc("/app/installations/456/access_tokens", func(w http.ResponseWriter, r *http.Request) { + auth := r.Header.Get("Authorization") + jwt := strings.TrimPrefix(auth, "Bearer ") + f.mu.Lock() + f.exchanges++ + f.lastJWT = jwt + f.tokenSeq++ + seq := f.tokenSeq + exp := f.expiresAt() + f.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + json.NewEncoder(w).Encode(map[string]any{ + "token": fmt.Sprintf("ghs_installation_%d", seq), + "expires_at": exp.UTC().Format(time.RFC3339), + }) + }) + f.srv = httptest.NewServer(mux) + t.Cleanup(f.srv.Close) + return f +} + +func (f *appFixture) verifyJWT(t *testing.T) { + t.Helper() + f.mu.Lock() + jwt := f.lastJWT + f.mu.Unlock() + parts := strings.Split(jwt, ".") + if len(parts) != 3 { + t.Fatalf("jwt not three-part: %q", jwt) + } + signingInput := parts[0] + "." + parts[1] + sig, err := base64.RawURLEncoding.DecodeString(parts[2]) + if err != nil { + t.Fatalf("decode sig: %v", err) + } + digest := sha256.Sum256([]byte(signingInput)) + if err := rsa.VerifyPKCS1v15(f.pub, crypto.SHA256, digest[:], sig); err != nil { + t.Fatalf("jwt signature invalid: %v", err) + } + var claims struct { + Iss string `json:"iss"` + Iat int64 `json:"iat"` + Exp int64 `json:"exp"` + } + cb, _ := base64.RawURLEncoding.DecodeString(parts[1]) + if err := json.Unmarshal(cb, &claims); err != nil { + t.Fatalf("decode claims: %v", err) + } + if claims.Iss != "123" { + t.Fatalf("iss = %q, want 123", claims.Iss) + } + if claims.Exp-claims.Iat > int64((10*time.Minute)/time.Second) { + t.Fatalf("jwt lifetime exceeds 10m: iat=%d exp=%d", claims.Iat, claims.Exp) + } + if claims.Iat > time.Now().Unix() { + t.Fatalf("iat not backdated: %d", claims.Iat) + } +} + +func newAppCred(t *testing.T, f *appFixture, pemKey string) *appCredential { + t.Helper() + c, err := newAppCredential(Options{ + AppID: "123", + InstallationID: "456", + PrivateKeyPEM: pemKey, + apiBaseURL: f.srv.URL, + httpClient: f.srv.Client(), + }) + if err != nil { + t.Fatalf("newAppCredential: %v", err) + } + return c +} + +func TestApp_MintsJWTAndExchangesForInstallationToken(t *testing.T) { + pemKey := testRSAKeyPEM(t) + f := newAppFixture(t, pemKey) + c := newAppCred(t, f, pemKey) + + tok, err := c.Token(context.Background()) + if err != nil { + t.Fatalf("token: %v", err) + } + if tok != "ghs_installation_1" { + t.Fatalf("token = %q, want ghs_installation_1", tok) + } + if f.exchanges != 1 { + t.Fatalf("exchanges = %d, want 1", f.exchanges) + } + f.verifyJWT(t) +} + +func TestApp_CachesInstallationToken(t *testing.T) { + pemKey := testRSAKeyPEM(t) + f := newAppFixture(t, pemKey) + c := newAppCred(t, f, pemKey) + + for i := 0; i < 5; i++ { + if _, err := c.Token(context.Background()); err != nil { + t.Fatalf("token: %v", err) + } + } + if f.exchanges != 1 { + t.Fatalf("exchanges = %d, want 1 (token should be cached)", f.exchanges) + } +} + +func TestApp_RefreshesNearExpiry(t *testing.T) { + pemKey := testRSAKeyPEM(t) + f := newAppFixture(t, pemKey) + // Token expires within refreshSkew, so every call must re-exchange. + f.expiresAt = func() time.Time { return time.Now().Add(2 * time.Minute) } + c := newAppCred(t, f, pemKey) + + t1, err := c.Token(context.Background()) + if err != nil { + t.Fatalf("token 1: %v", err) + } + t2, err := c.Token(context.Background()) + if err != nil { + t.Fatalf("token 2: %v", err) + } + if f.exchanges != 2 { + t.Fatalf("exchanges = %d, want 2 (near-expiry token must refresh)", f.exchanges) + } + if t1 == t2 { + t.Fatalf("expected a fresh token after refresh, both = %q", t1) + } +} + +func TestApp_ConcurrentTokenSingleFlights(t *testing.T) { + pemKey := testRSAKeyPEM(t) + f := newAppFixture(t, pemKey) + c := newAppCred(t, f, pemKey) + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := c.Token(context.Background()); err != nil { + t.Errorf("token: %v", err) + } + }() + } + wg.Wait() + if f.exchanges != 1 { + t.Fatalf("exchanges = %d, want 1 (concurrent calls must coalesce)", f.exchanges) + } +} diff --git a/internal/githubauth/credential.go b/internal/githubauth/credential.go new file mode 100644 index 0000000..31cfe71 --- /dev/null +++ b/internal/githubauth/credential.go @@ -0,0 +1,106 @@ +// Package githubauth provides the process-wide GitHub machine credential used to +// authenticate every outbound GitHub request (releases scan, ranged asset header +// fetches, and the generic-github byte proxy for private assets). The credential +// is delivered via env/secret only — it is never stored per-remote in the DB, +// never returned by any API, and never logged. +package githubauth + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" +) + +// Credential yields a bearer token for GitHub requests. Token may block to mint +// or refresh (the GitHub App path); an empty string means "no auth", which only +// happens when no credential is configured. +type Credential interface { + Token(ctx context.Context) (string, error) +} + +// Options is the raw, env-sourced auth configuration. Exactly one mode may be +// configured: a static token, or a GitHub App (id + installation id + private +// key). Partial App configuration is an error (fail closed); no fields at all is +// fine and yields a nil credential (anonymous, current behavior). +type Options struct { + // Token is a Personal Access Token (fine-grained or classic) sent verbatim + // as "Authorization: Bearer ". + Token string + + // GitHub App fields. PrivateKeyPEM and PrivateKeyPath are alternatives; the + // inline PEM wins when both are set. + AppID string + InstallationID string + PrivateKeyPEM string + PrivateKeyPath string + + // apiBaseURL overrides https://api.github.com for tests. Empty uses the real + // endpoint. httpClient likewise overrides the default client for tests. + apiBaseURL string + httpClient httpDoer +} + +// New builds the process credential from options, validating that auth is either +// fully configured or fully absent. It returns (nil, nil) when nothing is set. +func New(opts Options) (Credential, error) { + hasToken := opts.Token != "" + hasAppField := opts.AppID != "" || opts.InstallationID != "" || + opts.PrivateKeyPEM != "" || opts.PrivateKeyPath != "" + + switch { + case !hasToken && !hasAppField: + return nil, nil // no auth configured — anonymous is fine + case hasToken && hasAppField: + return nil, errors.New("github auth: both a token and GitHub App fields are set; configure exactly one") + case hasToken: + return staticToken{token: opts.Token}, nil + default: + return newAppCredential(opts) + } +} + +// staticToken is a fixed PAT credential. +type staticToken struct{ token string } + +func (s staticToken) Token(context.Context) (string, error) { return s.token, nil } + +// server is the process-wide credential set once at startup. A nil value means +// no server credential (anonymous). Access is guarded so a late SetServer in a +// test is race-free. +var ( + serverMu sync.RWMutex + server Credential +) + +// SetServer installs the process credential. Call once during startup. +func SetServer(c Credential) { + serverMu.Lock() + server = c + serverMu.Unlock() +} + +// Server returns the process credential, or nil if none is configured. +func Server() Credential { + serverMu.RLock() + defer serverMu.RUnlock() + return server +} + +// loadPrivateKeyPEM resolves the App private key bytes from the inline PEM or a +// file path, without ever returning the key material in an error message. +func loadPrivateKeyPEM(opts Options) ([]byte, error) { + if strings.TrimSpace(opts.PrivateKeyPEM) != "" { + return []byte(opts.PrivateKeyPEM), nil + } + if opts.PrivateKeyPath != "" { + b, err := os.ReadFile(opts.PrivateKeyPath) + if err != nil { + return nil, fmt.Errorf("github app: read private key file: %w", err) + } + return b, nil + } + return nil, errors.New("github app: no private key configured") +} diff --git a/internal/githubauth/credential_test.go b/internal/githubauth/credential_test.go new file mode 100644 index 0000000..87afe91 --- /dev/null +++ b/internal/githubauth/credential_test.go @@ -0,0 +1,77 @@ +package githubauth + +import ( + "context" + "testing" +) + +func TestNew_NoConfigIsAnonymous(t *testing.T) { + c, err := New(Options{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if c != nil { + t.Fatalf("expected nil credential when nothing configured, got %T", c) + } +} + +func TestNew_TokenMode(t *testing.T) { + c, err := New(Options{Token: "ghp_example"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + tok, err := c.Token(context.Background()) + if err != nil { + t.Fatalf("token: %v", err) + } + if tok != "ghp_example" { + t.Fatalf("token = %q, want ghp_example", tok) + } +} + +func TestNew_TokenAndAppConflict(t *testing.T) { + _, err := New(Options{Token: "ghp_example", AppID: "123"}) + if err == nil { + t.Fatal("expected error when both token and app fields are set") + } +} + +func TestNew_PartialAppFailsClosed(t *testing.T) { + cases := map[string]Options{ + "app id without key": {AppID: "123", InstallationID: "456"}, + "key without app id": {InstallationID: "456", PrivateKeyPEM: testRSAKeyPEM(t)}, + "app id without inst": {AppID: "123", PrivateKeyPEM: testRSAKeyPEM(t)}, + } + for name, opts := range cases { + t.Run(name, func(t *testing.T) { + if _, err := New(opts); err == nil { + t.Fatalf("expected fail-closed error for %q", name) + } + }) + } +} + +func TestNew_AppModeParsesKey(t *testing.T) { + c, err := New(Options{ + AppID: "123", + InstallationID: "456", + PrivateKeyPEM: testRSAKeyPEM(t), + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if _, ok := c.(*appCredential); !ok { + t.Fatalf("expected *appCredential, got %T", c) + } +} + +func TestNew_AppModeRejectsBadKey(t *testing.T) { + _, err := New(Options{ + AppID: "123", + InstallationID: "456", + PrivateKeyPEM: "-----BEGIN RSA PRIVATE KEY-----\nnope\n-----END RSA PRIVATE KEY-----", + }) + if err == nil { + t.Fatal("expected error for malformed private key") + } +} diff --git a/internal/provider/generic/generic.go b/internal/provider/generic/generic.go index 3374eb0..c909bf2 100644 --- a/internal/provider/generic/generic.go +++ b/internal/provider/generic/generic.go @@ -4,9 +4,11 @@ import ( "context" "encoding/base64" "net/http" + "net/url" "path" "strings" + "git.unkin.net/unkin/artifactapi/internal/githubauth" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -59,10 +61,42 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, return nil, nil } -func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { +// AuthHeaders authenticates outbound requests. A per-remote username/password +// (Basic auth) takes precedence. Otherwise, when the remote points at a GitHub +// host (e.g. a releases_remote proxying private release assets), the process-wide +// GitHub credential is attached as a bearer token so private downloads work. +func (p *Provider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) { h := http.Header{} if remote.Username != "" { h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password))) + return h, nil + } + if isGitHubHost(remote.BaseURL) { + if c := githubauth.Server(); c != nil { + tok, err := c.Token(ctx) + if err != nil { + return nil, err + } + if tok != "" { + h.Set("Authorization", "Bearer "+tok) + } + } } return h, nil } + +// isGitHubHost reports whether rawURL targets a GitHub API/download host that +// accepts the server credential. objects.githubusercontent.com is deliberately +// excluded: release-asset downloads 302-redirect there with a pre-signed URL +// that must not carry an Authorization header. +func isGitHubHost(rawURL string) bool { + u, err := url.Parse(rawURL) + if err != nil { + return false + } + switch strings.ToLower(u.Hostname()) { + case "github.com", "www.github.com", "api.github.com", "codeload.github.com", "uploads.github.com": + return true + } + return false +} diff --git a/internal/provider/generic/generic_test.go b/internal/provider/generic/generic_test.go index f3c683c..2044e38 100644 --- a/internal/provider/generic/generic_test.go +++ b/internal/provider/generic/generic_test.go @@ -4,11 +4,56 @@ import ( "context" "testing" + "git.unkin.net/unkin/artifactapi/internal/githubauth" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/internal/provider/generic" "git.unkin.net/unkin/artifactapi/pkg/models" ) +type staticCred string + +func (s staticCred) Token(context.Context) (string, error) { return string(s), nil } + +func TestProvider_AuthHeaders_GitHubServerCredential(t *testing.T) { + githubauth.SetServer(staticCred("ghs_server")) + t.Cleanup(func() { githubauth.SetServer(nil) }) + + p := &generic.Provider{} + h, err := p.AuthHeaders(context.Background(), models.Remote{BaseURL: "https://github.com"}) + if err != nil { + t.Fatalf("auth headers: %v", err) + } + if h.Get("Authorization") != "Bearer ghs_server" { + t.Fatalf("Authorization = %q, want Bearer ghs_server", h.Get("Authorization")) + } +} + +func TestProvider_AuthHeaders_NonGitHubHostNoServerCredential(t *testing.T) { + githubauth.SetServer(staticCred("ghs_server")) + t.Cleanup(func() { githubauth.SetServer(nil) }) + + p := &generic.Provider{} + h, _ := p.AuthHeaders(context.Background(), models.Remote{BaseURL: "https://example.com/downloads"}) + if h.Get("Authorization") != "" { + t.Fatalf("server credential must not be sent to non-github host, got %q", h.Get("Authorization")) + } +} + +func TestProvider_AuthHeaders_PerRemoteOverridesServerCredential(t *testing.T) { + githubauth.SetServer(staticCred("ghs_server")) + t.Cleanup(func() { githubauth.SetServer(nil) }) + + p := &generic.Provider{} + h, _ := p.AuthHeaders(context.Background(), models.Remote{ + BaseURL: "https://github.com", + Username: "user", + Password: "pass", + }) + if got := h.Get("Authorization"); got != "Basic dXNlcjpwYXNz" { + t.Fatalf("per-remote Basic auth must win, got %q", got) + } +} + func TestProvider_Type(t *testing.T) { p := &generic.Provider{} if p.Type() != models.PackageGeneric { diff --git a/internal/provider/rpm/github.go b/internal/provider/rpm/github.go index 03343dd..e28253f 100644 --- a/internal/provider/rpm/github.go +++ b/internal/provider/rpm/github.go @@ -20,6 +20,7 @@ import ( rpmlib "github.com/cavaliergopher/rpm" "golang.org/x/time/rate" + "git.unkin.net/unkin/artifactapi/internal/githubauth" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -76,6 +77,10 @@ type GitHubProvider struct { // the shared background work queue instead of an inline per-replica scan. syncer *Syncer + // serverCred overrides the process-wide GitHub credential for this provider + // instance. nil falls back to githubauth.Server(); set directly in tests. + serverCred githubauth.Credential + mu sync.Mutex scanning map[string]bool lastScan map[string]time.Time @@ -136,8 +141,8 @@ func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([ return nil, nil } -func (p *GitHubProvider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { - return githubHeaders(remote, false), nil +func (p *GitHubProvider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) { + return p.githubHeaders(ctx, remote, false) } // ServeRemote answers a request against a github_rpm remote. It refreshes the @@ -414,7 +419,11 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote if err != nil { return nil, "", false, err } - copyHeaders(req, githubHeaders(remote, true)) + hdr, err := p.githubHeaders(ctx, remote, true) + if err != nil { + return nil, "", false, err + } + copyHeaders(req, hdr) if page == 1 && etag != "" { req.Header.Set("If-None-Match", etag) } @@ -571,7 +580,11 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow if err != nil { return nil, false, err } - copyHeaders(req, githubHeaders(remote, false)) + hdr, err := p.githubHeaders(ctx, remote, false) + if err != nil { + return nil, false, err + } + copyHeaders(req, hdr) req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1)) if err := p.limiterWait(ctx); err != nil { @@ -599,7 +612,11 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote if err != nil { return "", err } - copyHeaders(req, githubHeaders(remote, false)) + hdr, err := p.githubHeaders(ctx, remote, false) + if err != nil { + return "", err + } + copyHeaders(req, hdr) if err := p.limiterWait(ctx); err != nil { return "", err @@ -638,23 +655,48 @@ func sha256FromDigest(digest string) (string, bool) { return "", false } -func githubHeaders(remote models.Remote, api bool) http.Header { +// githubHeaders builds the outbound headers for a GitHub request, attaching a +// bearer credential when one is available. A per-remote credential wins; absent +// that, the process-wide server credential is used; absent both, the request is +// unauthenticated (anonymous, subject to the 60/hr cap). +func (p *GitHubProvider) githubHeaders(ctx context.Context, remote models.Remote, api bool) (http.Header, error) { h := http.Header{} if api { h.Set("Accept", "application/vnd.github+json") h.Set("X-GitHub-Api-Version", "2022-11-28") } - if tok := githubToken(remote); tok != "" { + tok, err := p.githubToken(ctx, remote) + if err != nil { + return nil, err + } + if tok != "" { h.Set("Authorization", "Bearer "+tok) } - return h + return h, nil } -func githubToken(remote models.Remote) string { +// githubToken resolves the bearer token for a remote. Precedence: a per-remote +// credential (password, then username) overrides the server credential. +func (p *GitHubProvider) githubToken(ctx context.Context, remote models.Remote) (string, error) { if remote.Password != "" { - return remote.Password + return remote.Password, nil } - return remote.Username + if remote.Username != "" { + return remote.Username, nil + } + if c := p.serverCredential(); c != nil { + return c.Token(ctx) + } + return "", nil +} + +// serverCredential returns this provider's server credential, defaulting to the +// process-wide one installed at startup. +func (p *GitHubProvider) serverCredential() githubauth.Credential { + if p.serverCred != nil { + return p.serverCred + } + return githubauth.Server() } func copyHeaders(req *http.Request, h http.Header) { diff --git a/internal/provider/rpm/github_auth_test.go b/internal/provider/rpm/github_auth_test.go new file mode 100644 index 0000000..a1b7de0 --- /dev/null +++ b/internal/provider/rpm/github_auth_test.go @@ -0,0 +1,134 @@ +package rpm + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/githubauth" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// staticCred is a test Credential yielding a fixed token. +type staticCred string + +func (s staticCred) Token(context.Context) (string, error) { return string(s), nil } + +func TestGitHubServerCredentialAttachedToReleasesAndAssets(t *testing.T) { + fx := newGitHubFixture(t, true) + p := newTestProvider() + p.serverCred = staticCred("ghp_server_secret") + store := newFakeStore() + + if err := p.scan(context.Background(), fx.remote(), store); err != nil { + t.Fatalf("scan: %v", err) + } + if got := fx.releaseAuth; got != "Bearer ghp_server_secret" { + t.Fatalf("releases Authorization = %q, want Bearer ghp_server_secret", got) + } + if got := fx.assetAuth; got != "Bearer ghp_server_secret" { + t.Fatalf("asset Authorization = %q, want Bearer ghp_server_secret", got) + } +} + +func TestGitHubPerRemoteCredentialOverridesServer(t *testing.T) { + fx := newGitHubFixture(t, true) + p := newTestProvider() + p.serverCred = staticCred("ghp_server_secret") + store := newFakeStore() + + remote := fx.remote() + remote.Password = "ghp_remote_wins" + + if err := p.scan(context.Background(), remote, store); err != nil { + t.Fatalf("scan: %v", err) + } + if got := fx.releaseAuth; got != "Bearer ghp_remote_wins" { + t.Fatalf("releases Authorization = %q, want per-remote token to win", got) + } + if got := fx.assetAuth; got != "Bearer ghp_remote_wins" { + t.Fatalf("asset Authorization = %q, want per-remote token to win", got) + } +} + +func TestGitHubNoCredentialSendsNoAuthHeader(t *testing.T) { + fx := newGitHubFixture(t, true) + p := newTestProvider() // serverCred nil, package Server() unset in unit tests + store := newFakeStore() + + if err := p.scan(context.Background(), fx.remote(), store); err != nil { + t.Fatalf("scan: %v", err) + } + if fx.releaseAuth != "" { + t.Fatalf("expected no Authorization header, got %q", fx.releaseAuth) + } + if fx.assetAuth != "" { + t.Fatalf("expected no asset Authorization header, got %q", fx.assetAuth) + } + // Requests still succeed anonymously. + if rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm"); len(rows) != 1 { + t.Fatalf("anonymous scan should still derive metadata, got %d rows", len(rows)) + } +} + +func TestGitHubETag304FlowWithAuth(t *testing.T) { + fx := newGitHubFixture(t, true) + fx.etag = `"v1"` + p := newTestProvider() + p.serverCred = staticCred("ghp_server_secret") + store := newFakeStore() + + etag, changed, err := p.scanWithState(context.Background(), fx.remote(), store, "") + if err != nil { + t.Fatalf("first scan: %v", err) + } + if !changed || etag != `"v1"` { + t.Fatalf("first scan changed=%v etag=%q, want true and \"v1\"", changed, etag) + } + + // Re-scan with the captured ETag: a 304 means no change and no asset fetch. + etag2, changed2, err := p.scanWithState(context.Background(), fx.remote(), store, etag) + if err != nil { + t.Fatalf("second scan: %v", err) + } + if changed2 { + t.Fatal("expected no change on 304") + } + if etag2 != `"v1"` { + t.Fatalf("etag = %q, want preserved \"v1\"", etag2) + } + if fx.notModHit != 1 { + t.Fatalf("expected exactly one 304 response, got %d", fx.notModHit) + } + // The conditional request still carried the credential. + if fx.releaseAuth != "Bearer ghp_server_secret" { + t.Fatalf("conditional request Authorization = %q, want the server credential", fx.releaseAuth) + } +} + +// TestGitHubCredentialAbsentFromRemoteJSON asserts the server credential never +// appears in a remote's serialized API representation, and per-remote secrets +// stay redacted by the models.Remote json:"-" tags. +func TestGitHubCredentialAbsentFromRemoteJSON(t *testing.T) { + githubauth.SetServer(staticCred("ghp_super_secret_server_token")) + t.Cleanup(func() { githubauth.SetServer(nil) }) + + remote := models.Remote{ + Name: "acme-rpm", + PackageType: models.PackageGitHubRPM, + BaseURL: "https://api.github.com/repos/acme/tools", + Username: "per_remote_user", + Password: "per_remote_secret", + } + b, err := json.Marshal(remote) + if err != nil { + t.Fatalf("marshal remote: %v", err) + } + js := string(b) + for _, secret := range []string{"ghp_super_secret_server_token", "per_remote_secret", "per_remote_user"} { + if strings.Contains(js, secret) { + t.Fatalf("credential %q leaked into remote JSON: %s", secret, js) + } + } +} diff --git a/internal/provider/rpm/github_test.go b/internal/provider/rpm/github_test.go index f2eba4e..b8d523d 100644 --- a/internal/provider/rpm/github_test.go +++ b/internal/provider/rpm/github_test.go @@ -73,6 +73,8 @@ type githubFixture struct { etag string // when set, served as ETag; matching If-None-Match yields 304 releasesHit int // total releases-list requests (200 + 304) notModHit int // releases-list requests answered 304 + releaseAuth string // Authorization header seen on the last releases request + assetAuth string // Authorization header seen on the last asset request mu sync.Mutex } @@ -94,6 +96,7 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture { } f.mu.Lock() f.releasesHit++ + f.releaseAuth = r.Header.Get("Authorization") etag := f.etag if etag != "" && r.Header.Get("If-None-Match") == etag { f.notModHit++ @@ -130,6 +133,7 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture { } rng := r.Header.Get("Range") f.mu.Lock() + f.assetAuth = r.Header.Get("Authorization") if rng != "" { f.rangeHit[name]++ } else { diff --git a/internal/server/server.go b/internal/server/server.go index 4037d5a..1842ad7 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -19,6 +19,7 @@ import ( "git.unkin.net/unkin/artifactapi/internal/config" "git.unkin.net/unkin/artifactapi/internal/database" "git.unkin.net/unkin/artifactapi/internal/gc" + "git.unkin.net/unkin/artifactapi/internal/githubauth" _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine" _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" @@ -66,6 +67,25 @@ func New(cfg *config.Config, version string) (*Server, error) { return nil, fmt.Errorf("s3: %w", err) } + // Install the process-wide GitHub credential before any provider makes an + // outbound call. A misconfiguration (e.g. App id without a private key) fails + // closed here rather than silently falling back to anonymous. No credential + // configured is fine — requests stay anonymous. + ghCred, err := githubauth.New(githubauth.Options{ + Token: cfg.GitHubToken, + AppID: cfg.GitHubAppID, + InstallationID: cfg.GitHubAppInstallationID, + PrivateKeyPEM: cfg.GitHubAppPrivateKey, + PrivateKeyPath: cfg.GitHubAppPrivateKeyPath, + }) + if err != nil { + return nil, fmt.Errorf("github auth: %w", err) + } + githubauth.SetServer(ghCred) + if ghCred != nil { + slog.Info("github machine credential configured") + } + engine := proxy.NewEngine(db, redis, s3) localHandler := v2.NewLocalHandler(db, s3) virtEngine := virtual.NewEngine(db, engine)