From b727b990a272b7ced24378d851f769da95a5980c Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Mon, 10 Aug 2026 09:35:22 +1000 Subject: [PATCH] feat: add github_rpm metadata-only remote serving GitHub releases as a yum repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Publishing RPMs to GitHub releases is common, but consuming them with dnf requires repodata that GitHub does not provide, and mirroring every package into a local repo wastes storage and staleness-tracking on artifacts that already have a durable home. Expose GitHub releases as a first-class RPM source that synthesizes repodata on the fly and never precaches the packages. Add a `github_rpm` remote package type backed by a metadata-only provider: - Introduce a `RemoteServer` interception hook (the remote-side analog of `LocalIndexer`): `handleProxy` lets a provider fully answer a request before the byte-proxy engine, passing the request-derived proxy base URL and the DB as a `RemoteMetadataStore`. - Scan a repo's releases via the GitHub API (`base_url` = the releases API root) for `.rpm` assets, filtered by the remote's `patterns` (regex on asset filename) and reuse the existing local-rpm repodata generators to emit `repomd.xml`/`primary`/`filelists`/`other`. - Derive per-asset metadata without precaching: fetch only the RPM header via a ranged GET (retrying with a larger range on a truncated-header parse) for NEVRA, requires/provides/conflicts/obsoletes and files; take the sha256 from the GitHub asset `digest` when present, else compute it once by streaming. - Cache derived metadata in `rpm_metadata` keyed by asset path; re-scan no more often than `mutable_ttl`, pruning assets that disappear upstream. - Serve each package's `` as the github-relative download path so the client comes back to this remote, which 302-redirects to the `releases_remote` (an existing generic github.com remote) that streams the actual bytes. Reuse the existing `releases_remote` field as the redirect target — it already carries exactly this "downloads served by remote X" semantic end to end. Extend the shared RPM metadata model with conflicts/obsoletes (JSONB columns, added idempotently) so both local and github_rpm repodata resolve upgrades and conflicts; the local upload path now records them too. Tests cover header-range parsing with the retry loop, digest-vs-computed checksum selection, repodata synthesis with the redirect-able location href, the 302 redirect path, asset pattern filtering, and stale-asset pruning. --- README.md | 45 +++ internal/api/v1/proxy.go | 9 + internal/database/postgres.go | 5 + internal/database/rpm_metadata.go | 16 +- internal/provider/provider.go | 20 + internal/provider/rpm/github.go | 521 +++++++++++++++++++++++++++ internal/provider/rpm/github_test.go | 324 +++++++++++++++++ internal/provider/rpm/rpm.go | 26 ++ pkg/models/package_type.go | 2 + pkg/models/package_type_test.go | 1 + 10 files changed, 964 insertions(+), 5 deletions(-) create mode 100644 internal/provider/rpm/github.go create mode 100644 internal/provider/rpm/github_test.go diff --git a/README.md b/README.md index a9bbf2c..86aed17 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,54 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` | `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | | `terraform` | `*/versions` | `*/download/*/*` | | `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | +| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | Providers classify paths automatically. Users only configure what to proxy and TTLs. +### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache) + +A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` +repository without ever caching the packages. It scans releases for `.rpm` +assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ +obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package +metadata comes from a **ranged GET of just the RPM header** (the header sits at +the front of the file, so the whole package is never downloaded); the sha256 +checksum comes from the GitHub asset `digest` when present, else a one-time +lazy stream. Derived metadata is cached (keyed by asset) so repodata generation +is cheap on repeat, and refreshed no more often than `mutable_ttl`. + +Each package's `` points back at the remote, which **302-redirects** +the download to the `releases_remote` — an existing generic `github.com` remote +that streams the actual bytes. `dnf` follows the redirect transparently. + +```hcl +# Backend that serves the actual .rpm bytes from github.com. +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + patterns = [ + "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets + ] +} + +resource "artifactapi_remote_github_rpm" "acme-tools" { + name = "acme-tools" + base_url = "https://api.github.com/repos/acme/tools" # the releases API root + releases_remote = "github" # backend for downloads + mutable_ttl = 3600 # release re-scan interval + + # Optional: restrict which release assets become packages (regex on filename). + patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] + + # Optional: a token for private repos / higher API rate limits. + # password = "ghp_..." +} +``` + +`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. +The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages +from the synthesized metadata. + ## Terraform Remotes and virtuals are managed by Terraform. Each package type has its own resource: diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 377b784..bb0474b 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -83,6 +83,15 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { return } + // Metadata-only remotes (e.g. github_rpm) synthesize their own responses and + // redirect package downloads to a backend remote instead of proxying bytes. + if rs, ok := prov.(provider.RemoteServer); ok { + proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host) + if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) { + return + } + } + result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header) if err != nil { var proxyErr *proxy.ProxyError diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 0e5b4e8..baf2303 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -151,6 +151,8 @@ func (db *DB) migrate() error { packager TEXT DEFAULT '', requires JSONB DEFAULT '[]', provides JSONB DEFAULT '[]', + conflicts JSONB DEFAULT '[]', + obsoletes JSONB DEFAULT '[]', files JSONB DEFAULT '[]', changelogs JSONB DEFAULT '[]', created_at TIMESTAMPTZ DEFAULT NOW(), @@ -159,6 +161,9 @@ func (db *DB) migrate() error { CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name); + ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]'; + ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]'; + CREATE TABLE IF NOT EXISTS signing_keys ( purpose TEXT PRIMARY KEY, private_key_armor TEXT NOT NULL, diff --git a/internal/database/rpm_metadata.go b/internal/database/rpm_metadata.go index e0fc38a..a5cd842 100644 --- a/internal/database/rpm_metadata.go +++ b/internal/database/rpm_metadata.go @@ -10,6 +10,8 @@ import ( func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error { requiresJSON, _ := json.Marshal(meta.Requires) providesJSON, _ := json.Marshal(meta.Provides) + conflictsJSON, _ := json.Marshal(meta.Conflicts) + obsoletesJSON, _ := json.Marshal(meta.Obsoletes) filesJSON, _ := json.Marshal(meta.Files) changelogsJSON, _ := json.Marshal(meta.Changelogs) @@ -19,15 +21,15 @@ func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) name, epoch, version, release, arch, summary, description, rpm_size, installed_size, license, vendor, build_group, build_host, source_rpm, url, packager, - requires, provides, files, changelogs - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23) + requires, provides, conflicts, obsoletes, files, changelogs + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25) ON CONFLICT (repo_name, file_path) DO NOTHING `, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch, meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize, meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager, - requiresJSON, providesJSON, filesJSON, changelogsJSON, + requiresJSON, providesJSON, conflictsJSON, obsoletesJSON, filesJSON, changelogsJSON, ) return err } @@ -59,6 +61,8 @@ type RPMMetadataRow struct { Packager string Requires json.RawMessage Provides json.RawMessage + Conflicts json.RawMessage + Obsoletes json.RawMessage Files json.RawMessage Changelogs json.RawMessage } @@ -93,6 +97,8 @@ func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]pr } json.Unmarshal(r.Requires, &meta.Requires) json.Unmarshal(r.Provides, &meta.Provides) + json.Unmarshal(r.Conflicts, &meta.Conflicts) + json.Unmarshal(r.Obsoletes, &meta.Obsoletes) json.Unmarshal(r.Files, &meta.Files) json.Unmarshal(r.Changelogs, &meta.Changelogs) result[i] = meta @@ -106,7 +112,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada name, epoch, version, release, arch, summary, description, rpm_size, installed_size, license, vendor, build_group, build_host, source_rpm, url, packager, - requires, provides, files, changelogs + requires, provides, conflicts, obsoletes, files, changelogs FROM rpm_metadata WHERE repo_name = $1 ORDER BY name, epoch, version, release, arch @@ -124,7 +130,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada &r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch, &r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize, &r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager, - &r.Requires, &r.Provides, &r.Files, &r.Changelogs, + &r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs, ); err != nil { return nil, err } diff --git a/internal/provider/provider.go b/internal/provider/provider.go index eb698ae..121d3db 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -63,6 +63,24 @@ type MetadataStore interface { InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error } +// RemoteServer lets a remote provider fully answer a request itself instead of +// going through the byte-proxy engine. It is the remote-side analog of +// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize +// repodata from derived per-asset metadata and to redirect package downloads to +// a backend remote, without ever precaching the packages. Returning false lets +// the normal proxy path take over. +type RemoteServer interface { + ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool +} + +// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache +// and read the metadata it derives per upstream asset. *database.DB satisfies it. +type RemoteMetadataStore interface { + RPMMetadataReader + MetadataStore + MetadataDeleter +} + type MetadataDeleter interface { DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error } @@ -93,6 +111,8 @@ type RPMMetadata struct { Packager string Requires []RPMDep Provides []RPMDep + Conflicts []RPMDep + Obsoletes []RPMDep Files []RPMFile Changelogs []RPMChangelog } diff --git a/internal/provider/rpm/github.go b/internal/provider/rpm/github.go new file mode 100644 index 0000000..7d6fbc6 --- /dev/null +++ b/internal/provider/rpm/github.go @@ -0,0 +1,521 @@ +package rpm + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strings" + "sync" + "time" + + rpmlib "github.com/cavaliergopher/rpm" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(newGitHubProvider()) +} + +// Tuning knobs for the no-precache header fetch. Fields (not consts) so tests +// can shrink them against small fixtures. +const ( + defaultHeaderRangeInitial = 1 << 20 // 1 MiB — covers the header of almost every RPM + defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset + defaultReleasePageCap = 10 // 100 releases/page * 10 pages +) + +// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases +// for .rpm assets, derives per-asset RPM metadata via a ranged header fetch +// (never downloading whole packages), synthesizes yum repodata from that cached +// metadata, and redirects package downloads to a backend "releases_remote" +// (the generic github.com remote) that serves the actual bytes. +type GitHubProvider struct { + client *http.Client + + headerInitial int64 + headerMax int64 + pageCap int + + mu sync.Mutex + scanLock map[string]*sync.Mutex + lastScan map[string]time.Time +} + +func newGitHubProvider() *GitHubProvider { + return &GitHubProvider{ + client: &http.Client{}, + headerInitial: defaultHeaderRangeInitial, + headerMax: defaultHeaderRangeMax, + pageCap: defaultReleasePageCap, + scanLock: map[string]*sync.Mutex{}, + lastScan: map[string]time.Time{}, + } +} + +func (p *GitHubProvider) Type() models.PackageType { return models.PackageGitHubRPM } + +// Classify/ContentType/UpstreamURL/RewriteResponse/AuthHeaders satisfy the +// Provider interface. The proxy engine never reaches them for this type because +// ServeRemote handles every request, but they must exist for registry lookup. +func (p *GitHubProvider) Classify(path string) provider.Mutability { + if strings.HasPrefix(path, "repodata/") { + return provider.Mutable + } + return provider.Immutable +} + +func (p *GitHubProvider) ContentType(path string) string { + switch { + case strings.HasSuffix(path, ".rpm"): + return "application/x-rpm" + case strings.HasSuffix(path, ".xml.gz"): + return "application/gzip" + case strings.HasSuffix(path, ".xml"): + return "application/xml" + } + return "application/octet-stream" +} + +func (p *GitHubProvider) UpstreamURL(remote models.Remote, path string) string { + return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") +} + +func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) { + return nil, nil +} + +func (p *GitHubProvider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { + return githubHeaders(remote, false), nil +} + +// ServeRemote answers a request against a github_rpm remote. It refreshes the +// derived metadata (bounded by mutable_ttl), serves synthesized repodata, and +// 302-redirects .rpm downloads to the backend releases_remote. Returns false +// only for paths it does not own, letting the normal proxy path take over. +func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store provider.RemoteMetadataStore) bool { + p.ensureFresh(r.Context(), remote, store) + + if strings.HasPrefix(path, "repodata/") { + tail := strings.TrimPrefix(path, "repodata/") + lp := &Provider{} + switch { + case tail == "repomd.xml": + lp.serveRepomd(w, r, store, remote.Name) + case strings.HasSuffix(tail, "-primary.xml.gz"): + lp.servePrimary(w, r, store, remote.Name) + case strings.HasSuffix(tail, "-filelists.xml.gz"): + lp.serveFilelists(w, r, store, remote.Name) + case strings.HasSuffix(tail, "-other.xml.gz"): + lp.serveOther(w, r, store, remote.Name) + default: + http.Error(w, "not found", http.StatusNotFound) + } + return true + } + + if strings.HasSuffix(path, ".rpm") { + if remote.ReleasesRemote == "" { + http.Error(w, "github_rpm remote has no releases_remote configured for downloads", http.StatusInternalServerError) + return true + } + loc := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.ReleasesRemote + "/" + strings.TrimLeft(path, "/") + http.Redirect(w, r, loc, http.StatusFound) + return true + } + + return false +} + +func (p *GitHubProvider) ensureFresh(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) { + ttl := time.Duration(remote.MutableTTL) * time.Second + if ttl <= 0 { + ttl = 5 * time.Minute + } + + lock := p.lockFor(remote.Name) + lock.Lock() + defer lock.Unlock() + + p.mu.Lock() + last, ok := p.lastScan[remote.Name] + p.mu.Unlock() + if ok && time.Since(last) < ttl { + return + } + + if err := p.scan(ctx, remote, store); err != nil { + // Keep serving whatever metadata is already cached rather than 500ing. + slog.Error("github_rpm: release scan failed", "remote", remote.Name, "error", err) + return + } + + p.mu.Lock() + p.lastScan[remote.Name] = time.Now() + p.mu.Unlock() +} + +func (p *GitHubProvider) lockFor(name string) *sync.Mutex { + p.mu.Lock() + defer p.mu.Unlock() + l, ok := p.scanLock[name] + if !ok { + l = &sync.Mutex{} + p.scanLock[name] = l + } + return l +} + +func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error { + releases, err := p.fetchReleases(ctx, remote) + if err != nil { + return err + } + + existing, err := store.ListRPMMetadataEntries(ctx, remote.Name) + if err != nil { + return err + } + existingByPath := make(map[string]provider.RPMMetadata, len(existing)) + for _, m := range existing { + existingByPath[m.FilePath] = m + } + + allow, err := compilePatterns(remote.Patterns) + if err != nil { + return err + } + + seen := map[string]bool{} + for _, rel := range releases { + if rel.Draft { + continue + } + for _, asset := range rel.Assets { + if !strings.HasSuffix(strings.ToLower(asset.Name), ".rpm") { + continue + } + if !matchesAny(allow, asset.Name) { + continue + } + fp := assetPath(asset) + if fp == "" { + continue + } + seen[fp] = true + + if cur, ok := existingByPath[fp]; ok { + // Assets are effectively immutable; only re-derive when the + // upstream digest is known and no longer matches what we cached. + if asset.Digest == "" || cur.ContentHash == asset.Digest { + continue + } + _ = store.DeleteRPMMetadata(ctx, remote.Name, fp) + } + + meta, err := p.deriveAsset(ctx, remote, asset, fp) + if err != nil { + slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err) + continue + } + if err := store.InsertRPMMetadata(ctx, meta); err != nil { + slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err) + continue + } + slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch) + } + } + + for fp := range existingByPath { + if !seen[fp] { + _ = store.DeleteRPMMetadata(ctx, remote.Name, fp) + } + } + return nil +} + +type ghRelease struct { + TagName string `json:"tag_name"` + Draft bool `json:"draft"` + Assets []ghAsset `json:"assets"` +} + +type ghAsset struct { + Name string `json:"name"` + Size int64 `json:"size"` + BrowserDownloadURL string `json:"browser_download_url"` + Digest string `json:"digest"` +} + +func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote) ([]ghRelease, error) { + base := strings.TrimRight(remote.BaseURL, "/") + "/releases" + var all []ghRelease + for page := 1; page <= p.pageCap; page++ { + u := fmt.Sprintf("%s?per_page=100&page=%d", base, page) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + copyHeaders(req, githubHeaders(remote, true)) + + resp, err := p.client.Do(req) + if err != nil { + return nil, err + } + body, err := io.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode) + } + var releases []ghRelease + if err := json.Unmarshal(body, &releases); err != nil { + return nil, fmt.Errorf("decode releases: %w", err) + } + if len(releases) == 0 { + break + } + all = append(all, releases...) + if len(releases) < 100 { + break + } + } + return all, nil +} + +func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote, asset ghAsset, fp string) (*provider.RPMMetadata, error) { + pkg, err := p.fetchHeader(ctx, remote, asset.BrowserDownloadURL) + if err != nil { + return nil, err + } + + meta := &provider.RPMMetadata{ + RepoName: remote.Name, + FilePath: fp, + Name: pkg.Name(), + Epoch: pkg.Epoch(), + Version: pkg.Version(), + Release: pkg.Release(), + Arch: pkg.Architecture(), + Summary: pkg.Summary(), + Description: pkg.Description(), + RPMSize: asset.Size, + InstalledSize: int64(pkg.Size()), + License: pkg.License(), + Vendor: pkg.Vendor(), + Group: firstGroup(pkg.Groups()), + BuildHost: pkg.BuildHost(), + SourceRPM: pkg.SourceRPM(), + URL: pkg.URL(), + Packager: pkg.Packager(), + } + + for _, d := range pkg.Requires() { + meta.Requires = append(meta.Requires, rpmDepFromEntry(d)) + } + for _, d := range pkg.Provides() { + meta.Provides = append(meta.Provides, rpmDepFromEntry(d)) + } + for _, d := range pkg.Conflicts() { + meta.Conflicts = append(meta.Conflicts, rpmDepFromEntry(d)) + } + for _, d := range pkg.Obsoletes() { + meta.Obsoletes = append(meta.Obsoletes, rpmDepFromEntry(d)) + } + for _, f := range pkg.Files() { + rf := provider.RPMFile{Path: f.Name()} + if f.IsDir() { + rf.Type = "dir" + } + meta.Files = append(meta.Files, rf) + } + + if meta.Requires == nil { + meta.Requires = []provider.RPMDep{} + } + if meta.Provides == nil { + meta.Provides = []provider.RPMDep{} + } + if meta.Conflicts == nil { + meta.Conflicts = []provider.RPMDep{} + } + if meta.Obsoletes == nil { + meta.Obsoletes = []provider.RPMDep{} + } + if meta.Files == nil { + meta.Files = []provider.RPMFile{} + } + meta.Changelogs = []provider.RPMChangelog{} + + // The primary.xml pkgid checksum must be the sha256 of the whole package. + // Prefer GitHub's asset digest so we never download the body; only when it + // is absent (or not sha256) do we stream the asset once to compute it. + if h, ok := sha256FromDigest(asset.Digest); ok { + meta.ContentHash = "sha256:" + h + } else { + h, err := p.computeSHA256(ctx, remote, asset.BrowserDownloadURL) + if err != nil { + return nil, fmt.Errorf("compute sha256: %w", err) + } + meta.ContentHash = "sha256:" + h + } + + return meta, nil +} + +// fetchHeader pulls only the front of the package with a ranged GET and parses +// the RPM header from it. The header sits before the payload, so a small prefix +// is enough; on a truncated-header parse error it doubles the range and retries. +func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote, downloadURL string) (*rpmlib.Package, error) { + n := p.headerInitial + for { + body, full, err := p.rangeGet(ctx, remote, downloadURL, n) + if err != nil { + return nil, err + } + pkg, perr := rpmlib.Read(bytes.NewReader(body)) + if perr == nil { + return pkg, nil + } + truncated := errors.Is(perr, io.ErrUnexpectedEOF) || errors.Is(perr, io.EOF) + if truncated && !full && n < p.headerMax { + n *= 2 + if n > p.headerMax { + n = p.headerMax + } + continue + } + return nil, fmt.Errorf("parse rpm header: %w", perr) + } +} + +// rangeGet returns the first n bytes of downloadURL. full is true when the +// response body was shorter than n (i.e. we already have the whole object). +func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return nil, false, err + } + copyHeaders(req, githubHeaders(remote, false)) + req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1)) + + resp, err := p.client.Do(req) + if err != nil { + return nil, false, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, n)) + if err != nil { + return nil, false, err + } + full := int64(len(body)) < n + return body, full, nil +} + +func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return "", err + } + copyHeaders(req, githubHeaders(remote, false)) + + resp, err := p.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode) + } + + h := sha256.New() + if _, err := io.Copy(h, resp.Body); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// assetPath is the package's location relative to github.com — the path the +// backend releases_remote (base https://github.com) proxies. It doubles as the +// rpm_metadata key and the in primary.xml. +func assetPath(asset ghAsset) string { + u, err := url.Parse(asset.BrowserDownloadURL) + if err != nil { + return "" + } + return strings.TrimPrefix(u.Path, "/") +} + +func sha256FromDigest(digest string) (string, bool) { + if strings.HasPrefix(digest, "sha256:") { + return strings.TrimPrefix(digest, "sha256:"), true + } + return "", false +} + +func githubHeaders(remote models.Remote, api bool) http.Header { + h := http.Header{} + if api { + h.Set("Accept", "application/vnd.github+json") + h.Set("X-GitHub-Api-Version", "2022-11-28") + } + if tok := githubToken(remote); tok != "" { + h.Set("Authorization", "Bearer "+tok) + } + return h +} + +func githubToken(remote models.Remote) string { + if remote.Password != "" { + return remote.Password + } + return remote.Username +} + +func copyHeaders(req *http.Request, h http.Header) { + for k, vals := range h { + for _, v := range vals { + req.Header.Add(k, v) + } + } +} + +func compilePatterns(patterns []string) ([]*regexp.Regexp, error) { + var out []*regexp.Regexp + for _, p := range patterns { + re, err := regexp.Compile(p) + if err != nil { + return nil, fmt.Errorf("invalid pattern %q: %w", p, err) + } + out = append(out, re) + } + return out, nil +} + +func matchesAny(res []*regexp.Regexp, s string) bool { + if len(res) == 0 { + return true + } + for _, re := range res { + if re.MatchString(s) { + return true + } + } + return false +} diff --git a/internal/provider/rpm/github_test.go b/internal/provider/rpm/github_test.go new file mode 100644 index 0000000..2f6b854 --- /dev/null +++ b/internal/provider/rpm/github_test.go @@ -0,0 +1,324 @@ +package rpm + +import ( + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path, +// mirroring the (repo_name, file_path) uniqueness of the real table. +type fakeStore struct { + mu sync.Mutex + rows map[string]provider.RPMMetadata +} + +func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} } + +func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.rows[m.FilePath]; ok { + return nil // ON CONFLICT DO NOTHING + } + f.rows[m.FilePath] = *m + return nil +} + +func (f *fakeStore) DeleteRPMMetadata(_ context.Context, _, filePath string) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.rows, filePath) + return nil +} + +func (f *fakeStore) ListRPMMetadataEntries(_ context.Context, _ string) ([]provider.RPMMetadata, error) { + f.mu.Lock() + defer f.mu.Unlock() + out := make([]provider.RPMMetadata, 0, len(f.rows)) + for _, m := range f.rows { + out = append(out, m) + } + return out, nil +} + +// githubFixture serves the releases API and the .rpm asset downloads (with +// Range support) for a set of packages. digest controls whether the asset +// carries a sha256 digest (no-download path) or not (compute path). +type githubFixture struct { + srv *httptest.Server + rpmBytes map[string][]byte // asset filename -> bytes + rangeHit map[string]int // asset filename -> number of ranged GETs + fullHit map[string]int // asset filename -> number of full GETs + mu sync.Mutex +} + +func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture { + t.Helper() + f := &githubFixture{ + rpmBytes: map[string][]byte{}, + rangeHit: map[string]int{}, + fullHit: map[string]int{}, + } + f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64") + + mux := http.NewServeMux() + mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) { + page := r.URL.Query().Get("page") + if page != "" && page != "1" { + w.Write([]byte("[]")) + return + } + var assets []map[string]any + for name := range f.rpmBytes { + a := map[string]any{ + "name": name, + "size": len(f.rpmBytes[name]), + "browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2-3/" + name, + } + if withDigest { + sum := sha256.Sum256(f.rpmBytes[name]) + a["digest"] = "sha256:" + hex.EncodeToString(sum[:]) + } + assets = append(assets, a) + } + rel := []map[string]any{{"tag_name": "v1.2-3", "draft": false, "assets": assets}} + json.NewEncoder(w).Encode(rel) + }) + mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) { + name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + body, ok := f.rpmBytes[name] + if !ok { + http.Error(w, "not found", 404) + return + } + rng := r.Header.Get("Range") + f.mu.Lock() + if rng != "" { + f.rangeHit[name]++ + } else { + f.fullHit[name]++ + } + f.mu.Unlock() + + if rng == "" { + w.WriteHeader(200) + w.Write(body) + return + } + // Parse "bytes=0-N". + var end int + fmt.Sscanf(rng, "bytes=0-%d", &end) + if end >= len(body)-1 { + end = len(body) - 1 + } + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body))) + w.Header().Set("Content-Length", strconv.Itoa(end+1)) + w.WriteHeader(http.StatusPartialContent) + w.Write(body[:end+1]) + }) + f.srv = httptest.NewServer(mux) + t.Cleanup(f.srv.Close) + return f +} + +func (f *githubFixture) remote() models.Remote { + return models.Remote{ + Name: "acme-rpm", + PackageType: models.PackageGitHubRPM, + BaseURL: f.srv.URL + "/repos/acme/tools", + ReleasesRemote: "github", + MutableTTL: 3600, + } +} + +func newTestProvider() *GitHubProvider { + p := newGitHubProvider() + p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture + p.headerMax = 1 << 20 + return p +} + +func TestGitHubScanDerivesMetadataFromHeaderAndDigest(t *testing.T) { + fx := newGitHubFixture(t, true) + p := newTestProvider() + store := newFakeStore() + + if err := p.scan(context.Background(), fx.remote(), store); err != nil { + t.Fatalf("scan: %v", err) + } + + metas, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm") + if len(metas) != 1 { + t.Fatalf("want 1 metadata row, got %d", len(metas)) + } + m := metas[0] + if m.Name != "demo" || m.Version != "1.2" || m.Release != "3" || m.Arch != "x86_64" { + t.Fatalf("bad NEVRA: %+v", m) + } + // location href / redirect key must be the github-relative download path. + wantPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm" + if m.FilePath != wantPath { + t.Fatalf("FilePath = %q, want %q", m.FilePath, wantPath) + } + if int(m.RPMSize) != len(fx.rpmBytes["demo-1.2-3.x86_64.rpm"]) { + t.Fatalf("RPMSize = %d, want %d", m.RPMSize, len(fx.rpmBytes["demo-1.2-3.x86_64.rpm"])) + } + // Digest present => checksum from digest, no full download. + sum := sha256.Sum256(fx.rpmBytes["demo-1.2-3.x86_64.rpm"]) + if m.ContentHash != "sha256:"+hex.EncodeToString(sum[:]) { + t.Fatalf("ContentHash = %q, want digest", m.ContentHash) + } + if fx.fullHit["demo-1.2-3.x86_64.rpm"] != 0 { + t.Fatalf("expected no full download when digest present, got %d", fx.fullHit["demo-1.2-3.x86_64.rpm"]) + } + if fx.rangeHit["demo-1.2-3.x86_64.rpm"] == 0 { + t.Fatalf("expected ranged header fetch") + } +} + +func TestGitHubChecksumComputedWhenDigestAbsent(t *testing.T) { + fx := newGitHubFixture(t, false) + p := newTestProvider() + store := newFakeStore() + + if err := p.scan(context.Background(), fx.remote(), store); err != nil { + t.Fatalf("scan: %v", err) + } + metas, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm") + if len(metas) != 1 { + t.Fatalf("want 1 row, got %d", len(metas)) + } + sum := sha256.Sum256(fx.rpmBytes["demo-1.2-3.x86_64.rpm"]) + if metas[0].ContentHash != "sha256:"+hex.EncodeToString(sum[:]) { + t.Fatalf("computed checksum mismatch: %q", metas[0].ContentHash) + } + if fx.fullHit["demo-1.2-3.x86_64.rpm"] == 0 { + t.Fatalf("expected a full download to compute sha256 when digest absent") + } +} + +func TestGitHubServeRemoteRepodataAndRedirect(t *testing.T) { + fx := newGitHubFixture(t, true) + p := newTestProvider() + store := newFakeStore() + remote := fx.remote() + const proxyBase = "https://artifactapi.example" + + // repomd.xml is served and triggers the initial scan. + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/repodata/repomd.xml", nil) + if !p.ServeRemote(rec, req, remote, "repodata/repomd.xml", proxyBase, store) { + t.Fatal("ServeRemote did not handle repomd.xml") + } + if rec.Code != 200 || !strings.Contains(rec.Body.String(), "demo`) { + t.Fatalf("primary missing package: %s", primary) + } + if !strings.Contains(primary, ``) { + t.Fatalf("primary missing/incorrect location href: %s", primary) + } + + // A .rpm request redirects to the backend releases_remote. + rec = httptest.NewRecorder() + pkgPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm" + req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/"+pkgPath, nil) + if !p.ServeRemote(rec, req, remote, pkgPath, proxyBase, store) { + t.Fatal("ServeRemote did not handle .rpm") + } + if rec.Code != http.StatusFound { + t.Fatalf("want 302, got %d", rec.Code) + } + wantLoc := proxyBase + "/api/v1/remote/github/" + pkgPath + if got := rec.Header().Get("Location"); got != wantLoc { + t.Fatalf("Location = %q, want %q", got, wantLoc) + } +} + +func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) { + fx := newGitHubFixture(t, true) + p := newTestProvider() + store := newFakeStore() + remote := fx.remote() + remote.ReleasesRemote = "" + + rec := httptest.NewRecorder() + pkgPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm" + req := httptest.NewRequest(http.MethodGet, "/x", nil) + if !p.ServeRemote(rec, req, remote, pkgPath, "https://x", store) { + t.Fatal("expected handled") + } + if rec.Code != http.StatusInternalServerError { + t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code) + } +} + +func TestGitHubScanPrunesRemovedAssets(t *testing.T) { + fx := newGitHubFixture(t, true) + p := newTestProvider() + store := newFakeStore() + + if err := p.scan(context.Background(), fx.remote(), store); err != nil { + t.Fatalf("scan: %v", err) + } + if rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm"); len(rows) != 1 { + t.Fatalf("want 1 row after first scan, got %d", len(rows)) + } + + // Remove the asset upstream; a rescan must prune the stale metadata row. + delete(fx.rpmBytes, "demo-1.2-3.x86_64.rpm") + if err := p.scan(context.Background(), fx.remote(), store); err != nil { + t.Fatalf("rescan: %v", err) + } + if rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm"); len(rows) != 0 { + t.Fatalf("want 0 rows after prune, got %d", len(rows)) + } +} + +func TestGitHubAssetPatternFilter(t *testing.T) { + fx := newGitHubFixture(t, true) + fx.rpmBytes["other-9-9.aarch64.rpm"] = testsupport.MinimalRPM("other", "9", "9", "aarch64") + p := newTestProvider() + store := newFakeStore() + remote := fx.remote() + remote.Patterns = []string{`^demo-.*\.x86_64\.rpm$`} + + if err := p.scan(context.Background(), remote, store); err != nil { + t.Fatalf("scan: %v", err) + } + rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm") + if len(rows) != 1 || rows[0].Name != "demo" { + t.Fatalf("pattern filter failed, rows=%+v", rows) + } +} diff --git a/internal/provider/rpm/rpm.go b/internal/provider/rpm/rpm.go index e3ce8d6..939bed1 100644 --- a/internal/provider/rpm/rpm.go +++ b/internal/provider/rpm/rpm.go @@ -133,6 +133,12 @@ func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, conte for _, prov := range pkg.Provides() { meta.Provides = append(meta.Provides, rpmDepFromEntry(prov)) } + for _, con := range pkg.Conflicts() { + meta.Conflicts = append(meta.Conflicts, rpmDepFromEntry(con)) + } + for _, obs := range pkg.Obsoletes() { + meta.Obsoletes = append(meta.Obsoletes, rpmDepFromEntry(obs)) + } if meta.Requires == nil { meta.Requires = []provider.RPMDep{} @@ -140,6 +146,12 @@ func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, conte if meta.Provides == nil { meta.Provides = []provider.RPMDep{} } + if meta.Conflicts == nil { + meta.Conflicts = []provider.RPMDep{} + } + if meta.Obsoletes == nil { + meta.Obsoletes = []provider.RPMDep{} + } meta.Files = []provider.RPMFile{} meta.Changelogs = []provider.RPMChangelog{} @@ -363,6 +375,20 @@ func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte { } xmlBuf.WriteString(" \n") } + if len(m.Conflicts) > 0 { + xmlBuf.WriteString(" \n") + for _, d := range m.Conflicts { + writeRPMEntry(&xmlBuf, d) + } + xmlBuf.WriteString(" \n") + } + if len(m.Obsoletes) > 0 { + xmlBuf.WriteString(" \n") + for _, d := range m.Obsoletes { + writeRPMEntry(&xmlBuf, d) + } + xmlBuf.WriteString(" \n") + } fmt.Fprintf(&xmlBuf, " \n") fmt.Fprintf(&xmlBuf, "\n") diff --git a/pkg/models/package_type.go b/pkg/models/package_type.go index ea316a8..e168e96 100644 --- a/pkg/models/package_type.go +++ b/pkg/models/package_type.go @@ -15,6 +15,7 @@ const ( PackagePuppet PackageType = "puppet" PackageTerraform PackageType = "terraform" PackageGoProxy PackageType = "goproxy" + PackageGitHubRPM PackageType = "github_rpm" ) var validPackageTypes = map[PackageType]bool{ @@ -28,6 +29,7 @@ var validPackageTypes = map[PackageType]bool{ PackagePuppet: true, PackageTerraform: true, PackageGoProxy: true, + PackageGitHubRPM: true, } func (p PackageType) Valid() bool { diff --git a/pkg/models/package_type_test.go b/pkg/models/package_type_test.go index f842dca..9a41784 100644 --- a/pkg/models/package_type_test.go +++ b/pkg/models/package_type_test.go @@ -18,6 +18,7 @@ func TestPackageTypeValid(t *testing.T) { models.PackagePuppet, models.PackageTerraform, models.PackageGoProxy, + models.PackageGitHubRPM, } for _, pt := range valid { if !pt.Valid() {