feat: github_rpm metadata-only remote (GitHub releases as a yum repo, no precache) #107

Merged
benvin merged 2 commits from benvin/github-rpm-remote into master 2026-08-10 20:54:44 +10:00
10 changed files with 1069 additions and 15 deletions
+45
View File
@@ -32,9 +32,54 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
| `terraform` | `*/versions` | `*/download/*/*` |
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
Providers classify paths automatically. Users only configure what to proxy and TTLs.
### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache)
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
repository without ever caching the packages. It scans releases for `.rpm`
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
metadata comes from a **ranged GET of just the RPM header** (the header sits at
the front of the file, so the whole package is never downloaded); the sha256
checksum comes from the GitHub asset `digest` when present, else a one-time
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
is cheap on repeat, and refreshed no more often than `mutable_ttl`.
Each package's `<location>` points back at the remote, which **302-redirects**
the download to the `releases_remote` — an existing generic `github.com` remote
that streams the actual bytes. `dnf` follows the redirect transparently.
```hcl
# Backend that serves the actual .rpm bytes from github.com.
resource "artifactapi_remote_generic" "github" {
name = "github"
base_url = "https://github.com"
patterns = [
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
]
}
resource "artifactapi_remote_github_rpm" "acme-tools" {
name = "acme-tools"
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
releases_remote = "github" # backend for downloads
mutable_ttl = 3600 # release re-scan interval
# Optional: restrict which release assets become packages (regex on filename).
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
# Optional: a token for private repos / higher API rate limits.
# password = "ghp_..."
}
```
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
from the synthesized metadata.
## Terraform
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
+9
View File
@@ -83,6 +83,15 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
return
}
// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
// redirect package downloads to a backend remote instead of proxying bytes.
if rs, ok := prov.(provider.RemoteServer); ok {
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
return
}
}
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
if err != nil {
var proxyErr *proxy.ProxyError
+5
View File
@@ -151,6 +151,8 @@ func (db *DB) migrate() error {
packager TEXT DEFAULT '',
requires JSONB DEFAULT '[]',
provides JSONB DEFAULT '[]',
conflicts JSONB DEFAULT '[]',
obsoletes JSONB DEFAULT '[]',
files JSONB DEFAULT '[]',
changelogs JSONB DEFAULT '[]',
created_at TIMESTAMPTZ DEFAULT NOW(),
@@ -159,6 +161,9 @@ func (db *DB) migrate() error {
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
CREATE TABLE IF NOT EXISTS signing_keys (
purpose TEXT PRIMARY KEY,
private_key_armor TEXT NOT NULL,
+11 -5
View File
@@ -10,6 +10,8 @@ import (
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
requiresJSON, _ := json.Marshal(meta.Requires)
providesJSON, _ := json.Marshal(meta.Provides)
conflictsJSON, _ := json.Marshal(meta.Conflicts)
obsoletesJSON, _ := json.Marshal(meta.Obsoletes)
filesJSON, _ := json.Marshal(meta.Files)
changelogsJSON, _ := json.Marshal(meta.Changelogs)
@@ -19,15 +21,15 @@ func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata)
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
requires, provides, conflicts, obsoletes, files, changelogs
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
ON CONFLICT (repo_name, file_path) DO NOTHING
`,
meta.RepoName, meta.FilePath, meta.ContentHash,
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
requiresJSON, providesJSON, filesJSON, changelogsJSON,
requiresJSON, providesJSON, conflictsJSON, obsoletesJSON, filesJSON, changelogsJSON,
)
return err
}
@@ -59,6 +61,8 @@ type RPMMetadataRow struct {
Packager string
Requires json.RawMessage
Provides json.RawMessage
Conflicts json.RawMessage
Obsoletes json.RawMessage
Files json.RawMessage
Changelogs json.RawMessage
}
@@ -93,6 +97,8 @@ func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]pr
}
json.Unmarshal(r.Requires, &meta.Requires)
json.Unmarshal(r.Provides, &meta.Provides)
json.Unmarshal(r.Conflicts, &meta.Conflicts)
json.Unmarshal(r.Obsoletes, &meta.Obsoletes)
json.Unmarshal(r.Files, &meta.Files)
json.Unmarshal(r.Changelogs, &meta.Changelogs)
result[i] = meta
@@ -106,7 +112,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
requires, provides, conflicts, obsoletes, files, changelogs
FROM rpm_metadata
WHERE repo_name = $1
ORDER BY name, epoch, version, release, arch
@@ -124,7 +130,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
&r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs,
); err != nil {
return nil, err
}
+20
View File
@@ -63,6 +63,24 @@ type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
}
// RemoteServer lets a remote provider fully answer a request itself instead of
// going through the byte-proxy engine. It is the remote-side analog of
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
// repodata from derived per-asset metadata and to redirect package downloads to
// a backend remote, without ever precaching the packages. Returning false lets
// the normal proxy path take over.
type RemoteServer interface {
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
}
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
type RemoteMetadataStore interface {
RPMMetadataReader
MetadataStore
MetadataDeleter
}
type MetadataDeleter interface {
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
}
@@ -93,6 +111,8 @@ type RPMMetadata struct {
Packager string
Requires []RPMDep
Provides []RPMDep
Conflicts []RPMDep
Obsoletes []RPMDep
Files []RPMFile
Changelogs []RPMChangelog
}
+557
View File
@@ -0,0 +1,557 @@
package rpm
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"net/url"
"regexp"
"strings"
"sync"
"time"
rpmlib "github.com/cavaliergopher/rpm"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(newGitHubProvider())
}
// Tuning knobs for the no-precache header fetch. Fields (not consts) so tests
// can shrink them against small fixtures.
const (
defaultHeaderRangeInitial = 1 << 20 // 1 MiB — covers the header of almost every RPM
defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset
defaultReleasePageCap = 10 // 100 releases/page * 10 pages
// defaultScanTimeout bounds a detached background scan (which may do one
// ranged fetch per asset across every release) so it can never run forever.
defaultScanTimeout = 10 * time.Minute
// defaultServeTimeout bounds a repodata DB read served on a detached context.
defaultServeTimeout = 30 * time.Second
)
// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases
// for .rpm assets, derives per-asset RPM metadata via a ranged header fetch
// (never downloading whole packages), synthesizes yum repodata from that cached
// metadata, and redirects package downloads to a backend "releases_remote"
// (the generic github.com remote) that serves the actual bytes.
type GitHubProvider struct {
client *http.Client
headerInitial int64
headerMax int64
pageCap int
scanTimeout time.Duration
serveTimeout time.Duration
mu sync.Mutex
scanning map[string]bool
lastScan map[string]time.Time
}
func newGitHubProvider() *GitHubProvider {
return &GitHubProvider{
client: &http.Client{},
headerInitial: defaultHeaderRangeInitial,
headerMax: defaultHeaderRangeMax,
pageCap: defaultReleasePageCap,
scanTimeout: defaultScanTimeout,
serveTimeout: defaultServeTimeout,
scanning: map[string]bool{},
lastScan: map[string]time.Time{},
}
}
func (p *GitHubProvider) Type() models.PackageType { return models.PackageGitHubRPM }
// Classify/ContentType/UpstreamURL/RewriteResponse/AuthHeaders satisfy the
// Provider interface. The proxy engine never reaches them for this type because
// ServeRemote handles every request, but they must exist for registry lookup.
func (p *GitHubProvider) Classify(path string) provider.Mutability {
if strings.HasPrefix(path, "repodata/") {
return provider.Mutable
}
return provider.Immutable
}
func (p *GitHubProvider) ContentType(path string) string {
switch {
case strings.HasSuffix(path, ".rpm"):
return "application/x-rpm"
case strings.HasSuffix(path, ".xml.gz"):
return "application/gzip"
case strings.HasSuffix(path, ".xml"):
return "application/xml"
}
return "application/octet-stream"
}
func (p *GitHubProvider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *GitHubProvider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return githubHeaders(remote, false), nil
}
// ServeRemote answers a request against a github_rpm remote. It refreshes the
// derived metadata (bounded by mutable_ttl), serves synthesized repodata, and
// 302-redirects .rpm downloads to the backend releases_remote. Returns false
// only for paths it does not own, letting the normal proxy path take over.
func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store provider.RemoteMetadataStore) bool {
p.refresh(remote, store)
if strings.HasPrefix(path, "repodata/") {
// Serve repodata on a context detached from the inbound request: a
// client disconnect (e.g. dnf makecache timing out) must never cancel
// the metadata DB read and surface as a 500.
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
defer cancel()
sr := r.WithContext(sctx)
tail := strings.TrimPrefix(path, "repodata/")
lp := &Provider{}
switch {
case tail == "repomd.xml":
lp.serveRepomd(w, sr, store, remote.Name)
case strings.HasSuffix(tail, "-primary.xml.gz"):
lp.servePrimary(w, sr, store, remote.Name)
case strings.HasSuffix(tail, "-filelists.xml.gz"):
lp.serveFilelists(w, sr, store, remote.Name)
case strings.HasSuffix(tail, "-other.xml.gz"):
lp.serveOther(w, sr, store, remote.Name)
default:
http.Error(w, "not found", http.StatusNotFound)
}
return true
}
if strings.HasSuffix(path, ".rpm") {
if remote.ReleasesRemote == "" {
http.Error(w, "github_rpm remote has no releases_remote configured for downloads", http.StatusInternalServerError)
return true
}
loc := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.ReleasesRemote + "/" + strings.TrimLeft(path, "/")
http.Redirect(w, r, loc, http.StatusFound)
return true
}
return false
}
// refresh brings the derived metadata up to date without coupling the scan to
// the inbound request. When the cache is stale it single-flights a scan: if the
// cache already holds rows the scan runs in the background and the caller serves
// the current cache immediately; only a completely empty cache blocks on a
// bounded first scan (so the first client sees packages rather than an empty or
// 500 repodata).
func (p *GitHubProvider) refresh(remote models.Remote, store provider.RemoteMetadataStore) {
ttl := time.Duration(remote.MutableTTL) * time.Second
if ttl <= 0 {
ttl = 5 * time.Minute
}
p.mu.Lock()
last, ok := p.lastScan[remote.Name]
fresh := ok && time.Since(last) < ttl
if fresh || p.scanning[remote.Name] {
p.mu.Unlock()
return
}
p.scanning[remote.Name] = true
p.mu.Unlock()
empty := true
if rows, err := store.ListRPMMetadataEntries(context.Background(), remote.Name); err == nil {
empty = len(rows) == 0
}
if empty {
p.runScan(remote, store)
return
}
go p.runScan(remote, store)
}
// runScan derives metadata on a detached, bounded context so a client cancel
// can neither abort the shared derive nor poison the metadata read. The caller
// must have already claimed the single-flight slot (scanning[name] = true).
func (p *GitHubProvider) runScan(remote models.Remote, store provider.RemoteMetadataStore) {
defer func() {
p.mu.Lock()
delete(p.scanning, remote.Name)
p.mu.Unlock()
}()
ctx, cancel := context.WithTimeout(context.Background(), p.scanTimeout)
defer cancel()
if err := p.scan(ctx, remote, store); err != nil {
// Keep serving whatever metadata is already cached rather than 500ing.
slog.Error("github_rpm: release scan failed", "remote", remote.Name, "error", err)
return
}
p.mu.Lock()
p.lastScan[remote.Name] = time.Now()
p.mu.Unlock()
}
func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error {
releases, err := p.fetchReleases(ctx, remote)
if err != nil {
return err
}
existing, err := store.ListRPMMetadataEntries(ctx, remote.Name)
if err != nil {
return err
}
existingByPath := make(map[string]provider.RPMMetadata, len(existing))
for _, m := range existing {
existingByPath[m.FilePath] = m
}
allow, err := compilePatterns(remote.Patterns)
if err != nil {
return err
}
seen := map[string]bool{}
for _, rel := range releases {
if rel.Draft {
continue
}
for _, asset := range rel.Assets {
if !strings.HasSuffix(strings.ToLower(asset.Name), ".rpm") {
continue
}
if !matchesAny(allow, asset.Name) {
continue
}
fp := assetPath(asset)
if fp == "" {
continue
}
seen[fp] = true
if cur, ok := existingByPath[fp]; ok {
// Assets are effectively immutable; only re-derive when the
// upstream digest is known and no longer matches what we cached.
if asset.Digest == "" || cur.ContentHash == asset.Digest {
continue
}
_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
}
meta, err := p.deriveAsset(ctx, remote, asset, fp)
if err != nil {
slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
continue
}
if err := store.InsertRPMMetadata(ctx, meta); err != nil {
slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
continue
}
slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
}
}
for fp := range existingByPath {
if !seen[fp] {
_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
}
}
return nil
}
type ghRelease struct {
TagName string `json:"tag_name"`
Draft bool `json:"draft"`
Assets []ghAsset `json:"assets"`
}
type ghAsset struct {
Name string `json:"name"`
Size int64 `json:"size"`
BrowserDownloadURL string `json:"browser_download_url"`
Digest string `json:"digest"`
}
func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote) ([]ghRelease, error) {
base := strings.TrimRight(remote.BaseURL, "/") + "/releases"
var all []ghRelease
for page := 1; page <= p.pageCap; page++ {
u := fmt.Sprintf("%s?per_page=100&page=%d", base, page)
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
copyHeaders(req, githubHeaders(remote, true))
resp, err := p.client.Do(req)
if err != nil {
return nil, err
}
body, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
}
var releases []ghRelease
if err := json.Unmarshal(body, &releases); err != nil {
return nil, fmt.Errorf("decode releases: %w", err)
}
if len(releases) == 0 {
break
}
all = append(all, releases...)
if len(releases) < 100 {
break
}
}
return all, nil
}
func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote, asset ghAsset, fp string) (*provider.RPMMetadata, error) {
pkg, err := p.fetchHeader(ctx, remote, asset.BrowserDownloadURL)
if err != nil {
return nil, err
}
meta := &provider.RPMMetadata{
RepoName: remote.Name,
FilePath: fp,
Name: pkg.Name(),
Epoch: pkg.Epoch(),
Version: pkg.Version(),
Release: pkg.Release(),
Arch: pkg.Architecture(),
Summary: pkg.Summary(),
Description: pkg.Description(),
RPMSize: asset.Size,
InstalledSize: int64(pkg.Size()),
License: pkg.License(),
Vendor: pkg.Vendor(),
Group: firstGroup(pkg.Groups()),
BuildHost: pkg.BuildHost(),
SourceRPM: pkg.SourceRPM(),
URL: pkg.URL(),
Packager: pkg.Packager(),
}
for _, d := range pkg.Requires() {
meta.Requires = append(meta.Requires, rpmDepFromEntry(d))
}
for _, d := range pkg.Provides() {
meta.Provides = append(meta.Provides, rpmDepFromEntry(d))
}
for _, d := range pkg.Conflicts() {
meta.Conflicts = append(meta.Conflicts, rpmDepFromEntry(d))
}
for _, d := range pkg.Obsoletes() {
meta.Obsoletes = append(meta.Obsoletes, rpmDepFromEntry(d))
}
for _, f := range pkg.Files() {
rf := provider.RPMFile{Path: f.Name()}
if f.IsDir() {
rf.Type = "dir"
}
meta.Files = append(meta.Files, rf)
}
if meta.Requires == nil {
meta.Requires = []provider.RPMDep{}
}
if meta.Provides == nil {
meta.Provides = []provider.RPMDep{}
}
if meta.Conflicts == nil {
meta.Conflicts = []provider.RPMDep{}
}
if meta.Obsoletes == nil {
meta.Obsoletes = []provider.RPMDep{}
}
if meta.Files == nil {
meta.Files = []provider.RPMFile{}
}
meta.Changelogs = []provider.RPMChangelog{}
// The primary.xml pkgid checksum must be the sha256 of the whole package.
// Prefer GitHub's asset digest so we never download the body; only when it
// is absent (or not sha256) do we stream the asset once to compute it.
if h, ok := sha256FromDigest(asset.Digest); ok {
meta.ContentHash = "sha256:" + h
} else {
h, err := p.computeSHA256(ctx, remote, asset.BrowserDownloadURL)
if err != nil {
return nil, fmt.Errorf("compute sha256: %w", err)
}
meta.ContentHash = "sha256:" + h
}
return meta, nil
}
// fetchHeader pulls only the front of the package with a ranged GET and parses
// the RPM header from it. The header sits before the payload, so a small prefix
// is enough; on a truncated-header parse error it doubles the range and retries.
func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote, downloadURL string) (*rpmlib.Package, error) {
n := p.headerInitial
for {
body, full, err := p.rangeGet(ctx, remote, downloadURL, n)
if err != nil {
return nil, err
}
pkg, perr := rpmlib.Read(bytes.NewReader(body))
if perr == nil {
return pkg, nil
}
truncated := errors.Is(perr, io.ErrUnexpectedEOF) || errors.Is(perr, io.EOF)
if truncated && !full && n < p.headerMax {
n *= 2
if n > p.headerMax {
n = p.headerMax
}
continue
}
return nil, fmt.Errorf("parse rpm header: %w", perr)
}
}
// rangeGet returns the first n bytes of downloadURL. full is true when the
// response body was shorter than n (i.e. we already have the whole object).
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
if err != nil {
return nil, false, err
}
copyHeaders(req, githubHeaders(remote, false))
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1))
resp, err := p.client.Do(req)
if err != nil {
return nil, false, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
if err != nil {
return nil, false, err
}
full := int64(len(body)) < n
return body, full, nil
}
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
if err != nil {
return "", err
}
copyHeaders(req, githubHeaders(remote, false))
resp, err := p.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
}
h := sha256.New()
if _, err := io.Copy(h, resp.Body); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
// assetPath is the package's location relative to github.com — the path the
// backend releases_remote (base https://github.com) proxies. It doubles as the
// rpm_metadata key and the <location href> in primary.xml.
func assetPath(asset ghAsset) string {
u, err := url.Parse(asset.BrowserDownloadURL)
if err != nil {
return ""
}
return strings.TrimPrefix(u.Path, "/")
}
func sha256FromDigest(digest string) (string, bool) {
if strings.HasPrefix(digest, "sha256:") {
return strings.TrimPrefix(digest, "sha256:"), true
}
return "", false
}
func githubHeaders(remote models.Remote, api bool) http.Header {
h := http.Header{}
if api {
h.Set("Accept", "application/vnd.github+json")
h.Set("X-GitHub-Api-Version", "2022-11-28")
}
if tok := githubToken(remote); tok != "" {
h.Set("Authorization", "Bearer "+tok)
}
return h
}
func githubToken(remote models.Remote) string {
if remote.Password != "" {
return remote.Password
}
return remote.Username
}
func copyHeaders(req *http.Request, h http.Header) {
for k, vals := range h {
for _, v := range vals {
req.Header.Add(k, v)
}
}
}
func compilePatterns(patterns []string) ([]*regexp.Regexp, error) {
var out []*regexp.Regexp
for _, p := range patterns {
re, err := regexp.Compile(p)
if err != nil {
return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
}
out = append(out, re)
}
return out, nil
}
func matchesAny(res []*regexp.Regexp, s string) bool {
if len(res) == 0 {
return true
}
for _, re := range res {
if re.MatchString(s) {
return true
}
}
return false
}
+367
View File
@@ -0,0 +1,367 @@
package rpm
import (
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strconv"
"strings"
"sync"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
// mirroring the (repo_name, file_path) uniqueness of the real table.
type fakeStore struct {
mu sync.Mutex
rows map[string]provider.RPMMetadata
}
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
f.mu.Lock()
defer f.mu.Unlock()
if _, ok := f.rows[m.FilePath]; ok {
return nil // ON CONFLICT DO NOTHING
}
f.rows[m.FilePath] = *m
return nil
}
func (f *fakeStore) DeleteRPMMetadata(_ context.Context, _, filePath string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.rows, filePath)
return nil
}
func (f *fakeStore) ListRPMMetadataEntries(ctx context.Context, _ string) ([]provider.RPMMetadata, error) {
// Mirror pgx: a canceled/expired context fails the read. This is what
// poisons the repodata response if the read runs on the inbound request.
if err := ctx.Err(); err != nil {
return nil, err
}
f.mu.Lock()
defer f.mu.Unlock()
out := make([]provider.RPMMetadata, 0, len(f.rows))
for _, m := range f.rows {
out = append(out, m)
}
return out, nil
}
// githubFixture serves the releases API and the .rpm asset downloads (with
// Range support) for a set of packages. digest controls whether the asset
// carries a sha256 digest (no-download path) or not (compute path).
type githubFixture struct {
srv *httptest.Server
rpmBytes map[string][]byte // asset filename -> bytes
rangeHit map[string]int // asset filename -> number of ranged GETs
fullHit map[string]int // asset filename -> number of full GETs
mu sync.Mutex
}
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
t.Helper()
f := &githubFixture{
rpmBytes: map[string][]byte{},
rangeHit: map[string]int{},
fullHit: map[string]int{},
}
f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
mux := http.NewServeMux()
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
page := r.URL.Query().Get("page")
if page != "" && page != "1" {
w.Write([]byte("[]"))
return
}
var assets []map[string]any
for name := range f.rpmBytes {
a := map[string]any{
"name": name,
"size": len(f.rpmBytes[name]),
"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2-3/" + name,
}
if withDigest {
sum := sha256.Sum256(f.rpmBytes[name])
a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
}
assets = append(assets, a)
}
rel := []map[string]any{{"tag_name": "v1.2-3", "draft": false, "assets": assets}}
json.NewEncoder(w).Encode(rel)
})
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
body, ok := f.rpmBytes[name]
if !ok {
http.Error(w, "not found", 404)
return
}
rng := r.Header.Get("Range")
f.mu.Lock()
if rng != "" {
f.rangeHit[name]++
} else {
f.fullHit[name]++
}
f.mu.Unlock()
if rng == "" {
w.WriteHeader(200)
w.Write(body)
return
}
// Parse "bytes=0-N".
var end int
fmt.Sscanf(rng, "bytes=0-%d", &end)
if end >= len(body)-1 {
end = len(body) - 1
}
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
w.Header().Set("Content-Length", strconv.Itoa(end+1))
w.WriteHeader(http.StatusPartialContent)
w.Write(body[:end+1])
})
f.srv = httptest.NewServer(mux)
t.Cleanup(f.srv.Close)
return f
}
func (f *githubFixture) remote() models.Remote {
return models.Remote{
Name: "acme-rpm",
PackageType: models.PackageGitHubRPM,
BaseURL: f.srv.URL + "/repos/acme/tools",
ReleasesRemote: "github",
MutableTTL: 3600,
}
}
func newTestProvider() *GitHubProvider {
p := newGitHubProvider()
p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
p.headerMax = 1 << 20
return p
}
func TestGitHubScanDerivesMetadataFromHeaderAndDigest(t *testing.T) {
fx := newGitHubFixture(t, true)
p := newTestProvider()
store := newFakeStore()
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
t.Fatalf("scan: %v", err)
}
metas, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm")
if len(metas) != 1 {
t.Fatalf("want 1 metadata row, got %d", len(metas))
}
m := metas[0]
if m.Name != "demo" || m.Version != "1.2" || m.Release != "3" || m.Arch != "x86_64" {
t.Fatalf("bad NEVRA: %+v", m)
}
// location href / redirect key must be the github-relative download path.
wantPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"
if m.FilePath != wantPath {
t.Fatalf("FilePath = %q, want %q", m.FilePath, wantPath)
}
if int(m.RPMSize) != len(fx.rpmBytes["demo-1.2-3.x86_64.rpm"]) {
t.Fatalf("RPMSize = %d, want %d", m.RPMSize, len(fx.rpmBytes["demo-1.2-3.x86_64.rpm"]))
}
// Digest present => checksum from digest, no full download.
sum := sha256.Sum256(fx.rpmBytes["demo-1.2-3.x86_64.rpm"])
if m.ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
t.Fatalf("ContentHash = %q, want digest", m.ContentHash)
}
if fx.fullHit["demo-1.2-3.x86_64.rpm"] != 0 {
t.Fatalf("expected no full download when digest present, got %d", fx.fullHit["demo-1.2-3.x86_64.rpm"])
}
if fx.rangeHit["demo-1.2-3.x86_64.rpm"] == 0 {
t.Fatalf("expected ranged header fetch")
}
}
func TestGitHubChecksumComputedWhenDigestAbsent(t *testing.T) {
fx := newGitHubFixture(t, false)
p := newTestProvider()
store := newFakeStore()
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
t.Fatalf("scan: %v", err)
}
metas, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm")
if len(metas) != 1 {
t.Fatalf("want 1 row, got %d", len(metas))
}
sum := sha256.Sum256(fx.rpmBytes["demo-1.2-3.x86_64.rpm"])
if metas[0].ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
t.Fatalf("computed checksum mismatch: %q", metas[0].ContentHash)
}
if fx.fullHit["demo-1.2-3.x86_64.rpm"] == 0 {
t.Fatalf("expected a full download to compute sha256 when digest absent")
}
}
func TestGitHubServeRemoteRepodataAndRedirect(t *testing.T) {
fx := newGitHubFixture(t, true)
p := newTestProvider()
store := newFakeStore()
remote := fx.remote()
const proxyBase = "https://artifactapi.example"
// repomd.xml is served and triggers the initial scan.
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/repodata/repomd.xml", nil)
if !p.ServeRemote(rec, req, remote, "repodata/repomd.xml", proxyBase, store) {
t.Fatal("ServeRemote did not handle repomd.xml")
}
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "<repomd") {
t.Fatalf("repomd bad: code=%d body=%s", rec.Code, rec.Body.String())
}
// primary.xml.gz must carry the package with a location href that is the
// github-relative download path (so it resolves back to this remote and
// redirects to the backend).
rec = httptest.NewRecorder()
req = httptest.NewRequest(http.MethodGet, "/x", nil)
if !p.ServeRemote(rec, req, remote, "repodata/abc-primary.xml.gz", proxyBase, store) {
t.Fatal("ServeRemote did not handle primary")
}
gz, err := gzip.NewReader(rec.Body)
if err != nil {
t.Fatalf("gzip: %v", err)
}
xmlBytes, _ := io.ReadAll(gz)
primary := string(xmlBytes)
if !strings.Contains(primary, `<name>demo</name>`) {
t.Fatalf("primary missing package: %s", primary)
}
if !strings.Contains(primary, `<location href="acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"/>`) {
t.Fatalf("primary missing/incorrect location href: %s", primary)
}
// A .rpm request redirects to the backend releases_remote.
rec = httptest.NewRecorder()
pkgPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"
req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/"+pkgPath, nil)
if !p.ServeRemote(rec, req, remote, pkgPath, proxyBase, store) {
t.Fatal("ServeRemote did not handle .rpm")
}
if rec.Code != http.StatusFound {
t.Fatalf("want 302, got %d", rec.Code)
}
wantLoc := proxyBase + "/api/v1/remote/github/" + pkgPath
if got := rec.Header().Get("Location"); got != wantLoc {
t.Fatalf("Location = %q, want %q", got, wantLoc)
}
}
// TestGitHubServeRemoteCanceledRequestServesCache reproduces the cold-makecache
// 500: when the inbound request context is already canceled (dnf timed out and
// disconnected), the repodata read must not be run on that context and turned
// into a 500. With the cache already warm, the handler serves it as 200.
// Before the fix the read used r.Context() and returned 500; after the fix it
// runs on a detached context and serves the cached repomd.
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
fx := newGitHubFixture(t, true)
p := newTestProvider()
store := newFakeStore()
remote := fx.remote()
// Warm the cache and mark the scan fresh so ServeRemote does not re-derive.
if err := p.scan(context.Background(), remote, store); err != nil {
t.Fatalf("warm scan: %v", err)
}
p.mu.Lock()
p.lastScan[remote.Name] = time.Now()
p.mu.Unlock()
// Inbound request whose context is already canceled (client went away).
ctx, cancel := context.WithCancel(context.Background())
cancel()
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/repodata/repomd.xml", nil).WithContext(ctx)
if !p.ServeRemote(rec, req, remote, "repodata/repomd.xml", "https://x", store) {
t.Fatal("ServeRemote did not handle repomd.xml")
}
if rec.Code != http.StatusOK {
t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
}
if !strings.Contains(rec.Body.String(), "<repomd") {
t.Fatalf("expected repomd served from cache, got %s", rec.Body.String())
}
}
func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
fx := newGitHubFixture(t, true)
p := newTestProvider()
store := newFakeStore()
remote := fx.remote()
remote.ReleasesRemote = ""
rec := httptest.NewRecorder()
pkgPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"
req := httptest.NewRequest(http.MethodGet, "/x", nil)
if !p.ServeRemote(rec, req, remote, pkgPath, "https://x", store) {
t.Fatal("expected handled")
}
if rec.Code != http.StatusInternalServerError {
t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
}
}
func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
fx := newGitHubFixture(t, true)
p := newTestProvider()
store := newFakeStore()
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
t.Fatalf("scan: %v", err)
}
if rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm"); len(rows) != 1 {
t.Fatalf("want 1 row after first scan, got %d", len(rows))
}
// Remove the asset upstream; a rescan must prune the stale metadata row.
delete(fx.rpmBytes, "demo-1.2-3.x86_64.rpm")
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
t.Fatalf("rescan: %v", err)
}
if rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm"); len(rows) != 0 {
t.Fatalf("want 0 rows after prune, got %d", len(rows))
}
}
func TestGitHubAssetPatternFilter(t *testing.T) {
fx := newGitHubFixture(t, true)
fx.rpmBytes["other-9-9.aarch64.rpm"] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
p := newTestProvider()
store := newFakeStore()
remote := fx.remote()
remote.Patterns = []string{`^demo-.*\.x86_64\.rpm$`}
if err := p.scan(context.Background(), remote, store); err != nil {
t.Fatalf("scan: %v", err)
}
rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm")
if len(rows) != 1 || rows[0].Name != "demo" {
t.Fatalf("pattern filter failed, rows=%+v", rows)
}
}
+52 -10
View File
@@ -7,6 +7,7 @@ import (
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"errors"
"fmt"
"log/slog"
"net/http"
@@ -133,6 +134,12 @@ func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, conte
for _, prov := range pkg.Provides() {
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
}
for _, con := range pkg.Conflicts() {
meta.Conflicts = append(meta.Conflicts, rpmDepFromEntry(con))
}
for _, obs := range pkg.Obsoletes() {
meta.Obsoletes = append(meta.Obsoletes, rpmDepFromEntry(obs))
}
if meta.Requires == nil {
meta.Requires = []provider.RPMDep{}
@@ -140,6 +147,12 @@ func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, conte
if meta.Provides == nil {
meta.Provides = []provider.RPMDep{}
}
if meta.Conflicts == nil {
meta.Conflicts = []provider.RPMDep{}
}
if meta.Obsoletes == nil {
meta.Obsoletes = []provider.RPMDep{}
}
meta.Files = []provider.RPMFile{}
meta.Changelogs = []provider.RPMChangelog{}
@@ -229,10 +242,28 @@ func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileSt
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
}
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
// readMetadataEntries loads the repo's derived metadata, translating the read
// error into an HTTP response. A canceled/deadline-exceeded context (typically a
// client that went away) becomes a retryable 503 rather than a hard 500, so a
// dnf disconnect never looks like a server fault. ok is false when a response
// has already been written.
func readMetadataEntries(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) ([]provider.RPMMetadata, bool) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
slog.Warn("rpm: metadata read canceled", "repo", repoName, "error", err)
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
return nil, false
}
http.Error(w, err.Error(), http.StatusInternalServerError)
return nil, false
}
return metas, true
}
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, ok := readMetadataEntries(w, r, reader, repoName)
if !ok {
return
}
@@ -252,9 +283,8 @@ func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader pr
}
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
metas, ok := readMetadataEntries(w, r, reader, repoName)
if !ok {
return
}
@@ -264,9 +294,8 @@ func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader p
}
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
metas, ok := readMetadataEntries(w, r, reader, repoName)
if !ok {
return
}
@@ -276,9 +305,8 @@ func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader
}
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
metas, ok := readMetadataEntries(w, r, reader, repoName)
if !ok {
return
}
@@ -363,6 +391,20 @@ func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
}
xmlBuf.WriteString(" </rpm:requires>\n")
}
if len(m.Conflicts) > 0 {
xmlBuf.WriteString(" <rpm:conflicts>\n")
for _, d := range m.Conflicts {
writeRPMEntry(&xmlBuf, d)
}
xmlBuf.WriteString(" </rpm:conflicts>\n")
}
if len(m.Obsoletes) > 0 {
xmlBuf.WriteString(" <rpm:obsoletes>\n")
for _, d := range m.Obsoletes {
writeRPMEntry(&xmlBuf, d)
}
xmlBuf.WriteString(" </rpm:obsoletes>\n")
}
fmt.Fprintf(&xmlBuf, " </format>\n")
fmt.Fprintf(&xmlBuf, "</package>\n")
+2
View File
@@ -15,6 +15,7 @@ const (
PackagePuppet PackageType = "puppet"
PackageTerraform PackageType = "terraform"
PackageGoProxy PackageType = "goproxy"
PackageGitHubRPM PackageType = "github_rpm"
)
var validPackageTypes = map[PackageType]bool{
@@ -28,6 +29,7 @@ var validPackageTypes = map[PackageType]bool{
PackagePuppet: true,
PackageTerraform: true,
PackageGoProxy: true,
PackageGitHubRPM: true,
}
func (p PackageType) Valid() bool {
+1
View File
@@ -18,6 +18,7 @@ func TestPackageTypeValid(t *testing.T) {
models.PackagePuppet,
models.PackageTerraform,
models.PackageGoProxy,
models.PackageGitHubRPM,
}
for _, pt := range valid {
if !pt.Valid() {