feat: add github_rpm metadata-only remote serving GitHub releases as a yum repo
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/pre-commit Pipeline was successful

Publishing RPMs to GitHub releases is common, but consuming them with dnf
requires repodata that GitHub does not provide, and mirroring every package
into a local repo wastes storage and staleness-tracking on artifacts that
already have a durable home. Expose GitHub releases as a first-class RPM source
that synthesizes repodata on the fly and never precaches the packages.

Add a `github_rpm` remote package type backed by a metadata-only provider:

- Introduce a `RemoteServer` interception hook (the remote-side analog of
  `LocalIndexer`): `handleProxy` lets a provider fully answer a request before
  the byte-proxy engine, passing the request-derived proxy base URL and the DB
  as a `RemoteMetadataStore`.
- Scan a repo's releases via the GitHub API (`base_url` = the releases API
  root) for `.rpm` assets, filtered by the remote's `patterns` (regex on asset
  filename) and reuse the existing local-rpm repodata generators to emit
  `repomd.xml`/`primary`/`filelists`/`other`.
- Derive per-asset metadata without precaching: fetch only the RPM header via a
  ranged GET (retrying with a larger range on a truncated-header parse) for
  NEVRA, requires/provides/conflicts/obsoletes and files; take the sha256 from
  the GitHub asset `digest` when present, else compute it once by streaming.
- Cache derived metadata in `rpm_metadata` keyed by asset path; re-scan no more
  often than `mutable_ttl`, pruning assets that disappear upstream.
- Serve each package's `<location>` as the github-relative download path so the
  client comes back to this remote, which 302-redirects to the `releases_remote`
  (an existing generic github.com remote) that streams the actual bytes.

Reuse the existing `releases_remote` field as the redirect target — it already
carries exactly this "downloads served by remote X" semantic end to end.

Extend the shared RPM metadata model with conflicts/obsoletes (JSONB columns,
added idempotently) so both local and github_rpm repodata resolve upgrades and
conflicts; the local upload path now records them too.

Tests cover header-range parsing with the retry loop, digest-vs-computed
checksum selection, repodata synthesis with the redirect-able location href,
the 302 redirect path, asset pattern filtering, and stale-asset pruning.
This commit is contained in:
2026-08-10 09:35:22 +10:00
parent eee8ee1c31
commit b727b990a2
10 changed files with 964 additions and 5 deletions
+5
View File
@@ -151,6 +151,8 @@ func (db *DB) migrate() error {
packager TEXT DEFAULT '',
requires JSONB DEFAULT '[]',
provides JSONB DEFAULT '[]',
conflicts JSONB DEFAULT '[]',
obsoletes JSONB DEFAULT '[]',
files JSONB DEFAULT '[]',
changelogs JSONB DEFAULT '[]',
created_at TIMESTAMPTZ DEFAULT NOW(),
@@ -159,6 +161,9 @@ func (db *DB) migrate() error {
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
CREATE TABLE IF NOT EXISTS signing_keys (
purpose TEXT PRIMARY KEY,
private_key_armor TEXT NOT NULL,
+11 -5
View File
@@ -10,6 +10,8 @@ import (
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
requiresJSON, _ := json.Marshal(meta.Requires)
providesJSON, _ := json.Marshal(meta.Provides)
conflictsJSON, _ := json.Marshal(meta.Conflicts)
obsoletesJSON, _ := json.Marshal(meta.Obsoletes)
filesJSON, _ := json.Marshal(meta.Files)
changelogsJSON, _ := json.Marshal(meta.Changelogs)
@@ -19,15 +21,15 @@ func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata)
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
requires, provides, conflicts, obsoletes, files, changelogs
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
ON CONFLICT (repo_name, file_path) DO NOTHING
`,
meta.RepoName, meta.FilePath, meta.ContentHash,
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
requiresJSON, providesJSON, filesJSON, changelogsJSON,
requiresJSON, providesJSON, conflictsJSON, obsoletesJSON, filesJSON, changelogsJSON,
)
return err
}
@@ -59,6 +61,8 @@ type RPMMetadataRow struct {
Packager string
Requires json.RawMessage
Provides json.RawMessage
Conflicts json.RawMessage
Obsoletes json.RawMessage
Files json.RawMessage
Changelogs json.RawMessage
}
@@ -93,6 +97,8 @@ func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]pr
}
json.Unmarshal(r.Requires, &meta.Requires)
json.Unmarshal(r.Provides, &meta.Provides)
json.Unmarshal(r.Conflicts, &meta.Conflicts)
json.Unmarshal(r.Obsoletes, &meta.Obsoletes)
json.Unmarshal(r.Files, &meta.Files)
json.Unmarshal(r.Changelogs, &meta.Changelogs)
result[i] = meta
@@ -106,7 +112,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
requires, provides, conflicts, obsoletes, files, changelogs
FROM rpm_metadata
WHERE repo_name = $1
ORDER BY name, epoch, version, release, arch
@@ -124,7 +130,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
&r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs,
); err != nil {
return nil, err
}