Files
artifactapi/internal/database/rpm_metadata.go
T
unkinben d154fbf3f3 feat: github_rpm metadata-only remote (GitHub releases as a yum repo, no precache) (#107)
## Why

Publishing RPMs to GitHub releases is common, but consuming them with `dnf` requires repodata GitHub does not provide, and mirroring every package into a local repo wastes storage and staleness-tracking on artifacts that already have a durable home. This exposes GitHub releases as a first-class RPM source that synthesizes repodata on the fly and **never precaches the packages**.

## What

Add a `github_rpm` remote package type backed by a metadata-only provider.

- Introduce a `RemoteServer` interception hook (the remote-side analog of `LocalIndexer`): `handleProxy` lets a provider fully answer a request before the byte-proxy engine, passing the request-derived proxy base URL and the DB as a `RemoteMetadataStore`.
- Scan a repo's releases via the GitHub API (`base_url` = the releases API root) for `.rpm` assets, filtered by the remote's `patterns` (regex on asset filename), and reuse the existing local-rpm repodata generators to emit `repomd.xml`/`primary`/`filelists`/`other`.
- Derive per-asset metadata without precaching: fetch only the RPM header via a ranged GET (retrying with a larger range on a truncated-header parse) for NEVRA, requires/provides/conflicts/obsoletes and files; take the sha256 from the GitHub asset `digest` when present, else compute it once by streaming.
- Cache derived metadata in `rpm_metadata` keyed by asset path; re-scan no more often than `mutable_ttl`, pruning assets that disappear upstream.
- Serve each package's `<location>` as the github-relative download path so the client comes back to this remote, which **302-redirects** to the `releases_remote` (an existing generic github.com remote) that streams the actual bytes.

Reuse the existing `releases_remote` field as the redirect target — it already carries exactly this "downloads served by remote X" semantic end to end, so no new schema/model field is needed.

Extend the shared RPM metadata model with conflicts/obsoletes (JSONB columns, added idempotently) so both local and `github_rpm` repodata resolve upgrades and conflicts; the local upload path records them too.

## No-precache mechanics

- **Dependency metadata**: always from the ranged header fetch (header precedes payload; `rpm.Read` stops at the payload boundary), giving `dnf` full resolution. Default range 1 MiB, doubling to 16 MiB.
- **Checksum**: prefer the GitHub asset `digest` (no download); fall back to a one-time streamed sha256 only when absent. Header-only "minimal mode" (no deps) is rejected as a default because `dnf` needs accurate provides/requires and a correct pkgid checksum to install.

## Tests

Header-range parsing incl. the retry loop, digest-vs-computed checksum selection, repodata synthesis with the redirect-able `<location href>`, the 302 redirect path (and the guard when `releases_remote` is unset), asset pattern filtering, and stale-asset pruning. `go build`/`vet`/`test` green; pre-commit clean.

## Follow-ups

- `github_apk` / `github_deb` metadata-only remotes (same pattern; not in this PR).
- Terraform provider support for `artifactapi_remote_github_rpm` ships as a separate PR against `terraform-provider-artifactapi` (depends on this API surface).

Reviewed-on: #107
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
2026-08-10 20:54:44 +10:00

141 lines
4.4 KiB
Go

package database
import (
"context"
"encoding/json"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
requiresJSON, _ := json.Marshal(meta.Requires)
providesJSON, _ := json.Marshal(meta.Provides)
conflictsJSON, _ := json.Marshal(meta.Conflicts)
obsoletesJSON, _ := json.Marshal(meta.Obsoletes)
filesJSON, _ := json.Marshal(meta.Files)
changelogsJSON, _ := json.Marshal(meta.Changelogs)
_, err := db.Pool.Exec(ctx, `
INSERT INTO rpm_metadata (
repo_name, file_path, content_hash,
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, conflicts, obsoletes, files, changelogs
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
ON CONFLICT (repo_name, file_path) DO NOTHING
`,
meta.RepoName, meta.FilePath, meta.ContentHash,
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
requiresJSON, providesJSON, conflictsJSON, obsoletesJSON, filesJSON, changelogsJSON,
)
return err
}
func (db *DB) DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM rpm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
type RPMMetadataRow struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires json.RawMessage
Provides json.RawMessage
Conflicts json.RawMessage
Obsoletes json.RawMessage
Files json.RawMessage
Changelogs json.RawMessage
}
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
rows, err := db.ListRPMMetadata(ctx, repoName)
if err != nil {
return nil, err
}
result := make([]provider.RPMMetadata, len(rows))
for i, r := range rows {
meta := provider.RPMMetadata{
RepoName: r.RepoName,
FilePath: r.FilePath,
ContentHash: r.ContentHash,
Name: r.Name,
Epoch: r.Epoch,
Version: r.Version,
Release: r.Release,
Arch: r.Arch,
Summary: r.Summary,
Description: r.Description,
RPMSize: r.RPMSize,
InstalledSize: r.InstalledSize,
License: r.License,
Vendor: r.Vendor,
Group: r.Group,
BuildHost: r.BuildHost,
SourceRPM: r.SourceRPM,
URL: r.URL,
Packager: r.Packager,
}
json.Unmarshal(r.Requires, &meta.Requires)
json.Unmarshal(r.Provides, &meta.Provides)
json.Unmarshal(r.Conflicts, &meta.Conflicts)
json.Unmarshal(r.Obsoletes, &meta.Obsoletes)
json.Unmarshal(r.Files, &meta.Files)
json.Unmarshal(r.Changelogs, &meta.Changelogs)
result[i] = meta
}
return result, nil
}
func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetadataRow, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash,
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, conflicts, obsoletes, files, changelogs
FROM rpm_metadata
WHERE repo_name = $1
ORDER BY name, epoch, version, release, arch
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []RPMMetadataRow
for rows.Next() {
var r RPMMetadataRow
if err := rows.Scan(
&r.RepoName, &r.FilePath, &r.ContentHash,
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
&r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs,
); err != nil {
return nil, err
}
result = append(result, r)
}
return result, rows.Err()
}