feat: github_rpm metadata-only remote (GitHub releases as a yum repo, no precache) #107
@@ -32,9 +32,54 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
|
||||
| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` |
|
||||
| `terraform` | `*/versions` | `*/download/*/*` |
|
||||
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
|
||||
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
|
||||
|
||||
Providers classify paths automatically. Users only configure what to proxy and TTLs.
|
||||
|
||||
### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache)
|
||||
|
||||
A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum`
|
||||
repository without ever caching the packages. It scans releases for `.rpm`
|
||||
assets, derives each package's metadata (NEVRA, requires/provides/conflicts/
|
||||
obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package
|
||||
metadata comes from a **ranged GET of just the RPM header** (the header sits at
|
||||
the front of the file, so the whole package is never downloaded); the sha256
|
||||
checksum comes from the GitHub asset `digest` when present, else a one-time
|
||||
lazy stream. Derived metadata is cached (keyed by asset) so repodata generation
|
||||
is cheap on repeat, and refreshed no more often than `mutable_ttl`.
|
||||
|
||||
Each package's `<location>` points back at the remote, which **302-redirects**
|
||||
the download to the `releases_remote` — an existing generic `github.com` remote
|
||||
that streams the actual bytes. `dnf` follows the redirect transparently.
|
||||
|
||||
```hcl
|
||||
# Backend that serves the actual .rpm bytes from github.com.
|
||||
resource "artifactapi_remote_generic" "github" {
|
||||
name = "github"
|
||||
base_url = "https://github.com"
|
||||
patterns = [
|
||||
"acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets
|
||||
]
|
||||
}
|
||||
|
||||
resource "artifactapi_remote_github_rpm" "acme-tools" {
|
||||
name = "acme-tools"
|
||||
base_url = "https://api.github.com/repos/acme/tools" # the releases API root
|
||||
releases_remote = "github" # backend for downloads
|
||||
mutable_ttl = 3600 # release re-scan interval
|
||||
|
||||
# Optional: restrict which release assets become packages (regex on filename).
|
||||
patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"]
|
||||
|
||||
# Optional: a token for private repos / higher API rate limits.
|
||||
# password = "ghp_..."
|
||||
}
|
||||
```
|
||||
|
||||
`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`.
|
||||
The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages
|
||||
from the synthesized metadata.
|
||||
|
||||
## Terraform
|
||||
|
||||
Remotes and virtuals are managed by Terraform. Each package type has its own resource:
|
||||
|
||||
@@ -83,6 +83,15 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
// Metadata-only remotes (e.g. github_rpm) synthesize their own responses and
|
||||
// redirect package downloads to a backend remote instead of proxying bytes.
|
||||
if rs, ok := prov.(provider.RemoteServer); ok {
|
||||
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
|
||||
if rs.ServeRemote(w, r, *remote, path, proxyBaseURL, h.db) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
|
||||
if err != nil {
|
||||
var proxyErr *proxy.ProxyError
|
||||
|
||||
@@ -151,6 +151,8 @@ func (db *DB) migrate() error {
|
||||
packager TEXT DEFAULT '',
|
||||
requires JSONB DEFAULT '[]',
|
||||
provides JSONB DEFAULT '[]',
|
||||
conflicts JSONB DEFAULT '[]',
|
||||
obsoletes JSONB DEFAULT '[]',
|
||||
files JSONB DEFAULT '[]',
|
||||
changelogs JSONB DEFAULT '[]',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
@@ -159,6 +161,9 @@ func (db *DB) migrate() error {
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
||||
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signing_keys (
|
||||
purpose TEXT PRIMARY KEY,
|
||||
private_key_armor TEXT NOT NULL,
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
|
||||
requiresJSON, _ := json.Marshal(meta.Requires)
|
||||
providesJSON, _ := json.Marshal(meta.Provides)
|
||||
conflictsJSON, _ := json.Marshal(meta.Conflicts)
|
||||
obsoletesJSON, _ := json.Marshal(meta.Obsoletes)
|
||||
filesJSON, _ := json.Marshal(meta.Files)
|
||||
changelogsJSON, _ := json.Marshal(meta.Changelogs)
|
||||
|
||||
@@ -19,15 +21,15 @@ func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata)
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
|
||||
requires, provides, conflicts, obsoletes, files, changelogs
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24,$25)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
|
||||
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
|
||||
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
|
||||
requiresJSON, providesJSON, filesJSON, changelogsJSON,
|
||||
requiresJSON, providesJSON, conflictsJSON, obsoletesJSON, filesJSON, changelogsJSON,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -59,6 +61,8 @@ type RPMMetadataRow struct {
|
||||
Packager string
|
||||
Requires json.RawMessage
|
||||
Provides json.RawMessage
|
||||
Conflicts json.RawMessage
|
||||
Obsoletes json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
}
|
||||
@@ -93,6 +97,8 @@ func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]pr
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
json.Unmarshal(r.Conflicts, &meta.Conflicts)
|
||||
json.Unmarshal(r.Obsoletes, &meta.Obsoletes)
|
||||
json.Unmarshal(r.Files, &meta.Files)
|
||||
json.Unmarshal(r.Changelogs, &meta.Changelogs)
|
||||
result[i] = meta
|
||||
@@ -106,7 +112,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, files, changelogs
|
||||
requires, provides, conflicts, obsoletes, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
@@ -124,7 +130,7 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
|
||||
&r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -63,6 +63,24 @@ type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
}
|
||||
|
||||
// RemoteServer lets a remote provider fully answer a request itself instead of
|
||||
// going through the byte-proxy engine. It is the remote-side analog of
|
||||
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
|
||||
// repodata from derived per-asset metadata and to redirect package downloads to
|
||||
// a backend remote, without ever precaching the packages. Returning false lets
|
||||
// the normal proxy path take over.
|
||||
type RemoteServer interface {
|
||||
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
|
||||
}
|
||||
|
||||
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
|
||||
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
|
||||
type RemoteMetadataStore interface {
|
||||
RPMMetadataReader
|
||||
MetadataStore
|
||||
MetadataDeleter
|
||||
}
|
||||
|
||||
type MetadataDeleter interface {
|
||||
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
@@ -93,6 +111,8 @@ type RPMMetadata struct {
|
||||
Packager string
|
||||
Requires []RPMDep
|
||||
Provides []RPMDep
|
||||
Conflicts []RPMDep
|
||||
Obsoletes []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
}
|
||||
|
||||
@@ -0,0 +1,557 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
rpmlib "github.com/cavaliergopher/rpm"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(newGitHubProvider())
|
||||
}
|
||||
|
||||
// Tuning knobs for the no-precache header fetch. Fields (not consts) so tests
|
||||
// can shrink them against small fixtures.
|
||||
const (
|
||||
defaultHeaderRangeInitial = 1 << 20 // 1 MiB — covers the header of almost every RPM
|
||||
defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset
|
||||
defaultReleasePageCap = 10 // 100 releases/page * 10 pages
|
||||
|
||||
// defaultScanTimeout bounds a detached background scan (which may do one
|
||||
// ranged fetch per asset across every release) so it can never run forever.
|
||||
defaultScanTimeout = 10 * time.Minute
|
||||
// defaultServeTimeout bounds a repodata DB read served on a detached context.
|
||||
defaultServeTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases
|
||||
// for .rpm assets, derives per-asset RPM metadata via a ranged header fetch
|
||||
// (never downloading whole packages), synthesizes yum repodata from that cached
|
||||
// metadata, and redirects package downloads to a backend "releases_remote"
|
||||
// (the generic github.com remote) that serves the actual bytes.
|
||||
type GitHubProvider struct {
|
||||
client *http.Client
|
||||
|
||||
headerInitial int64
|
||||
headerMax int64
|
||||
pageCap int
|
||||
scanTimeout time.Duration
|
||||
serveTimeout time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool
|
||||
lastScan map[string]time.Time
|
||||
}
|
||||
|
||||
func newGitHubProvider() *GitHubProvider {
|
||||
return &GitHubProvider{
|
||||
client: &http.Client{},
|
||||
headerInitial: defaultHeaderRangeInitial,
|
||||
headerMax: defaultHeaderRangeMax,
|
||||
pageCap: defaultReleasePageCap,
|
||||
scanTimeout: defaultScanTimeout,
|
||||
serveTimeout: defaultServeTimeout,
|
||||
scanning: map[string]bool{},
|
||||
lastScan: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) Type() models.PackageType { return models.PackageGitHubRPM }
|
||||
|
||||
// Classify/ContentType/UpstreamURL/RewriteResponse/AuthHeaders satisfy the
|
||||
// Provider interface. The proxy engine never reaches them for this type because
|
||||
// ServeRemote handles every request, but they must exist for registry lookup.
|
||||
func (p *GitHubProvider) Classify(path string) provider.Mutability {
|
||||
if strings.HasPrefix(path, "repodata/") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".rpm"):
|
||||
return "application/x-rpm"
|
||||
case strings.HasSuffix(path, ".xml.gz"):
|
||||
return "application/gzip"
|
||||
case strings.HasSuffix(path, ".xml"):
|
||||
return "application/xml"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return githubHeaders(remote, false), nil
|
||||
}
|
||||
|
||||
// ServeRemote answers a request against a github_rpm remote. It refreshes the
|
||||
// derived metadata (bounded by mutable_ttl), serves synthesized repodata, and
|
||||
// 302-redirects .rpm downloads to the backend releases_remote. Returns false
|
||||
// only for paths it does not own, letting the normal proxy path take over.
|
||||
func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store provider.RemoteMetadataStore) bool {
|
||||
p.refresh(remote, store)
|
||||
|
||||
if strings.HasPrefix(path, "repodata/") {
|
||||
// Serve repodata on a context detached from the inbound request: a
|
||||
// client disconnect (e.g. dnf makecache timing out) must never cancel
|
||||
// the metadata DB read and surface as a 500.
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
sr := r.WithContext(sctx)
|
||||
|
||||
tail := strings.TrimPrefix(path, "repodata/")
|
||||
lp := &Provider{}
|
||||
switch {
|
||||
case tail == "repomd.xml":
|
||||
lp.serveRepomd(w, sr, store, remote.Name)
|
||||
case strings.HasSuffix(tail, "-primary.xml.gz"):
|
||||
lp.servePrimary(w, sr, store, remote.Name)
|
||||
case strings.HasSuffix(tail, "-filelists.xml.gz"):
|
||||
lp.serveFilelists(w, sr, store, remote.Name)
|
||||
case strings.HasSuffix(tail, "-other.xml.gz"):
|
||||
lp.serveOther(w, sr, store, remote.Name)
|
||||
default:
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".rpm") {
|
||||
if remote.ReleasesRemote == "" {
|
||||
http.Error(w, "github_rpm remote has no releases_remote configured for downloads", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
loc := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.ReleasesRemote + "/" + strings.TrimLeft(path, "/")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// refresh brings the derived metadata up to date without coupling the scan to
|
||||
// the inbound request. When the cache is stale it single-flights a scan: if the
|
||||
// cache already holds rows the scan runs in the background and the caller serves
|
||||
// the current cache immediately; only a completely empty cache blocks on a
|
||||
// bounded first scan (so the first client sees packages rather than an empty or
|
||||
// 500 repodata).
|
||||
func (p *GitHubProvider) refresh(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
ttl := time.Duration(remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 5 * time.Minute
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
last, ok := p.lastScan[remote.Name]
|
||||
fresh := ok && time.Since(last) < ttl
|
||||
if fresh || p.scanning[remote.Name] {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.scanning[remote.Name] = true
|
||||
p.mu.Unlock()
|
||||
|
||||
empty := true
|
||||
if rows, err := store.ListRPMMetadataEntries(context.Background(), remote.Name); err == nil {
|
||||
empty = len(rows) == 0
|
||||
}
|
||||
|
||||
if empty {
|
||||
p.runScan(remote, store)
|
||||
return
|
||||
}
|
||||
go p.runScan(remote, store)
|
||||
}
|
||||
|
||||
// runScan derives metadata on a detached, bounded context so a client cancel
|
||||
// can neither abort the shared derive nor poison the metadata read. The caller
|
||||
// must have already claimed the single-flight slot (scanning[name] = true).
|
||||
func (p *GitHubProvider) runScan(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.scanning, remote.Name)
|
||||
p.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), p.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := p.scan(ctx, remote, store); err != nil {
|
||||
// Keep serving whatever metadata is already cached rather than 500ing.
|
||||
slog.Error("github_rpm: release scan failed", "remote", remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error {
|
||||
releases, err := p.fetchReleases(ctx, remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
existing, err := store.ListRPMMetadataEntries(ctx, remote.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
existingByPath := make(map[string]provider.RPMMetadata, len(existing))
|
||||
for _, m := range existing {
|
||||
existingByPath[m.FilePath] = m
|
||||
}
|
||||
|
||||
allow, err := compilePatterns(remote.Patterns)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
continue
|
||||
}
|
||||
for _, asset := range rel.Assets {
|
||||
if !strings.HasSuffix(strings.ToLower(asset.Name), ".rpm") {
|
||||
continue
|
||||
}
|
||||
if !matchesAny(allow, asset.Name) {
|
||||
continue
|
||||
}
|
||||
fp := assetPath(asset)
|
||||
if fp == "" {
|
||||
continue
|
||||
}
|
||||
seen[fp] = true
|
||||
|
||||
if cur, ok := existingByPath[fp]; ok {
|
||||
// Assets are effectively immutable; only re-derive when the
|
||||
// upstream digest is known and no longer matches what we cached.
|
||||
if asset.Digest == "" || cur.ContentHash == asset.Digest {
|
||||
continue
|
||||
}
|
||||
_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := store.InsertRPMMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
}
|
||||
|
||||
for fp := range existingByPath {
|
||||
if !seen[fp] {
|
||||
_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Draft bool `json:"draft"`
|
||||
Assets []ghAsset `json:"assets"`
|
||||
}
|
||||
|
||||
type ghAsset struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
BrowserDownloadURL string `json:"browser_download_url"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote) ([]ghRelease, error) {
|
||||
base := strings.TrimRight(remote.BaseURL, "/") + "/releases"
|
||||
var all []ghRelease
|
||||
for page := 1; page <= p.pageCap; page++ {
|
||||
u := fmt.Sprintf("%s?per_page=100&page=%d", base, page)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
copyHeaders(req, githubHeaders(remote, true))
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
|
||||
}
|
||||
var releases []ghRelease
|
||||
if err := json.Unmarshal(body, &releases); err != nil {
|
||||
return nil, fmt.Errorf("decode releases: %w", err)
|
||||
}
|
||||
if len(releases) == 0 {
|
||||
break
|
||||
}
|
||||
all = append(all, releases...)
|
||||
if len(releases) < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote, asset ghAsset, fp string) (*provider.RPMMetadata, error) {
|
||||
pkg, err := p.fetchHeader(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
meta := &provider.RPMMetadata{
|
||||
RepoName: remote.Name,
|
||||
FilePath: fp,
|
||||
Name: pkg.Name(),
|
||||
Epoch: pkg.Epoch(),
|
||||
Version: pkg.Version(),
|
||||
Release: pkg.Release(),
|
||||
Arch: pkg.Architecture(),
|
||||
Summary: pkg.Summary(),
|
||||
Description: pkg.Description(),
|
||||
RPMSize: asset.Size,
|
||||
InstalledSize: int64(pkg.Size()),
|
||||
License: pkg.License(),
|
||||
Vendor: pkg.Vendor(),
|
||||
Group: firstGroup(pkg.Groups()),
|
||||
BuildHost: pkg.BuildHost(),
|
||||
SourceRPM: pkg.SourceRPM(),
|
||||
URL: pkg.URL(),
|
||||
Packager: pkg.Packager(),
|
||||
}
|
||||
|
||||
for _, d := range pkg.Requires() {
|
||||
meta.Requires = append(meta.Requires, rpmDepFromEntry(d))
|
||||
}
|
||||
for _, d := range pkg.Provides() {
|
||||
meta.Provides = append(meta.Provides, rpmDepFromEntry(d))
|
||||
}
|
||||
for _, d := range pkg.Conflicts() {
|
||||
meta.Conflicts = append(meta.Conflicts, rpmDepFromEntry(d))
|
||||
}
|
||||
for _, d := range pkg.Obsoletes() {
|
||||
meta.Obsoletes = append(meta.Obsoletes, rpmDepFromEntry(d))
|
||||
}
|
||||
for _, f := range pkg.Files() {
|
||||
rf := provider.RPMFile{Path: f.Name()}
|
||||
if f.IsDir() {
|
||||
rf.Type = "dir"
|
||||
}
|
||||
meta.Files = append(meta.Files, rf)
|
||||
}
|
||||
|
||||
if meta.Requires == nil {
|
||||
meta.Requires = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Provides == nil {
|
||||
meta.Provides = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Conflicts == nil {
|
||||
meta.Conflicts = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Obsoletes == nil {
|
||||
meta.Obsoletes = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Files == nil {
|
||||
meta.Files = []provider.RPMFile{}
|
||||
}
|
||||
meta.Changelogs = []provider.RPMChangelog{}
|
||||
|
||||
// The primary.xml pkgid checksum must be the sha256 of the whole package.
|
||||
// Prefer GitHub's asset digest so we never download the body; only when it
|
||||
// is absent (or not sha256) do we stream the asset once to compute it.
|
||||
if h, ok := sha256FromDigest(asset.Digest); ok {
|
||||
meta.ContentHash = "sha256:" + h
|
||||
} else {
|
||||
h, err := p.computeSHA256(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compute sha256: %w", err)
|
||||
}
|
||||
meta.ContentHash = "sha256:" + h
|
||||
}
|
||||
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// fetchHeader pulls only the front of the package with a ranged GET and parses
|
||||
// the RPM header from it. The header sits before the payload, so a small prefix
|
||||
// is enough; on a truncated-header parse error it doubles the range and retries.
|
||||
func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote, downloadURL string) (*rpmlib.Package, error) {
|
||||
n := p.headerInitial
|
||||
for {
|
||||
body, full, err := p.rangeGet(ctx, remote, downloadURL, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pkg, perr := rpmlib.Read(bytes.NewReader(body))
|
||||
if perr == nil {
|
||||
return pkg, nil
|
||||
}
|
||||
truncated := errors.Is(perr, io.ErrUnexpectedEOF) || errors.Is(perr, io.EOF)
|
||||
if truncated && !full && n < p.headerMax {
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
n = p.headerMax
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("parse rpm header: %w", perr)
|
||||
}
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
copyHeaders(req, githubHeaders(remote, false))
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1))
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
copyHeaders(req, githubHeaders(remote, false))
|
||||
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, resp.Body); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// assetPath is the package's location relative to github.com — the path the
|
||||
// backend releases_remote (base https://github.com) proxies. It doubles as the
|
||||
// rpm_metadata key and the <location href> in primary.xml.
|
||||
func assetPath(asset ghAsset) string {
|
||||
u, err := url.Parse(asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(u.Path, "/")
|
||||
}
|
||||
|
||||
func sha256FromDigest(digest string) (string, bool) {
|
||||
if strings.HasPrefix(digest, "sha256:") {
|
||||
return strings.TrimPrefix(digest, "sha256:"), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func githubHeaders(remote models.Remote, api bool) http.Header {
|
||||
h := http.Header{}
|
||||
if api {
|
||||
h.Set("Accept", "application/vnd.github+json")
|
||||
h.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
}
|
||||
if tok := githubToken(remote); tok != "" {
|
||||
h.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
func githubToken(remote models.Remote) string {
|
||||
if remote.Password != "" {
|
||||
return remote.Password
|
||||
}
|
||||
return remote.Username
|
||||
}
|
||||
|
||||
func copyHeaders(req *http.Request, h http.Header) {
|
||||
for k, vals := range h {
|
||||
for _, v := range vals {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func compilePatterns(patterns []string) ([]*regexp.Regexp, error) {
|
||||
var out []*regexp.Regexp
|
||||
for _, p := range patterns {
|
||||
re, err := regexp.Compile(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
|
||||
}
|
||||
out = append(out, re)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func matchesAny(res []*regexp.Regexp, s string) bool {
|
||||
if len(res) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, re := range res {
|
||||
if re.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
|
||||
// mirroring the (repo_name, file_path) uniqueness of the real table.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]provider.RPMMetadata
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
|
||||
|
||||
func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.rows[m.FilePath]; ok {
|
||||
return nil // ON CONFLICT DO NOTHING
|
||||
}
|
||||
f.rows[m.FilePath] = *m
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) DeleteRPMMetadata(_ context.Context, _, filePath string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.rows, filePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListRPMMetadataEntries(ctx context.Context, _ string) ([]provider.RPMMetadata, error) {
|
||||
// Mirror pgx: a canceled/expired context fails the read. This is what
|
||||
// poisons the repodata response if the read runs on the inbound request.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]provider.RPMMetadata, 0, len(f.rows))
|
||||
for _, m := range f.rows {
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// githubFixture serves the releases API and the .rpm asset downloads (with
|
||||
// Range support) for a set of packages. digest controls whether the asset
|
||||
// carries a sha256 digest (no-download path) or not (compute path).
|
||||
type githubFixture struct {
|
||||
srv *httptest.Server
|
||||
rpmBytes map[string][]byte // asset filename -> bytes
|
||||
rangeHit map[string]int // asset filename -> number of ranged GETs
|
||||
fullHit map[string]int // asset filename -> number of full GETs
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
rpmBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
}
|
||||
f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
|
||||
page := r.URL.Query().Get("page")
|
||||
if page != "" && page != "1" {
|
||||
w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
var assets []map[string]any
|
||||
for name := range f.rpmBytes {
|
||||
a := map[string]any{
|
||||
"name": name,
|
||||
"size": len(f.rpmBytes[name]),
|
||||
"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2-3/" + name,
|
||||
}
|
||||
if withDigest {
|
||||
sum := sha256.Sum256(f.rpmBytes[name])
|
||||
a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
assets = append(assets, a)
|
||||
}
|
||||
rel := []map[string]any{{"tag_name": "v1.2-3", "draft": false, "assets": assets}}
|
||||
json.NewEncoder(w).Encode(rel)
|
||||
})
|
||||
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
body, ok := f.rpmBytes[name]
|
||||
if !ok {
|
||||
http.Error(w, "not found", 404)
|
||||
return
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
} else {
|
||||
f.fullHit[name]++
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
if rng == "" {
|
||||
w.WriteHeader(200)
|
||||
w.Write(body)
|
||||
return
|
||||
}
|
||||
// Parse "bytes=0-N".
|
||||
var end int
|
||||
fmt.Sscanf(rng, "bytes=0-%d", &end)
|
||||
if end >= len(body)-1 {
|
||||
end = len(body) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(end+1))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(body[:end+1])
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *githubFixture) remote() models.Remote {
|
||||
return models.Remote{
|
||||
Name: "acme-rpm",
|
||||
PackageType: models.PackageGitHubRPM,
|
||||
BaseURL: f.srv.URL + "/repos/acme/tools",
|
||||
ReleasesRemote: "github",
|
||||
MutableTTL: 3600,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestProvider() *GitHubProvider {
|
||||
p := newGitHubProvider()
|
||||
p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
|
||||
p.headerMax = 1 << 20
|
||||
return p
|
||||
}
|
||||
|
||||
func TestGitHubScanDerivesMetadataFromHeaderAndDigest(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
metas, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 metadata row, got %d", len(metas))
|
||||
}
|
||||
m := metas[0]
|
||||
if m.Name != "demo" || m.Version != "1.2" || m.Release != "3" || m.Arch != "x86_64" {
|
||||
t.Fatalf("bad NEVRA: %+v", m)
|
||||
}
|
||||
// location href / redirect key must be the github-relative download path.
|
||||
wantPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"
|
||||
if m.FilePath != wantPath {
|
||||
t.Fatalf("FilePath = %q, want %q", m.FilePath, wantPath)
|
||||
}
|
||||
if int(m.RPMSize) != len(fx.rpmBytes["demo-1.2-3.x86_64.rpm"]) {
|
||||
t.Fatalf("RPMSize = %d, want %d", m.RPMSize, len(fx.rpmBytes["demo-1.2-3.x86_64.rpm"]))
|
||||
}
|
||||
// Digest present => checksum from digest, no full download.
|
||||
sum := sha256.Sum256(fx.rpmBytes["demo-1.2-3.x86_64.rpm"])
|
||||
if m.ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("ContentHash = %q, want digest", m.ContentHash)
|
||||
}
|
||||
if fx.fullHit["demo-1.2-3.x86_64.rpm"] != 0 {
|
||||
t.Fatalf("expected no full download when digest present, got %d", fx.fullHit["demo-1.2-3.x86_64.rpm"])
|
||||
}
|
||||
if fx.rangeHit["demo-1.2-3.x86_64.rpm"] == 0 {
|
||||
t.Fatalf("expected ranged header fetch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubChecksumComputedWhenDigestAbsent(t *testing.T) {
|
||||
fx := newGitHubFixture(t, false)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
metas, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 row, got %d", len(metas))
|
||||
}
|
||||
sum := sha256.Sum256(fx.rpmBytes["demo-1.2-3.x86_64.rpm"])
|
||||
if metas[0].ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("computed checksum mismatch: %q", metas[0].ContentHash)
|
||||
}
|
||||
if fx.fullHit["demo-1.2-3.x86_64.rpm"] == 0 {
|
||||
t.Fatalf("expected a full download to compute sha256 when digest absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRepodataAndRedirect(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
// repomd.xml is served and triggers the initial scan.
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/repodata/repomd.xml", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "repodata/repomd.xml", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle repomd.xml")
|
||||
}
|
||||
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "<repomd") {
|
||||
t.Fatalf("repomd bad: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// primary.xml.gz must carry the package with a location href that is the
|
||||
// github-relative download path (so it resolves back to this remote and
|
||||
// redirects to the backend).
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "repodata/abc-primary.xml.gz", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle primary")
|
||||
}
|
||||
gz, err := gzip.NewReader(rec.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
xmlBytes, _ := io.ReadAll(gz)
|
||||
primary := string(xmlBytes)
|
||||
if !strings.Contains(primary, `<name>demo</name>`) {
|
||||
t.Fatalf("primary missing package: %s", primary)
|
||||
}
|
||||
if !strings.Contains(primary, `<location href="acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"/>`) {
|
||||
t.Fatalf("primary missing/incorrect location href: %s", primary)
|
||||
}
|
||||
|
||||
// A .rpm request redirects to the backend releases_remote.
|
||||
rec = httptest.NewRecorder()
|
||||
pkgPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/"+pkgPath, nil)
|
||||
if !p.ServeRemote(rec, req, remote, pkgPath, proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle .rpm")
|
||||
}
|
||||
if rec.Code != http.StatusFound {
|
||||
t.Fatalf("want 302, got %d", rec.Code)
|
||||
}
|
||||
wantLoc := proxyBase + "/api/v1/remote/github/" + pkgPath
|
||||
if got := rec.Header().Get("Location"); got != wantLoc {
|
||||
t.Fatalf("Location = %q, want %q", got, wantLoc)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitHubServeRemoteCanceledRequestServesCache reproduces the cold-makecache
|
||||
// 500: when the inbound request context is already canceled (dnf timed out and
|
||||
// disconnected), the repodata read must not be run on that context and turned
|
||||
// into a 500. With the cache already warm, the handler serves it as 200.
|
||||
// Before the fix the read used r.Context() and returned 500; after the fix it
|
||||
// runs on a detached context and serves the cached repomd.
|
||||
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
// Warm the cache and mark the scan fresh so ServeRemote does not re-derive.
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
|
||||
// Inbound request whose context is already canceled (client went away).
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/repodata/repomd.xml", nil).WithContext(ctx)
|
||||
|
||||
if !p.ServeRemote(rec, req, remote, "repodata/repomd.xml", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle repomd.xml")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "<repomd") {
|
||||
t.Fatalf("expected repomd served from cache, got %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.ReleasesRemote = ""
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
pkgPath := "acme/tools/releases/download/v1.2-3/demo-1.2-3.x86_64.rpm"
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, pkgPath, "https://x", store) {
|
||||
t.Fatal("expected handled")
|
||||
}
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm"); len(rows) != 1 {
|
||||
t.Fatalf("want 1 row after first scan, got %d", len(rows))
|
||||
}
|
||||
|
||||
// Remove the asset upstream; a rescan must prune the stale metadata row.
|
||||
delete(fx.rpmBytes, "demo-1.2-3.x86_64.rpm")
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("rescan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm"); len(rows) != 0 {
|
||||
t.Fatalf("want 0 rows after prune, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubAssetPatternFilter(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.rpmBytes["other-9-9.aarch64.rpm"] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.Patterns = []string{`^demo-.*\.x86_64\.rpm$`}
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
rows, _ := store.ListRPMMetadataEntries(context.Background(), "acme-rpm")
|
||||
if len(rows) != 1 || rows[0].Name != "demo" {
|
||||
t.Fatalf("pattern filter failed, rows=%+v", rows)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
@@ -133,6 +134,12 @@ func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, conte
|
||||
for _, prov := range pkg.Provides() {
|
||||
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
|
||||
}
|
||||
for _, con := range pkg.Conflicts() {
|
||||
meta.Conflicts = append(meta.Conflicts, rpmDepFromEntry(con))
|
||||
}
|
||||
for _, obs := range pkg.Obsoletes() {
|
||||
meta.Obsoletes = append(meta.Obsoletes, rpmDepFromEntry(obs))
|
||||
}
|
||||
|
||||
if meta.Requires == nil {
|
||||
meta.Requires = []provider.RPMDep{}
|
||||
@@ -140,6 +147,12 @@ func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, conte
|
||||
if meta.Provides == nil {
|
||||
meta.Provides = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Conflicts == nil {
|
||||
meta.Conflicts = []provider.RPMDep{}
|
||||
}
|
||||
if meta.Obsoletes == nil {
|
||||
meta.Obsoletes = []provider.RPMDep{}
|
||||
}
|
||||
meta.Files = []provider.RPMFile{}
|
||||
meta.Changelogs = []provider.RPMChangelog{}
|
||||
|
||||
@@ -229,10 +242,28 @@ func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileSt
|
||||
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
// readMetadataEntries loads the repo's derived metadata, translating the read
|
||||
// error into an HTTP response. A canceled/deadline-exceeded context (typically a
|
||||
// client that went away) becomes a retryable 503 rather than a hard 500, so a
|
||||
// dnf disconnect never looks like a server fault. ok is false when a response
|
||||
// has already been written.
|
||||
func readMetadataEntries(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) ([]provider.RPMMetadata, bool) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
slog.Warn("rpm: metadata read canceled", "repo", repoName, "error", err)
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return nil, false
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return nil, false
|
||||
}
|
||||
return metas, true
|
||||
}
|
||||
|
||||
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, ok := readMetadataEntries(w, r, reader, repoName)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -252,9 +283,8 @@ func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader pr
|
||||
}
|
||||
|
||||
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
metas, ok := readMetadataEntries(w, r, reader, repoName)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -264,9 +294,8 @@ func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader p
|
||||
}
|
||||
|
||||
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
metas, ok := readMetadataEntries(w, r, reader, repoName)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -276,9 +305,8 @@ func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader
|
||||
}
|
||||
|
||||
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
|
||||
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
metas, ok := readMetadataEntries(w, r, reader, repoName)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -363,6 +391,20 @@ func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:requires>\n")
|
||||
}
|
||||
if len(m.Conflicts) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:conflicts>\n")
|
||||
for _, d := range m.Conflicts {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:conflicts>\n")
|
||||
}
|
||||
if len(m.Obsoletes) > 0 {
|
||||
xmlBuf.WriteString(" <rpm:obsoletes>\n")
|
||||
for _, d := range m.Obsoletes {
|
||||
writeRPMEntry(&xmlBuf, d)
|
||||
}
|
||||
xmlBuf.WriteString(" </rpm:obsoletes>\n")
|
||||
}
|
||||
|
||||
fmt.Fprintf(&xmlBuf, " </format>\n")
|
||||
fmt.Fprintf(&xmlBuf, "</package>\n")
|
||||
|
||||
@@ -15,6 +15,7 @@ const (
|
||||
PackagePuppet PackageType = "puppet"
|
||||
PackageTerraform PackageType = "terraform"
|
||||
PackageGoProxy PackageType = "goproxy"
|
||||
PackageGitHubRPM PackageType = "github_rpm"
|
||||
)
|
||||
|
||||
var validPackageTypes = map[PackageType]bool{
|
||||
@@ -28,6 +29,7 @@ var validPackageTypes = map[PackageType]bool{
|
||||
PackagePuppet: true,
|
||||
PackageTerraform: true,
|
||||
PackageGoProxy: true,
|
||||
PackageGitHubRPM: true,
|
||||
}
|
||||
|
||||
func (p PackageType) Valid() bool {
|
||||
|
||||
@@ -18,6 +18,7 @@ func TestPackageTypeValid(t *testing.T) {
|
||||
models.PackagePuppet,
|
||||
models.PackageTerraform,
|
||||
models.PackageGoProxy,
|
||||
models.PackageGitHubRPM,
|
||||
}
|
||||
for _, pt := range valid {
|
||||
if !pt.Valid() {
|
||||
|
||||
Reference in New Issue
Block a user