unkin-agent
b2a6be8eb5
Add the Debian/apt analog of github_rpm: a metadata-only remote that scans a GitHub repo's releases for .deb assets, derives per-asset control metadata via a ranged prefix fetch (never downloading whole packages), synthesizes a flat apt repository, and redirects .deb downloads to a backend releases_remote. - Add PackageGitHubDeb to the package-type enum + validity map. - Add the github_deb provider (internal/provider/deb/github.go): ServeRemote serves Packages/Packages.gz/Release, 404s the signed index variants (consumed via [trusted=yes]), and 302-redirects *.deb to the releases_remote; deriveAsset ranged-GETs the ar prefix, locates control.tar.*, and parses the control paragraph, doubling the range on truncation; sha256 comes from the asset digest when present, else a one-time full stream. - Add the github_deb background Syncer (internal/provider/deb/syncer.go): its own worker pool, shared rate limiter, deduped queue, and DB-lease-gated scans. - Add github_deb_sync_state table plus ListGitHubDebRemotes/Claim/Release DB helpers (separate from the rpm ones). - Prime github_deb remotes on create and run the deb syncer alongside the rpm one; route prime-on-create by package type. - Reuse the deb apt-index generators and control parser; skip empty hash lines in the Packages index so a SHA256-only metadata entry is valid.
421 lines
13 KiB
Go
421 lines
13 KiB
Go
package deb
|
|
|
|
import (
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"git.unkin.net/unkin/artifactapi/internal/provider"
|
|
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
// fakeStore is an in-memory provider.RemoteMetadataStore + DebMetadataReader
|
|
// keyed by file_path, mirroring the (repo_name, file_path) uniqueness of the
|
|
// real deb_metadata table.
|
|
type fakeStore struct {
|
|
mu sync.Mutex
|
|
rows map[string]provider.DebMetadata
|
|
}
|
|
|
|
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.DebMetadata{}} }
|
|
|
|
func (f *fakeStore) InsertDebMetadata(_ context.Context, m *provider.DebMetadata) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if _, ok := f.rows[m.FilePath]; ok {
|
|
return nil // ON CONFLICT DO NOTHING
|
|
}
|
|
f.rows[m.FilePath] = *m
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeStore) DeleteDebMetadata(_ context.Context, _, filePath string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
delete(f.rows, filePath)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
|
func (f *fakeStore) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
|
func (f *fakeStore) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
|
|
return nil, nil
|
|
}
|
|
|
|
func (f *fakeStore) ListDebMetadataEntries(ctx context.Context, _ string) ([]provider.DebMetadata, error) {
|
|
if err := ctx.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
out := make([]provider.DebMetadata, 0, len(f.rows))
|
|
for _, m := range f.rows {
|
|
out = append(out, m)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// githubFixture serves the releases API and the .deb asset downloads (with Range
|
|
// support) for a set of packages. digest controls whether the asset carries a
|
|
// sha256 digest (no-download path) or not (compute path).
|
|
type githubFixture struct {
|
|
srv *httptest.Server
|
|
debBytes map[string][]byte
|
|
rangeHit map[string]int
|
|
fullHit map[string]int
|
|
etag string
|
|
releasesHit int
|
|
notModHit int
|
|
releaseAuth string
|
|
assetAuth string
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
|
t.Helper()
|
|
f := &githubFixture{
|
|
debBytes: map[string][]byte{},
|
|
rangeHit: map[string]int{},
|
|
fullHit: map[string]int{},
|
|
}
|
|
f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
|
|
|
|
mux := http.NewServeMux()
|
|
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
|
|
page := r.URL.Query().Get("page")
|
|
if page != "" && page != "1" {
|
|
w.Write([]byte("[]"))
|
|
return
|
|
}
|
|
f.mu.Lock()
|
|
f.releasesHit++
|
|
f.releaseAuth = r.Header.Get("Authorization")
|
|
etag := f.etag
|
|
if etag != "" && r.Header.Get("If-None-Match") == etag {
|
|
f.notModHit++
|
|
f.mu.Unlock()
|
|
w.WriteHeader(http.StatusNotModified)
|
|
return
|
|
}
|
|
f.mu.Unlock()
|
|
if etag != "" {
|
|
w.Header().Set("ETag", etag)
|
|
}
|
|
var assets []map[string]any
|
|
for name := range f.debBytes {
|
|
a := map[string]any{
|
|
"name": name,
|
|
"size": len(f.debBytes[name]),
|
|
"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2-3/" + name,
|
|
}
|
|
if withDigest {
|
|
sum := sha256.Sum256(f.debBytes[name])
|
|
a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
|
|
}
|
|
assets = append(assets, a)
|
|
}
|
|
rel := []map[string]any{{"tag_name": "v1.2-3", "draft": false, "assets": assets}}
|
|
json.NewEncoder(w).Encode(rel)
|
|
})
|
|
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
|
|
name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
|
body, ok := f.debBytes[name]
|
|
if !ok {
|
|
http.Error(w, "not found", 404)
|
|
return
|
|
}
|
|
rng := r.Header.Get("Range")
|
|
f.mu.Lock()
|
|
f.assetAuth = r.Header.Get("Authorization")
|
|
if rng != "" {
|
|
f.rangeHit[name]++
|
|
} else {
|
|
f.fullHit[name]++
|
|
}
|
|
f.mu.Unlock()
|
|
|
|
if rng == "" {
|
|
w.WriteHeader(200)
|
|
w.Write(body)
|
|
return
|
|
}
|
|
var end int
|
|
fmt.Sscanf(rng, "bytes=0-%d", &end)
|
|
if end >= len(body)-1 {
|
|
end = len(body) - 1
|
|
}
|
|
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
|
|
w.Header().Set("Content-Length", strconv.Itoa(end+1))
|
|
w.WriteHeader(http.StatusPartialContent)
|
|
w.Write(body[:end+1])
|
|
})
|
|
f.srv = httptest.NewServer(mux)
|
|
t.Cleanup(f.srv.Close)
|
|
return f
|
|
}
|
|
|
|
func (f *githubFixture) remote() models.Remote {
|
|
return models.Remote{
|
|
Name: "acme-deb",
|
|
PackageType: models.PackageGitHubDeb,
|
|
BaseURL: f.srv.URL + "/repos/acme/tools",
|
|
ReleasesRemote: "github",
|
|
MutableTTL: 3600,
|
|
}
|
|
}
|
|
|
|
func newTestProvider() *GitHubProvider {
|
|
p := newGitHubProvider()
|
|
p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
|
|
p.headerMax = 1 << 20
|
|
return p
|
|
}
|
|
|
|
const demoPath = "acme/tools/releases/download/v1.2-3/demo_1.2-3_amd64.deb"
|
|
|
|
func TestGitHubScanDerivesControlFromPrefixAndDigest(t *testing.T) {
|
|
fx := newGitHubFixture(t, true)
|
|
p := newTestProvider()
|
|
store := newFakeStore()
|
|
|
|
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
|
t.Fatalf("scan: %v", err)
|
|
}
|
|
|
|
metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
|
if len(metas) != 1 {
|
|
t.Fatalf("want 1 metadata row, got %d", len(metas))
|
|
}
|
|
m := metas[0]
|
|
if m.Name != "demo" || m.Version != "1.2-3" || m.Architecture != "amd64" {
|
|
t.Fatalf("bad control fields: %+v", m)
|
|
}
|
|
if m.FilePath != demoPath {
|
|
t.Fatalf("FilePath = %q, want %q", m.FilePath, demoPath)
|
|
}
|
|
if int(m.Size) != len(fx.debBytes["demo_1.2-3_amd64.deb"]) {
|
|
t.Fatalf("Size = %d, want %d", m.Size, len(fx.debBytes["demo_1.2-3_amd64.deb"]))
|
|
}
|
|
sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
|
|
if m.SHA256 != hex.EncodeToString(sum[:]) {
|
|
t.Fatalf("SHA256 = %q, want digest", m.SHA256)
|
|
}
|
|
if m.ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
|
|
t.Fatalf("ContentHash = %q", m.ContentHash)
|
|
}
|
|
if m.MD5 != "" {
|
|
t.Fatalf("MD5 should be unset for metadata-only derive, got %q", m.MD5)
|
|
}
|
|
if fx.fullHit["demo_1.2-3_amd64.deb"] != 0 {
|
|
t.Fatalf("expected no full download when digest present, got %d", fx.fullHit["demo_1.2-3_amd64.deb"])
|
|
}
|
|
if fx.rangeHit["demo_1.2-3_amd64.deb"] == 0 {
|
|
t.Fatalf("expected ranged control fetch")
|
|
}
|
|
if !strings.Contains(m.Control, "Package: demo") {
|
|
t.Fatalf("raw control not captured: %q", m.Control)
|
|
}
|
|
}
|
|
|
|
func TestGitHubChecksumComputedWhenDigestAbsent(t *testing.T) {
|
|
fx := newGitHubFixture(t, false)
|
|
p := newTestProvider()
|
|
store := newFakeStore()
|
|
|
|
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
|
t.Fatalf("scan: %v", err)
|
|
}
|
|
metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
|
if len(metas) != 1 {
|
|
t.Fatalf("want 1 row, got %d", len(metas))
|
|
}
|
|
sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
|
|
if metas[0].SHA256 != hex.EncodeToString(sum[:]) {
|
|
t.Fatalf("computed checksum mismatch: %q", metas[0].SHA256)
|
|
}
|
|
if fx.fullHit["demo_1.2-3_amd64.deb"] == 0 {
|
|
t.Fatalf("expected a full download to compute sha256 when digest absent")
|
|
}
|
|
}
|
|
|
|
func TestGitHubServeRemoteIndexAndRedirect(t *testing.T) {
|
|
fx := newGitHubFixture(t, true)
|
|
p := newTestProvider()
|
|
store := newFakeStore()
|
|
remote := fx.remote()
|
|
const proxyBase = "https://artifactapi.example"
|
|
|
|
// Release is served and triggers the initial scan.
|
|
rec := httptest.NewRecorder()
|
|
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Release", nil)
|
|
if !p.ServeRemote(rec, req, remote, "Release", proxyBase, store) {
|
|
t.Fatal("ServeRemote did not handle Release")
|
|
}
|
|
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "Architectures:") {
|
|
t.Fatalf("Release bad: code=%d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "amd64") {
|
|
t.Fatalf("Release missing arch: %s", rec.Body.String())
|
|
}
|
|
|
|
// Packages carries the package with a Filename that is the github-relative
|
|
// download path (so it resolves back to this remote and redirects).
|
|
rec = httptest.NewRecorder()
|
|
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
|
if !p.ServeRemote(rec, req, remote, "Packages", proxyBase, store) {
|
|
t.Fatal("ServeRemote did not handle Packages")
|
|
}
|
|
pkgs := rec.Body.String()
|
|
if !strings.Contains(pkgs, "Package: demo") {
|
|
t.Fatalf("Packages missing package: %s", pkgs)
|
|
}
|
|
if !strings.Contains(pkgs, "Filename: "+demoPath) {
|
|
t.Fatalf("Packages missing/incorrect Filename: %s", pkgs)
|
|
}
|
|
if !strings.Contains(pkgs, "SHA256: ") {
|
|
t.Fatalf("Packages missing SHA256: %s", pkgs)
|
|
}
|
|
if strings.Contains(pkgs, "MD5sum:") {
|
|
t.Fatalf("Packages should omit empty MD5sum: %s", pkgs)
|
|
}
|
|
|
|
// Packages.gz decompresses to the same content.
|
|
rec = httptest.NewRecorder()
|
|
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
|
if !p.ServeRemote(rec, req, remote, "Packages.gz", proxyBase, store) {
|
|
t.Fatal("ServeRemote did not handle Packages.gz")
|
|
}
|
|
gz, err := gzip.NewReader(rec.Body)
|
|
if err != nil {
|
|
t.Fatalf("gzip: %v", err)
|
|
}
|
|
unz, _ := io.ReadAll(gz)
|
|
if !strings.Contains(string(unz), "Package: demo") {
|
|
t.Fatalf("Packages.gz missing package: %s", unz)
|
|
}
|
|
|
|
// InRelease/Release.gpg 404 (unsigned, consumed via [trusted=yes]).
|
|
for _, sp := range []string{"InRelease", "Release.gpg"} {
|
|
rec = httptest.NewRecorder()
|
|
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
|
if !p.ServeRemote(rec, req, remote, sp, proxyBase, store) {
|
|
t.Fatalf("ServeRemote did not handle %s", sp)
|
|
}
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Fatalf("%s want 404, got %d", sp, rec.Code)
|
|
}
|
|
}
|
|
|
|
// A .deb request redirects to the backend releases_remote.
|
|
rec = httptest.NewRecorder()
|
|
req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/"+demoPath, nil)
|
|
if !p.ServeRemote(rec, req, remote, demoPath, proxyBase, store) {
|
|
t.Fatal("ServeRemote did not handle .deb")
|
|
}
|
|
if rec.Code != http.StatusFound {
|
|
t.Fatalf("want 302, got %d", rec.Code)
|
|
}
|
|
wantLoc := proxyBase + "/api/v1/remote/github/" + demoPath
|
|
if got := rec.Header().Get("Location"); got != wantLoc {
|
|
t.Fatalf("Location = %q, want %q", got, wantLoc)
|
|
}
|
|
}
|
|
|
|
// A canceled inbound request must still serve the warm cache (detached context),
|
|
// not turn the metadata read into a 500.
|
|
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
|
|
fx := newGitHubFixture(t, true)
|
|
p := newTestProvider()
|
|
store := newFakeStore()
|
|
remote := fx.remote()
|
|
|
|
if err := p.scan(context.Background(), remote, store); err != nil {
|
|
t.Fatalf("warm scan: %v", err)
|
|
}
|
|
p.mu.Lock()
|
|
p.lastScan[remote.Name] = time.Now()
|
|
p.mu.Unlock()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
rec := httptest.NewRecorder()
|
|
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil).WithContext(ctx)
|
|
|
|
if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
|
|
t.Fatal("ServeRemote did not handle Packages")
|
|
}
|
|
if rec.Code != http.StatusOK {
|
|
t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
|
|
}
|
|
if !strings.Contains(rec.Body.String(), "Package: demo") {
|
|
t.Fatalf("expected Packages served from cache, got %s", rec.Body.String())
|
|
}
|
|
}
|
|
|
|
func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
|
|
fx := newGitHubFixture(t, true)
|
|
p := newTestProvider()
|
|
store := newFakeStore()
|
|
remote := fx.remote()
|
|
remote.ReleasesRemote = ""
|
|
|
|
rec := httptest.NewRecorder()
|
|
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
|
if !p.ServeRemote(rec, req, remote, demoPath, "https://x", store) {
|
|
t.Fatal("expected handled")
|
|
}
|
|
if rec.Code != http.StatusInternalServerError {
|
|
t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
|
|
}
|
|
}
|
|
|
|
func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
|
|
fx := newGitHubFixture(t, true)
|
|
p := newTestProvider()
|
|
store := newFakeStore()
|
|
|
|
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
|
t.Fatalf("scan: %v", err)
|
|
}
|
|
if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 1 {
|
|
t.Fatalf("want 1 row after first scan, got %d", len(rows))
|
|
}
|
|
|
|
delete(fx.debBytes, "demo_1.2-3_amd64.deb")
|
|
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
|
t.Fatalf("rescan: %v", err)
|
|
}
|
|
if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 0 {
|
|
t.Fatalf("want 0 rows after prune, got %d", len(rows))
|
|
}
|
|
}
|
|
|
|
func TestGitHubAssetPatternFilter(t *testing.T) {
|
|
fx := newGitHubFixture(t, true)
|
|
fx.debBytes["other_9_arm64.deb"] = testsupport.MinimalDeb("other", "9", "arm64")
|
|
p := newTestProvider()
|
|
store := newFakeStore()
|
|
remote := fx.remote()
|
|
remote.Patterns = []string{`^demo_.*_amd64\.deb$`}
|
|
|
|
if err := p.scan(context.Background(), remote, store); err != nil {
|
|
t.Fatalf("scan: %v", err)
|
|
}
|
|
rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
|
if len(rows) != 1 || rows[0].Name != "demo" {
|
|
t.Fatalf("pattern filter failed, rows=%+v", rows)
|
|
}
|
|
}
|