Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 unkinben
|
8ced48901f |
@@ -3,16 +3,11 @@
|
||||
package e2edocker
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
func uploadFile(t *testing.T, repo, filePath string, body []byte, contentType string) {
|
||||
@@ -96,98 +91,3 @@ func TestLocalRPMRepodata(t *testing.T) {
|
||||
t.Fatalf("repomd.xml not a valid repodata document: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalDebRepo uploads a .deb and validates that the flat apt index
|
||||
// (Packages / Release) is generated automatically from the parsed control
|
||||
// stanza (the deb-local analog of rpm repodata generation).
|
||||
func TestLocalDebRepo(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-deb","package_type":"deb","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-deb")
|
||||
|
||||
deb := testsupport.MinimalDeb("e2e-testpkg", "1.0.0", "amd64")
|
||||
uploadFile(t, "local-deb", "e2e-testpkg_1.0.0_amd64.deb", deb, "application/vnd.debian.binary-package")
|
||||
|
||||
// The index is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-deb/Packages"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("Packages: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
pkgs := string(body)
|
||||
for _, want := range []string{"Package: e2e-testpkg", "Version: 1.0.0", "Architecture: amd64", "Filename: pool/e2e-testpkg_1.0.0_amd64.deb", "SHA256:"} {
|
||||
if !strings.Contains(pkgs, want) {
|
||||
t.Fatalf("Packages missing %q:\n%s", want, pkgs)
|
||||
}
|
||||
}
|
||||
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/Release"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("Release: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
rel := string(body)
|
||||
for _, want := range []string{"Architectures: amd64", "SHA256:", "Packages"} {
|
||||
if !strings.Contains(rel, want) {
|
||||
t.Fatalf("Release missing %q:\n%s", want, rel)
|
||||
}
|
||||
}
|
||||
|
||||
// The .deb downloads back byte-identical from its pool path.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-deb/pool/e2e-testpkg_1.0.0_amd64.deb"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download deb: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, deb) {
|
||||
t.Fatalf("deb content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// TestLocalAlpineIndex uploads an .apk to an alpine local repo and validates
|
||||
// that a per-arch APKINDEX.tar.gz is generated automatically from the parsed
|
||||
// .PKGINFO (the apk-local analog of rpm repodata / deb Packages generation).
|
||||
func TestLocalAlpineIndex(t *testing.T) {
|
||||
createRepo(t, `{"name":"local-alpine","package_type":"alpine","repo_type":"local"}`)
|
||||
defer deleteRepo(t, "local-alpine")
|
||||
|
||||
apk := testsupport.MinimalApk("e2e-testpkg", "1.0-r0", "x86_64")
|
||||
uploadFile(t, "local-alpine", "x86_64/e2e-testpkg-1.0-r0.apk", apk, "application/vnd.android.package-archive")
|
||||
|
||||
// The index is generated asynchronously after upload; poll for it.
|
||||
resp, body := getEventually(t, api("/api/v1/local/local-alpine/x86_64/APKINDEX.tar.gz"), 15*time.Second)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("APKINDEX: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
|
||||
zr, err := gzip.NewReader(bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not gzip: %v", err)
|
||||
}
|
||||
tarBytes, _ := io.ReadAll(zr)
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
var index string
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not tar: %v", err)
|
||||
}
|
||||
if hdr.Name == "APKINDEX" {
|
||||
b, _ := io.ReadAll(tr)
|
||||
index = string(b)
|
||||
}
|
||||
}
|
||||
for _, want := range []string{"P:e2e-testpkg", "V:1.0-r0", "A:x86_64", "C:Q1", "S:", "I:"} {
|
||||
if !strings.Contains(index, want) {
|
||||
t.Fatalf("APKINDEX missing %q:\n%s", want, index)
|
||||
}
|
||||
}
|
||||
|
||||
// The .apk downloads back byte-identical from its arch path.
|
||||
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-alpine/x86_64/e2e-testpkg-1.0-r0.apk"), nil, "")
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("download apk: status %d: %s", resp.StatusCode, body)
|
||||
}
|
||||
if !bytes.Equal(body, apk) {
|
||||
t.Fatalf("apk content mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,13 +9,11 @@ require (
|
||||
github.com/go-chi/chi/v5 v5.3.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.10.0
|
||||
github.com/klauspost/compress v1.19.2
|
||||
github.com/minio/minio-go/v7 v7.2.0
|
||||
github.com/redis/go-redis/v9 v9.20.0
|
||||
github.com/testcontainers/testcontainers-go v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0
|
||||
github.com/testcontainers/testcontainers-go/modules/redis v0.42.0
|
||||
github.com/ulikunitz/xz v0.5.16
|
||||
golang.org/x/crypto v0.51.0
|
||||
golang.org/x/time v0.15.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
@@ -53,6 +51,7 @@ require (
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
|
||||
github.com/klauspost/crc32 v1.3.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
|
||||
|
||||
@@ -85,8 +85,8 @@ github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0=
|
||||
github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8=
|
||||
github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao=
|
||||
github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
|
||||
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
|
||||
@@ -189,8 +189,6 @@ github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYI
|
||||
github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI=
|
||||
github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw=
|
||||
github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ=
|
||||
github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0=
|
||||
github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
|
||||
@@ -12,20 +12,18 @@ import (
|
||||
)
|
||||
|
||||
// Primer enqueues a background metadata prime for a newly created remote so the
|
||||
// create call never blocks on a derive. *rpm.Syncer and *deb.Syncer satisfy it.
|
||||
// create call never blocks on a derive. *rpm.Syncer satisfies it.
|
||||
type Primer interface {
|
||||
EnqueuePrime(remote models.Remote)
|
||||
}
|
||||
|
||||
type RemotesHandler struct {
|
||||
db *database.DB
|
||||
primers map[models.PackageType]Primer
|
||||
db *database.DB
|
||||
primer Primer
|
||||
}
|
||||
|
||||
// NewRemotesHandler wires the handler to the per-type metadata primers. primers
|
||||
// may be nil; a package type with no registered primer simply skips priming.
|
||||
func NewRemotesHandler(db *database.DB, primers map[models.PackageType]Primer) *RemotesHandler {
|
||||
return &RemotesHandler{db: db, primers: primers}
|
||||
func NewRemotesHandler(db *database.DB, primer Primer) *RemotesHandler {
|
||||
return &RemotesHandler{db: db, primer: primer}
|
||||
}
|
||||
|
||||
func (h *RemotesHandler) Routes() chi.Router {
|
||||
@@ -86,10 +84,10 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
// Prime a metadata-only remote (github_rpm/github_deb) in the background so
|
||||
// its first index request is served from cache instead of a cold derive.
|
||||
if primer := h.primers[remote.PackageType]; primer != nil {
|
||||
primer.EnqueuePrime(remote)
|
||||
// Prime a github_rpm remote's metadata in the background so its first
|
||||
// repodata request is served from cache instead of a cold on-demand derive.
|
||||
if h.primer != nil && remote.PackageType == models.PackageGitHubRPM {
|
||||
h.primer.EnqueuePrime(remote)
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, remote)
|
||||
}
|
||||
|
||||
@@ -1,71 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ListGitHubAlpineRemotes returns every github_alpine remote so the syncer can
|
||||
// sweep them on each poll tick.
|
||||
func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubAlpine)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubAlpineSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_alpine_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubAlpineSyncLease records the completed scan and frees the lease.
|
||||
// Only the owning replica may release; last_synced_at advances so the next poll
|
||||
// waits a full freshness window, and etag is persisted for the next conditional
|
||||
// request.
|
||||
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_alpine_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertAlpineMetadata(ctx context.Context, meta *provider.AlpineMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO alpine_metadata (
|
||||
repo_name, file_path, content_hash, checksum,
|
||||
name, version, arch, download_size, installed_size,
|
||||
description, url, license, origin, maintainer,
|
||||
build_time, commit_hash, provider_priority,
|
||||
depends, provides, install_if
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash, meta.Checksum,
|
||||
meta.Name, meta.Version, meta.Arch, meta.DownloadSize, meta.InstalledSize,
|
||||
meta.Description, meta.URL, meta.License, meta.Origin, meta.Maintainer,
|
||||
meta.BuildTime, meta.Commit, meta.ProviderPriority,
|
||||
strings.Join(meta.Depends, " "), strings.Join(meta.Provides, " "), strings.Join(meta.InstallIf, " "),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM alpine_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]provider.AlpineMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash, checksum,
|
||||
name, version, arch, download_size, installed_size,
|
||||
description, url, license, origin, maintainer,
|
||||
build_time, commit_hash, provider_priority,
|
||||
depends, provides, install_if
|
||||
FROM alpine_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, arch, file_path
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.AlpineMetadata
|
||||
for rows.Next() {
|
||||
var m provider.AlpineMetadata
|
||||
var depends, provides, installIf string
|
||||
if err := rows.Scan(
|
||||
&m.RepoName, &m.FilePath, &m.ContentHash, &m.Checksum,
|
||||
&m.Name, &m.Version, &m.Arch, &m.DownloadSize, &m.InstalledSize,
|
||||
&m.Description, &m.URL, &m.License, &m.Origin, &m.Maintainer,
|
||||
&m.BuildTime, &m.Commit, &m.ProviderPriority,
|
||||
&depends, &provides, &installIf,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Depends = strings.Fields(depends)
|
||||
m.Provides = strings.Fields(provides)
|
||||
m.InstallIf = strings.Fields(installIf)
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// ListGitHubDebRemotes returns every github_deb remote so the syncer can sweep
|
||||
// them on each poll tick.
|
||||
func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error) {
|
||||
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubDeb)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var remotes []models.Remote
|
||||
for rows.Next() {
|
||||
var r models.Remote
|
||||
if err := scanRemote(rows, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
remotes = append(remotes, r)
|
||||
}
|
||||
return remotes, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
|
||||
// succeeds only when the remote is due (never synced, or synced longer than
|
||||
// freshness ago) and no live lease is held by another replica. A zero freshness
|
||||
// (prime scans) ignores the recency gate. The returned etag is the stored
|
||||
// releases-list ETag, shared across replicas.
|
||||
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
row := db.Pool.QueryRow(ctx, `
|
||||
INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
||||
VALUES ($1, $2, now() + make_interval(secs => $4))
|
||||
ON CONFLICT (remote_name) DO UPDATE
|
||||
SET sync_lease_owner = $2,
|
||||
sync_lease_expires = now() + make_interval(secs => $4)
|
||||
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
||||
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
||||
RETURNING s.etag
|
||||
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
||||
|
||||
var etag string
|
||||
if err := row.Scan(&etag); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, "", nil
|
||||
}
|
||||
return false, "", err
|
||||
}
|
||||
return true, etag, nil
|
||||
}
|
||||
|
||||
// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
|
||||
// the owning replica may release; last_synced_at advances so the next poll waits
|
||||
// a full freshness window, and etag is persisted for the next conditional request.
|
||||
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
UPDATE github_deb_sync_state
|
||||
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
||||
WHERE remote_name = $1 AND sync_lease_owner = $2
|
||||
`, remoteName, owner, syncedAt, etag)
|
||||
return err
|
||||
}
|
||||
@@ -1,90 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func seedGitHubDebRemote(t *testing.T, name string) {
|
||||
t.Helper()
|
||||
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
||||
Name: name, PackageType: models.PackageGitHubDeb, RepoType: models.RepoTypeRemote,
|
||||
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
|
||||
}); err != nil {
|
||||
t.Fatalf("seed github_deb remote: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGitHubDebSyncLease exercises the real SQL: exactly one replica may hold the
|
||||
// lease, the recency window blocks a too-soon periodic re-claim, and a prime
|
||||
// (freshness 0) bypasses recency but still respects a live lease.
|
||||
func TestGitHubDebSyncLease(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "ghdeb-lease-" + time.Now().Format("150405.000000")
|
||||
seedGitHubDebRemote(t, name)
|
||||
|
||||
const lease = 15 * time.Minute
|
||||
freshness := time.Hour
|
||||
|
||||
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
if etag != "" {
|
||||
t.Fatalf("initial etag should be empty, got %q", etag)
|
||||
}
|
||||
|
||||
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 claim err: %v", err)
|
||||
}
|
||||
if claimed2 {
|
||||
t.Fatal("replica-2 claimed while replica-1 holds the lease")
|
||||
}
|
||||
|
||||
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
||||
t.Fatalf("release: %v", err)
|
||||
}
|
||||
|
||||
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
||||
if err != nil {
|
||||
t.Fatalf("replica-2 recency claim err: %v", err)
|
||||
}
|
||||
if claimed3 {
|
||||
t.Fatal("periodic claim succeeded inside the freshness window")
|
||||
}
|
||||
|
||||
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
|
||||
if err != nil || !claimed4 {
|
||||
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
|
||||
}
|
||||
if etag4 != `"etag-1"` {
|
||||
t.Fatalf("prime claim etag = %q, want persisted \"etag-1\"", etag4)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListGitHubDebRemotes(t *testing.T) {
|
||||
requireDB(t)
|
||||
name := "ghdeb-list-" + time.Now().Format("150405.000000")
|
||||
seedGitHubDebRemote(t, name)
|
||||
seedRemote(t, "generic-"+time.Now().Format("150405.000000"))
|
||||
|
||||
remotes, err := testDB.ListGitHubDebRemotes(ctx())
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, r := range remotes {
|
||||
if r.PackageType != models.PackageGitHubDeb {
|
||||
t.Fatalf("non-github_deb remote returned: %s (%s)", r.Name, r.PackageType)
|
||||
}
|
||||
if r.Name == name {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("seeded remote %q not returned", name)
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func (db *DB) InsertDebMetadata(ctx context.Context, meta *provider.DebMetadata) error {
|
||||
_, err := db.Pool.Exec(ctx, `
|
||||
INSERT INTO deb_metadata (
|
||||
repo_name, file_path, content_hash,
|
||||
name, version, architecture, control,
|
||||
size, md5, sha256
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)
|
||||
ON CONFLICT (repo_name, file_path) DO NOTHING
|
||||
`,
|
||||
meta.RepoName, meta.FilePath, meta.ContentHash,
|
||||
meta.Name, meta.Version, meta.Architecture, meta.Control,
|
||||
meta.Size, meta.MD5, meta.SHA256,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteDebMetadata(ctx context.Context, repoName, filePath string) error {
|
||||
_, err := db.Pool.Exec(ctx, `DELETE FROM deb_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) ListDebMetadataEntries(ctx context.Context, repoName string) ([]provider.DebMetadata, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT repo_name, file_path, content_hash,
|
||||
name, version, architecture, control,
|
||||
size, md5, sha256, created_at
|
||||
FROM deb_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, version, architecture, file_path
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var result []provider.DebMetadata
|
||||
for rows.Next() {
|
||||
var m provider.DebMetadata
|
||||
if err := rows.Scan(
|
||||
&m.RepoName, &m.FilePath, &m.ContentHash,
|
||||
&m.Name, &m.Version, &m.Architecture, &m.Control,
|
||||
&m.Size, &m.MD5, &m.SHA256, &m.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, m)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
@@ -164,53 +164,6 @@ func (db *DB) migrate() error {
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
|
||||
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
|
||||
|
||||
CREATE TABLE IF NOT EXISTS deb_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
architecture TEXT NOT NULL,
|
||||
control TEXT NOT NULL,
|
||||
size BIGINT DEFAULT 0,
|
||||
md5 TEXT DEFAULT '',
|
||||
sha256 TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS alpine_metadata (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
repo_name TEXT NOT NULL,
|
||||
file_path TEXT NOT NULL,
|
||||
content_hash TEXT NOT NULL,
|
||||
checksum TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL,
|
||||
arch TEXT NOT NULL,
|
||||
download_size BIGINT DEFAULT 0,
|
||||
installed_size BIGINT DEFAULT 0,
|
||||
description TEXT DEFAULT '',
|
||||
url TEXT DEFAULT '',
|
||||
license TEXT DEFAULT '',
|
||||
origin TEXT DEFAULT '',
|
||||
maintainer TEXT DEFAULT '',
|
||||
build_time BIGINT DEFAULT 0,
|
||||
commit_hash TEXT DEFAULT '',
|
||||
provider_priority TEXT DEFAULT '',
|
||||
depends TEXT DEFAULT '',
|
||||
provides TEXT DEFAULT '',
|
||||
install_if TEXT DEFAULT '',
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(repo_name, file_path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
|
||||
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
@@ -219,22 +172,6 @@ func (db *DB) migrate() error {
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_deb_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS github_alpine_sync_state (
|
||||
remote_name TEXT PRIMARY KEY,
|
||||
etag TEXT DEFAULT '',
|
||||
last_synced_at TIMESTAMPTZ,
|
||||
sync_lease_owner TEXT DEFAULT '',
|
||||
sync_lease_expires TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS signing_keys (
|
||||
purpose TEXT PRIMARY KEY,
|
||||
private_key_armor TEXT NOT NULL,
|
||||
|
||||
@@ -3,7 +3,6 @@ package database
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
@@ -66,7 +65,6 @@ type RPMMetadataRow struct {
|
||||
Obsoletes json.RawMessage
|
||||
Files json.RawMessage
|
||||
Changelogs json.RawMessage
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
|
||||
@@ -96,7 +94,6 @@ func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]pr
|
||||
SourceRPM: r.SourceRPM,
|
||||
URL: r.URL,
|
||||
Packager: r.Packager,
|
||||
CreatedAt: r.CreatedAt,
|
||||
}
|
||||
json.Unmarshal(r.Requires, &meta.Requires)
|
||||
json.Unmarshal(r.Provides, &meta.Provides)
|
||||
@@ -115,11 +112,10 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
name, epoch, version, release, arch,
|
||||
summary, description, rpm_size, installed_size,
|
||||
license, vendor, build_group, build_host, source_rpm, url, packager,
|
||||
requires, provides, conflicts, obsoletes, files, changelogs,
|
||||
created_at
|
||||
requires, provides, conflicts, obsoletes, files, changelogs
|
||||
FROM rpm_metadata
|
||||
WHERE repo_name = $1
|
||||
ORDER BY name, epoch, version, release, arch, file_path
|
||||
ORDER BY name, epoch, version, release, arch
|
||||
`, repoName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -135,7 +131,6 @@ func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetada
|
||||
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
|
||||
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
|
||||
&r.Requires, &r.Provides, &r.Conflicts, &r.Obsoletes, &r.Files, &r.Changelogs,
|
||||
&r.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -1,27 +1,12 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"archive/tar"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -61,349 +46,3 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte,
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
// --- LocalUploader: hosting real .apk packages -----------------------------
|
||||
|
||||
// ValidateUpload accepts any *.apk and preserves the client-supplied directory
|
||||
// (the arch prefix) as the storage path, since arch cannot be parsed from the
|
||||
// filename alone and the generic uploader hands us only the path. apk clients
|
||||
// fetch packages at <arch>/<file>.apk, so publishers upload to that same path;
|
||||
// AfterUpload records the true arch (from .PKGINFO) for index filtering.
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
clean := strings.TrimPrefix(path.Clean("/"+filePath), "/")
|
||||
filename := clean
|
||||
if i := strings.LastIndex(clean, "/"); i >= 0 {
|
||||
filename = clean[i+1:]
|
||||
}
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".apk") {
|
||||
return "", "", fmt.Errorf("file must be a .apk package")
|
||||
}
|
||||
return clean, "application/vnd.android.package-archive", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := storagePath
|
||||
if i := strings.LastIndex(storagePath, "/"); i >= 0 {
|
||||
filename = storagePath[i+1:]
|
||||
}
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
raw, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
meta, err := parseApk(raw)
|
||||
if err != nil {
|
||||
slog.Error("alpine metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
meta.RepoName = repoName
|
||||
meta.FilePath = storagePath
|
||||
meta.ContentHash = contentHash
|
||||
meta.DownloadSize = blobSize
|
||||
|
||||
if meta.Name == "" || meta.Arch == "" {
|
||||
slog.Error("alpine metadata: .PKGINFO missing pkgname/arch", "repo", repoName, "path", storagePath)
|
||||
return
|
||||
}
|
||||
|
||||
store, ok := db.(provider.AlpineMetadataStore)
|
||||
if !ok {
|
||||
slog.Error("alpine metadata: store does not support alpine metadata", "repo", repoName)
|
||||
return
|
||||
}
|
||||
if err := store.InsertAlpineMetadata(ctx, meta); err != nil {
|
||||
slog.Error("alpine metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("alpine metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
deleter, ok := db.(provider.AlpineMetadataDeleter)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err := deleter.DeleteAlpineMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("alpine metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("alpine metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- LocalIndexer: generating a per-arch APKINDEX.tar.gz -------------------
|
||||
|
||||
// normalizeIndexPath collapses apk's dot-segment prefix: an /etc/apk/repositories
|
||||
// line of "<url>/api/v1/local/<name>" makes apk request "./<arch>/APKINDEX.tar.gz".
|
||||
// Mirrors deb's flat-repo normalization.
|
||||
func normalizeIndexPath(p string) string {
|
||||
return strings.TrimPrefix(path.Clean("/"+p), "/")
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
clean := normalizeIndexPath(reqPath)
|
||||
if !strings.HasSuffix(clean, "APKINDEX.tar.gz") {
|
||||
return false
|
||||
}
|
||||
arch := strings.TrimSuffix(clean, "APKINDEX.tar.gz")
|
||||
arch = strings.Trim(arch, "/")
|
||||
if arch == "" || strings.Contains(arch, "/") {
|
||||
http.Error(w, "APKINDEX must be requested per-arch: <arch>/APKINDEX.tar.gz", http.StatusNotFound)
|
||||
return true
|
||||
}
|
||||
|
||||
reader, ok := files.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
metas, err := reader.ListAlpineMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
slog.Warn("alpine: metadata read canceled", "repo", repoName, "error", err)
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return true
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
var filtered []provider.AlpineMetadata
|
||||
for _, m := range metas {
|
||||
if m.Arch == arch {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateAPKIndex(filtered))
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("alpine local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
// --- pure-Go .apk parsing --------------------------------------------------
|
||||
|
||||
// parseApk reads an .apk (up to three concatenated, independently gzipped tar
|
||||
// streams: optional signature, control, data). It locates the control stream by
|
||||
// its .PKGINFO member, computes the apk pull checksum C: = "Q1" +
|
||||
// base64(sha1(<control gzip stream bytes>)), and reads the .PKGINFO fields.
|
||||
func parseApk(raw []byte) (*provider.AlpineMetadata, error) {
|
||||
members, err := gzipMembers(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, m := range members {
|
||||
pkginfo, ok := pkginfoFromTar(m.tar)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
meta := parsePkginfo(pkginfo)
|
||||
sum := sha1.Sum(m.raw)
|
||||
meta.Checksum = "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
return meta, nil
|
||||
}
|
||||
return nil, errors.New("no .PKGINFO found in any .apk gzip stream")
|
||||
}
|
||||
|
||||
type gzMember struct {
|
||||
raw []byte // the raw bytes of this gzip stream (for the Q1 checksum)
|
||||
tar []byte // the decompressed tar payload
|
||||
}
|
||||
|
||||
// gzipMembers splits the concatenated gzip streams, returning each stream's raw
|
||||
// bytes alongside its decompressed tar. It relies on bytes.Reader being an
|
||||
// io.ByteReader (so compress/gzip does not over-read past a member's trailer)
|
||||
// to recover exact stream boundaries via Multistream(false)+Reset.
|
||||
func gzipMembers(data []byte) ([]gzMember, error) {
|
||||
br := bytes.NewReader(data)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var members []gzMember
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
end := len(data) - br.Len()
|
||||
members = append(members, gzMember{raw: data[prev:end], tar: out})
|
||||
prev = end
|
||||
if err := zr.Reset(br); err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
func pkginfoFromTar(tarBytes []byte) (string, bool) {
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
if strings.TrimPrefix(hdr.Name, "./") == ".PKGINFO" {
|
||||
b, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return string(b), true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parsePkginfo reads the "key = value" .PKGINFO text, collecting the repeated
|
||||
// depend/provides/install_if keys into slices.
|
||||
func parsePkginfo(text string) *provider.AlpineMetadata {
|
||||
m := &provider.AlpineMetadata{}
|
||||
sc := bufio.NewScanner(strings.NewReader(text))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := strings.TrimSpace(sc.Text())
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
idx := strings.Index(line, "=")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
val := strings.TrimSpace(line[idx+1:])
|
||||
switch key {
|
||||
case "pkgname":
|
||||
m.Name = val
|
||||
case "pkgver":
|
||||
m.Version = val
|
||||
case "arch":
|
||||
m.Arch = val
|
||||
case "pkgdesc":
|
||||
m.Description = val
|
||||
case "url":
|
||||
m.URL = val
|
||||
case "license":
|
||||
m.License = val
|
||||
case "origin":
|
||||
m.Origin = val
|
||||
case "maintainer":
|
||||
m.Maintainer = val
|
||||
case "builddate":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
m.BuildTime = n
|
||||
}
|
||||
case "commit":
|
||||
m.Commit = val
|
||||
case "size":
|
||||
if n, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
m.InstalledSize = n
|
||||
}
|
||||
case "provider_priority":
|
||||
m.ProviderPriority = val
|
||||
case "depend":
|
||||
if val != "" {
|
||||
m.Depends = append(m.Depends, val)
|
||||
}
|
||||
case "provides":
|
||||
if val != "" {
|
||||
m.Provides = append(m.Provides, val)
|
||||
}
|
||||
case "install_if":
|
||||
if val != "" {
|
||||
m.InstallIf = append(m.InstallIf, val)
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// generateAPKIndex builds the APKINDEX.tar.gz = gzip(tar(APKINDEX)) for the
|
||||
// given (already arch-filtered) rows. Records are blank-line separated; fields
|
||||
// follow the canonical C/P/V/A/S/I/T/U/L/o/m/t/c/k/D/p/i order and empties are
|
||||
// omitted. Unsigned (clients use --allow-untrusted), matching rpm gpgcheck=0.
|
||||
func generateAPKIndex(metas []provider.AlpineMetadata) []byte {
|
||||
var idx bytes.Buffer
|
||||
for i, m := range metas {
|
||||
if i > 0 {
|
||||
idx.WriteString("\n")
|
||||
}
|
||||
writeField(&idx, "C", m.Checksum)
|
||||
writeField(&idx, "P", m.Name)
|
||||
writeField(&idx, "V", m.Version)
|
||||
writeField(&idx, "A", m.Arch)
|
||||
writeField(&idx, "S", intField(m.DownloadSize))
|
||||
writeField(&idx, "I", intField(m.InstalledSize))
|
||||
writeField(&idx, "T", m.Description)
|
||||
writeField(&idx, "U", m.URL)
|
||||
writeField(&idx, "L", m.License)
|
||||
writeField(&idx, "o", m.Origin)
|
||||
writeField(&idx, "m", m.Maintainer)
|
||||
writeField(&idx, "t", intField(m.BuildTime))
|
||||
writeField(&idx, "c", m.Commit)
|
||||
writeField(&idx, "k", m.ProviderPriority)
|
||||
writeField(&idx, "D", strings.Join(m.Depends, " "))
|
||||
writeField(&idx, "p", strings.Join(m.Provides, " "))
|
||||
writeField(&idx, "i", strings.Join(m.InstallIf, " "))
|
||||
}
|
||||
|
||||
var tarBuf bytes.Buffer
|
||||
tw := tar.NewWriter(&tarBuf)
|
||||
body := idx.Bytes()
|
||||
// ModTime is pinned to the Unix epoch (never wall clock) so APKINDEX.tar.gz
|
||||
// is byte-identical across replicas and regenerations (issue #117); apk
|
||||
// clients ignore the tar mtime.
|
||||
tw.WriteHeader(&tar.Header{Name: "APKINDEX", Mode: 0o644, Size: int64(len(body)), Typeflag: tar.TypeReg, ModTime: time.Unix(0, 0)})
|
||||
tw.Write(body)
|
||||
tw.Close()
|
||||
|
||||
var gzBuf bytes.Buffer
|
||||
gz := gzip.NewWriter(&gzBuf)
|
||||
gz.Write(tarBuf.Bytes())
|
||||
gz.Close()
|
||||
return gzBuf.Bytes()
|
||||
}
|
||||
|
||||
func writeField(b *bytes.Buffer, key, val string) {
|
||||
if val == "" {
|
||||
return
|
||||
}
|
||||
b.WriteString(key)
|
||||
b.WriteString(":")
|
||||
b.WriteString(val)
|
||||
b.WriteString("\n")
|
||||
}
|
||||
|
||||
func intField(n int64) string {
|
||||
if n == 0 {
|
||||
return ""
|
||||
}
|
||||
return strconv.FormatInt(n, 10)
|
||||
}
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"io"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func apkFixture() []provider.AlpineMetadata {
|
||||
return []provider.AlpineMetadata{
|
||||
{
|
||||
RepoName: "r", FilePath: "x86_64/aaa-1.0-r0.apk", Checksum: "Q1aaa",
|
||||
Name: "aaa", Version: "1.0-r0", Arch: "x86_64", DownloadSize: 100, InstalledSize: 10,
|
||||
Description: "pkg aaa", URL: "https://a", License: "MIT",
|
||||
Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}, BuildTime: 1710000000,
|
||||
},
|
||||
{
|
||||
RepoName: "r", FilePath: "x86_64/bbb-2.0-r0.apk", Checksum: "Q1bbb",
|
||||
Name: "bbb", Version: "2.0-r0", Arch: "x86_64", DownloadSize: 200, InstalledSize: 20,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPKIndexDeterministic asserts APKINDEX.tar.gz is byte-identical across two
|
||||
// generations separated by wall-clock time, so the two no-affinity replicas and
|
||||
// every regeneration serve the same bytes (issue #117).
|
||||
func TestAPKIndexDeterministic(t *testing.T) {
|
||||
metas := apkFixture()
|
||||
|
||||
first := generateAPKIndex(metas)
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
second := generateAPKIndex(metas)
|
||||
|
||||
if !bytes.Equal(first, second) {
|
||||
t.Error("APKINDEX.tar.gz differs across generations")
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPKIndexTarModTimePinned guards the tar header: its ModTime must be the
|
||||
// pinned Unix epoch, never wall clock. Fails if a future edit stamps time.Now().
|
||||
func TestAPKIndexTarModTimePinned(t *testing.T) {
|
||||
metas := apkFixture()
|
||||
|
||||
zr, err := gzip.NewReader(bytes.NewReader(generateAPKIndex(metas)))
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
if !zr.ModTime.IsZero() && zr.ModTime.Unix() != 0 {
|
||||
t.Errorf("gzip header ModTime = %v, want zero/epoch", zr.ModTime)
|
||||
}
|
||||
|
||||
tarBytes, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
t.Fatalf("gunzip: %v", err)
|
||||
}
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
t.Fatalf("tar: %v", err)
|
||||
}
|
||||
if hdr.Name != "APKINDEX" {
|
||||
t.Fatalf("tar entry = %q, want APKINDEX", hdr.Name)
|
||||
}
|
||||
if hdr.ModTime.Unix() != 0 {
|
||||
t.Errorf("APKINDEX tar ModTime = %v (unix %d), want epoch (0)", hdr.ModTime, hdr.ModTime.Unix())
|
||||
}
|
||||
}
|
||||
@@ -1,324 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
)
|
||||
|
||||
type fakeBlobReader struct{ data []byte }
|
||||
|
||||
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type errBlobReader struct{}
|
||||
|
||||
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return nil, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// fakeAlpineStore satisfies provider.MetadataStore (shared) and
|
||||
// provider.AlpineMetadataStore, recording the row AfterUpload writes.
|
||||
type fakeAlpineStore struct{ inserted *provider.AlpineMetadata }
|
||||
|
||||
func (f *fakeAlpineStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeAlpineStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeAlpineStore) InsertAlpineMetadata(_ context.Context, m *provider.AlpineMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeAlpineDeleter satisfies provider.MetadataDeleter and AlpineMetadataDeleter.
|
||||
type fakeAlpineDeleter struct{ deleted bool }
|
||||
|
||||
func (f *fakeAlpineDeleter) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeAlpineDeleter) DeleteDebMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeAlpineDeleter) DeleteAlpineMetadata(context.Context, string, string) error {
|
||||
f.deleted = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// fakeAlpineReader is a FileStore that also serves alpine metadata rows.
|
||||
type fakeAlpineReader struct{ metas []provider.AlpineMetadata }
|
||||
|
||||
func (f fakeAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f fakeAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
type errAlpineReader struct{}
|
||||
|
||||
func (errAlpineReader) ListAlpineMetadataEntries(context.Context, string) ([]provider.AlpineMetadata, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
func (errAlpineReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (errAlpineReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestAlpineValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("x86_64/foo-1.0-r0.apk")
|
||||
if err != nil || sp != "x86_64/foo-1.0-r0.apk" || ct != "application/vnd.android.package-archive" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
// Dot-segment prefix is normalized away.
|
||||
if sp, _, err := p.ValidateUpload("./aarch64/bar-2.0-r1.apk"); err != nil || sp != "aarch64/bar-2.0-r1.apk" {
|
||||
t.Errorf("dot-seg: sp=%q err=%v", sp, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("foo.rpm"); err == nil {
|
||||
t.Error("expected error for non-apk")
|
||||
}
|
||||
resp := p.UploadResponse("x86_64/foo-1.0-r0.apk", "sha256:abc", 42)
|
||||
if resp["filename"] != "foo-1.0-r0.apk" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) {
|
||||
t.Errorf("upload response %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterUpload(t *testing.T) {
|
||||
data := testsupport.MinimalApk("hello", "1.0-r0", "x86_64")
|
||||
store := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "myrepo", "x86_64/hello-1.0-r0.apk",
|
||||
"sha256:deadbeef", fakeBlobReader{data: data}, store)
|
||||
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("no metadata inserted")
|
||||
}
|
||||
if m.Name != "hello" || m.Version != "1.0-r0" || m.Arch != "x86_64" {
|
||||
t.Errorf("unexpected metadata: %+v", m)
|
||||
}
|
||||
if m.DownloadSize != int64(len(data)) {
|
||||
t.Errorf("DownloadSize = %d, want %d", m.DownloadSize, len(data))
|
||||
}
|
||||
if m.InstalledSize != 4 {
|
||||
t.Errorf("InstalledSize = %d, want 4", m.InstalledSize)
|
||||
}
|
||||
if m.License != "MIT" || m.Origin != "hello" || !strings.HasPrefix(m.Maintainer, "e2e") {
|
||||
t.Errorf("scalar fields not parsed: %+v", m)
|
||||
}
|
||||
if len(m.Depends) != 1 || m.Depends[0] != "so:libc.musl-x86_64.so.1" {
|
||||
t.Errorf("Depends = %v", m.Depends)
|
||||
}
|
||||
if len(m.Provides) != 1 || m.Provides[0] != "cmd:hello=1.0-r0" {
|
||||
t.Errorf("Provides = %v", m.Provides)
|
||||
}
|
||||
|
||||
// The Q1 checksum is the sha1 of the CONTROL gzip stream (the member whose
|
||||
// tar carries .PKGINFO), not of the whole file.
|
||||
controlRaw := controlStreamBytes(t, data)
|
||||
sum := sha1.Sum(controlRaw)
|
||||
want := "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
if m.Checksum != want {
|
||||
t.Errorf("Checksum = %q, want %q (sha1 of control stream)", m.Checksum, want)
|
||||
}
|
||||
// And explicitly NOT the sha1 of the whole apk.
|
||||
whole := sha1.Sum(data)
|
||||
if m.Checksum == "Q1"+base64.StdEncoding.EncodeToString(whole[:]) {
|
||||
t.Error("Checksum was computed over the whole file, not the control stream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterUploadErrors(t *testing.T) {
|
||||
store := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", errBlobReader{}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("no metadata should be inserted on download error")
|
||||
}
|
||||
store2 := &fakeAlpineStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "x86_64/p.apk", "sha256:x", fakeBlobReader{data: []byte("not an apk")}, store2)
|
||||
if store2.inserted != nil {
|
||||
t.Error("no metadata should be inserted on parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineAfterDelete(t *testing.T) {
|
||||
d := &fakeAlpineDeleter{}
|
||||
if err := (&Provider{}).AfterDelete(context.Background(), "r", "x86_64/p.apk", d); err != nil {
|
||||
t.Fatalf("AfterDelete: %v", err)
|
||||
}
|
||||
if !d.deleted {
|
||||
t.Error("DeleteAlpineMetadata not called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10,
|
||||
Description: "pkg aaa", URL: "https://a", License: "MIT", Depends: []string{"so:libc"}, Provides: []string{"cmd:aaa"}},
|
||||
{Name: "bbb", Version: "2.0-r0", Arch: "aarch64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
|
||||
}}
|
||||
|
||||
// x86_64 index contains only aaa, with its fields, and not bbb.
|
||||
w := serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("code %d", w.Code)
|
||||
}
|
||||
idx := untarIndex(t, w.Body.Bytes())
|
||||
for _, want := range []string{"C:Q1aaa", "P:aaa", "V:1.0-r0", "A:x86_64", "S:100", "I:10", "T:pkg aaa", "U:https://a", "L:MIT", "D:so:libc", "p:cmd:aaa"} {
|
||||
if !strings.Contains(idx, want) {
|
||||
t.Errorf("x86_64 APKINDEX missing %q:\n%s", want, idx)
|
||||
}
|
||||
}
|
||||
if strings.Contains(idx, "P:bbb") {
|
||||
t.Errorf("x86_64 APKINDEX leaked aarch64 package:\n%s", idx)
|
||||
}
|
||||
|
||||
// aarch64 index contains only bbb.
|
||||
w = serveIndex(t, p, reader, "aarch64/APKINDEX.tar.gz")
|
||||
idx = untarIndex(t, w.Body.Bytes())
|
||||
if !strings.Contains(idx, "P:bbb") || strings.Contains(idx, "P:aaa") {
|
||||
t.Errorf("aarch64 filtering wrong:\n%s", idx)
|
||||
}
|
||||
|
||||
// Non-index and .apk paths are not owned by the indexer.
|
||||
for _, path := range []string{"x86_64/foo-1.0-r0.apk", "x86_64/", "README"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if p.ServeLocalIndex(w, r, reader, "repo", path) {
|
||||
t.Errorf("ServeLocalIndex should return false for %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Empty fields are omitted from the record (bbb has no description/url).
|
||||
func TestAlpineIndexOmitsEmptyFields(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "bbb", Version: "2.0-r0", Arch: "x86_64", Checksum: "Q1bbb", DownloadSize: 200, InstalledSize: 20},
|
||||
}}
|
||||
idx := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
for _, absent := range []string{"T:", "U:", "L:", "D:", "p:", "i:", "o:", "m:", "c:", "k:"} {
|
||||
if strings.Contains(idx, absent) {
|
||||
t.Errorf("empty field %q should be omitted:\n%s", absent, idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// apk requests "./<arch>/APKINDEX.tar.gz" for a bare repo base URL; the
|
||||
// dot-segment must be collapsed and yield the same bytes as the plain path.
|
||||
func TestAlpineServeLocalIndexDotSegment(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{metas: []provider.AlpineMetadata{
|
||||
{Name: "aaa", Version: "1.0-r0", Arch: "x86_64", Checksum: "Q1aaa", DownloadSize: 100, InstalledSize: 10},
|
||||
}}
|
||||
plain := untarIndex(t, serveIndex(t, p, reader, "x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
dotted := untarIndex(t, serveIndex(t, p, reader, "./x86_64/APKINDEX.tar.gz").Body.Bytes())
|
||||
if plain != dotted {
|
||||
t.Errorf("dot-segment path differs:\nplain=%q\ndotted=%q", plain, dotted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeLocalIndexArchRequired(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeAlpineReader{}
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "repo", "APKINDEX.tar.gz") {
|
||||
t.Fatal("bare APKINDEX should be owned (and rejected) by the indexer")
|
||||
}
|
||||
if w.Code != http.StatusNotFound {
|
||||
t.Errorf("bare APKINDEX code = %d, want 404", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineServeMetadataError(t *testing.T) {
|
||||
p := &Provider{}
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/x86_64/APKINDEX.tar.gz", nil)
|
||||
p.ServeLocalIndex(w, r, errAlpineReader{}, "repo", "x86_64/APKINDEX.tar.gz")
|
||||
if w.Code != 500 {
|
||||
t.Errorf("failing reader code = %d, want 500", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlpineGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeAlpineReader{}, "r", "x86_64/APKINDEX.tar.gz"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
|
||||
func serveIndex(t *testing.T, p *Provider, files provider.FileStore, path string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, files, "repo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// untarIndex un-gzips and un-tars an APKINDEX.tar.gz and returns the APKINDEX text.
|
||||
func untarIndex(t *testing.T, gzTar []byte) string {
|
||||
t.Helper()
|
||||
zr, err := gzip.NewReader(bytes.NewReader(gzTar))
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not gzip: %v", err)
|
||||
}
|
||||
tarBytes, _ := io.ReadAll(zr)
|
||||
tr := tar.NewReader(bytes.NewReader(tarBytes))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("APKINDEX not tar: %v", err)
|
||||
}
|
||||
if hdr.Name == "APKINDEX" {
|
||||
b, _ := io.ReadAll(tr)
|
||||
return string(b)
|
||||
}
|
||||
}
|
||||
t.Fatal("no APKINDEX member in tarball")
|
||||
return ""
|
||||
}
|
||||
|
||||
// controlStreamBytes returns the raw bytes of the gzip stream whose tar carries
|
||||
// .PKGINFO, so the test can independently compute the expected Q1 checksum.
|
||||
func controlStreamBytes(t *testing.T, apk []byte) []byte {
|
||||
t.Helper()
|
||||
br := bytes.NewReader(apk)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, _ := io.ReadAll(zr)
|
||||
end := len(apk) - br.Len()
|
||||
tr := tar.NewReader(bytes.NewReader(out))
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if strings.TrimPrefix(h.Name, "./") == ".PKGINFO" {
|
||||
return apk[prev:end]
|
||||
}
|
||||
}
|
||||
prev = end
|
||||
if err := zr.Reset(br); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
t.Fatal("no control stream found")
|
||||
return nil
|
||||
}
|
||||
@@ -1,714 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// gitHubProvider is the process-wide singleton for github_alpine. The background
|
||||
// Syncer binds its shared rate limiter and work queue onto this instance so the
|
||||
// request path and the syncer drive the same derive machinery.
|
||||
var gitHubProvider = newGitHubProvider()
|
||||
|
||||
func init() {
|
||||
provider.Register(gitHubProvider)
|
||||
}
|
||||
|
||||
// Tuning knobs for the no-precache control fetch. An .apk is up to three
|
||||
// concatenated gzip streams (optional signature, control, data); the control
|
||||
// stream carrying .PKGINFO sits near the front, so a small prefix reliably
|
||||
// covers it.
|
||||
const (
|
||||
defaultHeaderRangeInitial = 32 << 10 // 32 KiB — covers the control stream of almost every .apk
|
||||
defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset
|
||||
defaultReleasePageCap = 10 // 100 releases/page * 10 pages
|
||||
|
||||
defaultScanTimeout = 10 * time.Minute
|
||||
defaultServeTimeout = 30 * time.Second
|
||||
defaultColdWait = 8 * time.Second
|
||||
)
|
||||
|
||||
// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases
|
||||
// for .apk assets, derives per-asset .PKGINFO metadata via a ranged prefix fetch
|
||||
// (never downloading whole packages), synthesizes a per-arch APKINDEX from that
|
||||
// cached metadata, and redirects package downloads to a backend "releases_remote"
|
||||
// (the generic github.com remote) that serves the actual bytes.
|
||||
type GitHubProvider struct {
|
||||
client *http.Client
|
||||
|
||||
headerInitial int64
|
||||
headerMax int64
|
||||
pageCap int
|
||||
scanTimeout time.Duration
|
||||
serveTimeout time.Duration
|
||||
coldWait time.Duration
|
||||
|
||||
limiter *rate.Limiter
|
||||
syncer *Syncer
|
||||
|
||||
serverCred githubauth.Credential
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool
|
||||
lastScan map[string]time.Time
|
||||
}
|
||||
|
||||
func newGitHubProvider() *GitHubProvider {
|
||||
return &GitHubProvider{
|
||||
client: &http.Client{},
|
||||
headerInitial: defaultHeaderRangeInitial,
|
||||
headerMax: defaultHeaderRangeMax,
|
||||
pageCap: defaultReleasePageCap,
|
||||
scanTimeout: defaultScanTimeout,
|
||||
serveTimeout: defaultServeTimeout,
|
||||
coldWait: defaultColdWait,
|
||||
scanning: map[string]bool{},
|
||||
lastScan: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) limiterWait(ctx context.Context) error {
|
||||
if p.limiter == nil {
|
||||
return nil
|
||||
}
|
||||
return p.limiter.Wait(ctx)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) Type() models.PackageType { return models.PackageGitHubAlpine }
|
||||
|
||||
func (p *GitHubProvider) Classify(path string) provider.Mutability {
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".apk"):
|
||||
return "application/vnd.android.package-archive"
|
||||
case strings.HasSuffix(path, ".tar.gz"):
|
||||
return "application/gzip"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) {
|
||||
return p.githubHeaders(ctx, remote, false)
|
||||
}
|
||||
|
||||
// ServeRemote answers a request against a github_alpine remote. It refreshes the
|
||||
// derived metadata (bounded by mutable_ttl), serves a synthesized per-arch
|
||||
// APKINDEX.tar.gz, and 302-redirects .apk downloads to the backend
|
||||
// releases_remote. Returns false only for paths it does not own.
|
||||
func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, reqPath, proxyBaseURL string, store provider.RemoteMetadataStore) bool {
|
||||
p.onRequest(remote, store)
|
||||
|
||||
// apk requests the index at "./<arch>/APKINDEX.tar.gz"; collapse the
|
||||
// dot-segment before matching, mirroring the local indexer.
|
||||
path := normalizeIndexPath(reqPath)
|
||||
|
||||
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
|
||||
p.serveIndex(w, r, remote, path, store)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".apk") {
|
||||
if remote.ReleasesRemote == "" {
|
||||
http.Error(w, "github_alpine remote has no releases_remote configured for downloads", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
p.serveApkRedirect(w, r, remote, path, proxyBaseURL, store)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// serveApkRedirect resolves an apk-reconstructed download path — apk builds
|
||||
// "<arch>/<name>-<version>.apk" itself because APKINDEX carries no filename — to
|
||||
// the real github-relative asset path stored on the metadata row, then redirects
|
||||
// to the backend releases_remote. Passing the inbound path through verbatim would
|
||||
// point at a nonexistent, allowlist-denied github.com path.
|
||||
func (p *GitHubProvider) serveApkRedirect(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store provider.RemoteMetadataStore) {
|
||||
arch := strings.TrimSuffix(path[:strings.LastIndex(path, "/")+1], "/")
|
||||
basename := path[strings.LastIndex(path, "/")+1:]
|
||||
if arch == "" || strings.Contains(arch, "/") {
|
||||
http.Error(w, "apk download must be requested per-arch: <arch>/<name>-<version>.apk", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
rows, err := reader.ListAlpineMetadataEntries(sctx, remote.Name)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
for _, row := range rows {
|
||||
if row.Arch == arch && row.Name+"-"+row.Version+".apk" == basename {
|
||||
loc := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.ReleasesRemote + "/" + strings.TrimLeft(row.FilePath, "/")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return
|
||||
}
|
||||
}
|
||||
http.Error(w, "package not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serveIndex(w http.ResponseWriter, r *http.Request, remote models.Remote, path string, store provider.RemoteMetadataStore) {
|
||||
arch := strings.TrimSuffix(path, "APKINDEX.tar.gz")
|
||||
arch = strings.Trim(arch, "/")
|
||||
if arch == "" || strings.Contains(arch, "/") {
|
||||
http.Error(w, "APKINDEX must be requested per-arch: <arch>/APKINDEX.tar.gz", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Serve on a context detached from the inbound request so a client disconnect
|
||||
// never cancels the metadata DB read and surfaces as a 500.
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
|
||||
if p.syncer != nil && !p.ensurePrimed(sctx, remote, store) {
|
||||
w.Header().Set("Retry-After", "5")
|
||||
http.Error(w, "metadata is being prepared, retry shortly", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "alpine metadata not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
metas, err := reader.ListAlpineMetadataEntries(sctx, remote.Name)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
var filtered []provider.AlpineMetadata
|
||||
for _, m := range metas {
|
||||
if m.Arch == arch {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateAPKIndex(filtered))
|
||||
}
|
||||
|
||||
// onRequest keeps a remote's derived metadata fresh off the request path.
|
||||
func (p *GitHubProvider) onRequest(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, false)
|
||||
return
|
||||
}
|
||||
p.refresh(remote, store)
|
||||
}
|
||||
|
||||
// ensurePrimed returns true once the remote has at least one cached row. On an
|
||||
// empty cache it enqueues a prime and polls briefly for it to land.
|
||||
func (p *GitHubProvider) ensurePrimed(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) bool {
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, true)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(p.coldWait)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(400 * time.Millisecond):
|
||||
}
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) cacheEmpty(ctx context.Context, store provider.RemoteMetadataStore, name string) bool {
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
rows, err := reader.ListAlpineMetadataEntries(ctx, name)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return len(rows) == 0
|
||||
}
|
||||
|
||||
// refresh brings the derived metadata up to date without coupling the scan to
|
||||
// the inbound request (legacy inline path used without a syncer / in unit tests).
|
||||
func (p *GitHubProvider) refresh(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
ttl := time.Duration(remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 5 * time.Minute
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
last, ok := p.lastScan[remote.Name]
|
||||
fresh := ok && time.Since(last) < ttl
|
||||
if fresh || p.scanning[remote.Name] {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.scanning[remote.Name] = true
|
||||
p.mu.Unlock()
|
||||
|
||||
if p.cacheEmpty(context.Background(), store, remote.Name) {
|
||||
p.runScan(remote, store)
|
||||
return
|
||||
}
|
||||
go p.runScan(remote, store)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) runScan(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.scanning, remote.Name)
|
||||
p.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), p.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := p.scan(ctx, remote, store); err != nil {
|
||||
slog.Error("github_alpine: release scan failed", "remote", remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// scan runs a full unconditional derive. Retained for the legacy inline refresh
|
||||
// path and existing tests; the syncer uses scanWithState.
|
||||
func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error {
|
||||
_, _, err := p.scanWithState(ctx, remote, store, "")
|
||||
return err
|
||||
}
|
||||
|
||||
// scanWithState derives metadata incrementally. It sends the prior releases-list
|
||||
// ETag as a conditional request: a 304 means nothing changed. On a 200 it diffs
|
||||
// the release assets against the cache, derives only new/changed assets, prunes
|
||||
// assets that disappeared, and returns the new ETag.
|
||||
func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore, etag string) (newEtag string, changed bool, err error) {
|
||||
inserter, ok := store.(provider.AlpineMetadataStore)
|
||||
if !ok {
|
||||
return etag, false, errors.New("store does not support alpine metadata writes")
|
||||
}
|
||||
deleter, ok := store.(provider.AlpineMetadataDeleter)
|
||||
if !ok {
|
||||
return etag, false, errors.New("store does not support alpine metadata deletes")
|
||||
}
|
||||
reader, ok := store.(provider.AlpineMetadataReader)
|
||||
if !ok {
|
||||
return etag, false, errors.New("store does not support alpine metadata reads")
|
||||
}
|
||||
|
||||
releases, newEtag, notModified, err := p.fetchReleases(ctx, remote, etag)
|
||||
if err != nil {
|
||||
return etag, false, err
|
||||
}
|
||||
if notModified {
|
||||
return etag, false, nil
|
||||
}
|
||||
|
||||
existing, err := reader.ListAlpineMetadataEntries(ctx, remote.Name)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
existingByPath := make(map[string]provider.AlpineMetadata, len(existing))
|
||||
for _, m := range existing {
|
||||
existingByPath[m.FilePath] = m
|
||||
}
|
||||
|
||||
allow, err := compilePatterns(remote.Patterns)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
continue
|
||||
}
|
||||
for _, asset := range rel.Assets {
|
||||
if !strings.HasSuffix(strings.ToLower(asset.Name), ".apk") {
|
||||
continue
|
||||
}
|
||||
if !matchesAny(allow, asset.Name) {
|
||||
continue
|
||||
}
|
||||
fp := assetPath(asset)
|
||||
if fp == "" {
|
||||
continue
|
||||
}
|
||||
seen[fp] = true
|
||||
|
||||
if cur, ok := existingByPath[fp]; ok {
|
||||
if asset.Digest == "" || cur.ContentHash == asset.Digest {
|
||||
continue
|
||||
}
|
||||
_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_alpine: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
slog.Info("github_alpine: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
|
||||
}
|
||||
}
|
||||
|
||||
for fp := range existingByPath {
|
||||
if !seen[fp] {
|
||||
_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return newEtag, true, nil
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Draft bool `json:"draft"`
|
||||
Assets []ghAsset `json:"assets"`
|
||||
}
|
||||
|
||||
type ghAsset struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
BrowserDownloadURL string `json:"browser_download_url"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// fetchReleases lists a repo's releases, sending the prior ETag as If-None-Match
|
||||
// on page 1 so an unchanged repo short-circuits to notModified. Every call waits
|
||||
// on the shared limiter first.
|
||||
func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote, etag string) (all []ghRelease, newEtag string, notModified bool, err error) {
|
||||
base := strings.TrimRight(remote.BaseURL, "/") + "/releases"
|
||||
for page := 1; page <= p.pageCap; page++ {
|
||||
u := fmt.Sprintf("%s?per_page=100&page=%d", base, page)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
if page == 1 && etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if page == 1 && resp.StatusCode == http.StatusNotModified {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, etag, true, nil
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
respEtag := resp.Header.Get("ETag")
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
|
||||
}
|
||||
if page == 1 {
|
||||
newEtag = respEtag
|
||||
}
|
||||
var releases []ghRelease
|
||||
if err := json.Unmarshal(body, &releases); err != nil {
|
||||
return nil, "", false, fmt.Errorf("decode releases: %w", err)
|
||||
}
|
||||
if len(releases) == 0 {
|
||||
break
|
||||
}
|
||||
all = append(all, releases...)
|
||||
if len(releases) < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, newEtag, false, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote, asset ghAsset, fp string) (*provider.AlpineMetadata, error) {
|
||||
meta, err := p.fetchPkginfo(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if meta.Name == "" || meta.Arch == "" {
|
||||
return nil, errors.New(".PKGINFO missing pkgname/arch")
|
||||
}
|
||||
|
||||
meta.RepoName = remote.Name
|
||||
meta.FilePath = fp
|
||||
// S: the on-disk .apk size comes straight from the releases API, so we never
|
||||
// download the body just to size it.
|
||||
meta.DownloadSize = asset.Size
|
||||
// ContentHash records the GitHub asset digest (when present) purely so the
|
||||
// next scan can detect a changed asset; unlike deb it is not the index
|
||||
// checksum (that is the Q1 control-stream sum already set in fetchPkginfo).
|
||||
if asset.Digest != "" {
|
||||
meta.ContentHash = asset.Digest
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// fetchPkginfo pulls only the front of the .apk with a ranged GET and derives the
|
||||
// .PKGINFO fields plus the apk pull checksum (C: = Q1 + base64(sha1(control gzip
|
||||
// stream))). The control stream sits near the front, so a small prefix suffices;
|
||||
// a prefix that truncates it doubles the range and retries.
|
||||
func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote, downloadURL string) (*provider.AlpineMetadata, error) {
|
||||
n := p.headerInitial
|
||||
for {
|
||||
body, full, err := p.rangeGet(ctx, remote, downloadURL, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, complete, perr := pkginfoFromPrefix(body)
|
||||
if perr != nil {
|
||||
return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
|
||||
}
|
||||
if complete {
|
||||
return meta, nil
|
||||
}
|
||||
if full || n >= p.headerMax {
|
||||
return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
|
||||
}
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
n = p.headerMax
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pkginfoFromPrefix parses the concatenated gzip streams present in a front
|
||||
// prefix of an .apk. It walks each fully-covered gzip member until it finds the
|
||||
// control stream (the one whose tar carries .PKGINFO), computes the Q1 pull
|
||||
// checksum from that stream's raw bytes, and reads the .PKGINFO fields. A prefix
|
||||
// too short to fully cover the control stream returns complete=false so the
|
||||
// caller can widen the range.
|
||||
func pkginfoFromPrefix(prefix []byte) (meta *provider.AlpineMetadata, complete bool, err error) {
|
||||
br := bytes.NewReader(prefix)
|
||||
zr, zerr := gzip.NewReader(br)
|
||||
if zerr != nil {
|
||||
if zerr == io.EOF || zerr == io.ErrUnexpectedEOF {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, zerr
|
||||
}
|
||||
prev := 0
|
||||
for {
|
||||
zr.Multistream(false)
|
||||
out, rerr := io.ReadAll(zr)
|
||||
if rerr != nil {
|
||||
// A member truncated by the range boundary is not an error — widen.
|
||||
if rerr == io.ErrUnexpectedEOF || rerr == io.EOF {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, rerr
|
||||
}
|
||||
end := len(prefix) - br.Len()
|
||||
raw := prefix[prev:end]
|
||||
|
||||
if pkginfo, ok := pkginfoFromTar(out); ok {
|
||||
m := parsePkginfo(pkginfo)
|
||||
sum := sha1.Sum(raw)
|
||||
m.Checksum = "Q1" + base64.StdEncoding.EncodeToString(sum[:])
|
||||
return m, true, nil
|
||||
}
|
||||
|
||||
prev = end
|
||||
if rsterr := zr.Reset(br); rsterr != nil {
|
||||
if rsterr == io.EOF {
|
||||
// No more complete members in the prefix; the control stream is
|
||||
// either not covered yet or genuinely absent — let the caller
|
||||
// decide by widening (or hitting the full-object guard).
|
||||
return nil, false, nil
|
||||
}
|
||||
if rsterr == io.ErrUnexpectedEOF {
|
||||
return nil, false, nil
|
||||
}
|
||||
return nil, false, rsterr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, false)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1))
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
}
|
||||
|
||||
// assetPath is the package's location relative to github.com — the path the
|
||||
// backend releases_remote (base https://github.com) proxies. It doubles as the
|
||||
// alpine_metadata key and the redirect target, so an .apk download resolves back
|
||||
// to this remote and redirects to the backend.
|
||||
func assetPath(asset ghAsset) string {
|
||||
u, err := url.Parse(asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(u.Path, "/")
|
||||
}
|
||||
|
||||
// githubHeaders builds the outbound headers for a GitHub request, attaching a
|
||||
// bearer credential when one is available. A per-remote credential wins; absent
|
||||
// that, the process-wide server credential is used; absent both, the request is
|
||||
// unauthenticated.
|
||||
func (p *GitHubProvider) githubHeaders(ctx context.Context, remote models.Remote, api bool) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if api {
|
||||
h.Set("Accept", "application/vnd.github+json")
|
||||
h.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
}
|
||||
tok, err := p.githubToken(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tok != "" {
|
||||
h.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// githubToken resolves the bearer token for a remote. Precedence: a per-remote
|
||||
// credential (password, then username) overrides the server credential.
|
||||
func (p *GitHubProvider) githubToken(ctx context.Context, remote models.Remote) (string, error) {
|
||||
if remote.Password != "" {
|
||||
return remote.Password, nil
|
||||
}
|
||||
if remote.Username != "" {
|
||||
return remote.Username, nil
|
||||
}
|
||||
if c := p.serverCredential(); c != nil {
|
||||
return c.Token(ctx)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serverCredential() githubauth.Credential {
|
||||
if p.serverCred != nil {
|
||||
return p.serverCred
|
||||
}
|
||||
return githubauth.Server()
|
||||
}
|
||||
|
||||
func copyHeaders(req *http.Request, h http.Header) {
|
||||
for k, vals := range h {
|
||||
for _, v := range vals {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func compilePatterns(patterns []string) ([]*regexp.Regexp, error) {
|
||||
var out []*regexp.Regexp
|
||||
for _, p := range patterns {
|
||||
re, err := regexp.Compile(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
|
||||
}
|
||||
out = append(out, re)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func matchesAny(res []*regexp.Regexp, s string) bool {
|
||||
if len(res) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, re := range res {
|
||||
if re.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,497 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeStore is an in-memory provider.RemoteMetadataStore + AlpineMetadata
|
||||
// store/reader/deleter keyed by file_path, mirroring the (repo_name, file_path)
|
||||
// uniqueness of the real alpine_metadata table.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]provider.AlpineMetadata
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.AlpineMetadata{}} }
|
||||
|
||||
func (f *fakeStore) InsertAlpineMetadata(_ context.Context, m *provider.AlpineMetadata) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.rows[m.FilePath]; ok {
|
||||
return nil // ON CONFLICT DO NOTHING
|
||||
}
|
||||
f.rows[m.FilePath] = *m
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) DeleteAlpineMetadata(_ context.Context, _, filePath string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.rows, filePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListAlpineMetadataEntries(ctx context.Context, _ string) ([]provider.AlpineMetadata, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]provider.AlpineMetadata, 0, len(f.rows))
|
||||
for _, m := range f.rows {
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// The generic RemoteMetadataStore surface (rpm/deb) is unused by the alpine
|
||||
// github provider but required to satisfy the interface passed to ServeRemote.
|
||||
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeStore) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeStore) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeStore) DeleteDebMetadata(context.Context, string, string) error { return nil }
|
||||
|
||||
var _ provider.RemoteMetadataStore = (*fakeStore)(nil)
|
||||
|
||||
// githubFixture serves the releases API and the .apk asset downloads (with Range
|
||||
// support) for a set of packages. digest controls whether the asset carries a
|
||||
// sha256 digest (change-detection path) or not.
|
||||
type githubFixture struct {
|
||||
srv *httptest.Server
|
||||
apkBytes map[string][]byte
|
||||
rangeHit map[string]int
|
||||
fullHit map[string]int
|
||||
etag string
|
||||
releasesHit int
|
||||
notModHit int
|
||||
releaseAuth string
|
||||
assetAuth string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
apkBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
}
|
||||
f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
|
||||
page := r.URL.Query().Get("page")
|
||||
if page != "" && page != "1" {
|
||||
w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.releasesHit++
|
||||
f.releaseAuth = r.Header.Get("Authorization")
|
||||
etag := f.etag
|
||||
if etag != "" && r.Header.Get("If-None-Match") == etag {
|
||||
f.notModHit++
|
||||
f.mu.Unlock()
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if etag != "" {
|
||||
w.Header().Set("ETag", etag)
|
||||
}
|
||||
var assets []map[string]any
|
||||
for name := range f.apkBytes {
|
||||
a := map[string]any{
|
||||
"name": name,
|
||||
"size": len(f.apkBytes[name]),
|
||||
"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2.3/" + name,
|
||||
}
|
||||
if withDigest {
|
||||
sum := sha256.Sum256(f.apkBytes[name])
|
||||
a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
assets = append(assets, a)
|
||||
}
|
||||
rel := []map[string]any{{"tag_name": "v1.2.3", "draft": false, "assets": assets}}
|
||||
json.NewEncoder(w).Encode(rel)
|
||||
})
|
||||
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
body, ok := f.apkBytes[name]
|
||||
if !ok {
|
||||
http.Error(w, "not found", 404)
|
||||
return
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
} else {
|
||||
f.fullHit[name]++
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
if rng == "" {
|
||||
w.WriteHeader(200)
|
||||
w.Write(body)
|
||||
return
|
||||
}
|
||||
var end int
|
||||
fmt.Sscanf(rng, "bytes=0-%d", &end)
|
||||
if end >= len(body)-1 {
|
||||
end = len(body) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(end+1))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(body[:end+1])
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *githubFixture) remote() models.Remote {
|
||||
return models.Remote{
|
||||
Name: "acme-apk",
|
||||
PackageType: models.PackageGitHubAlpine,
|
||||
BaseURL: f.srv.URL + "/repos/acme/tools",
|
||||
ReleasesRemote: "github",
|
||||
MutableTTL: 3600,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestProvider() *GitHubProvider {
|
||||
p := newGitHubProvider()
|
||||
p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
|
||||
p.headerMax = 1 << 20
|
||||
return p
|
||||
}
|
||||
|
||||
const demoPath = "acme/tools/releases/download/v1.2.3/demo-1.2.3-r0.apk"
|
||||
|
||||
func TestGitHubScanDerivesPkginfoFromPrefix(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
metas, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 metadata row, got %d", len(metas))
|
||||
}
|
||||
m := metas[0]
|
||||
if m.Name != "demo" || m.Version != "1.2.3-r0" || m.Arch != "x86_64" {
|
||||
t.Fatalf("bad .PKGINFO fields: %+v", m)
|
||||
}
|
||||
if m.FilePath != demoPath {
|
||||
t.Fatalf("FilePath = %q, want %q", m.FilePath, demoPath)
|
||||
}
|
||||
if int(m.DownloadSize) != len(fx.apkBytes["demo-1.2.3-r0.apk"]) {
|
||||
t.Fatalf("DownloadSize = %d, want %d", m.DownloadSize, len(fx.apkBytes["demo-1.2.3-r0.apk"]))
|
||||
}
|
||||
if !strings.HasPrefix(m.Checksum, "Q1") {
|
||||
t.Fatalf("Checksum not a Q1 pull checksum: %q", m.Checksum)
|
||||
}
|
||||
// The C: checksum must equal Q1 over the raw control gzip stream, matching the
|
||||
// local-upload parser applied to the same bytes.
|
||||
want, err := parseApk(fx.apkBytes["demo-1.2.3-r0.apk"])
|
||||
if err != nil {
|
||||
t.Fatalf("reference parseApk: %v", err)
|
||||
}
|
||||
if m.Checksum != want.Checksum {
|
||||
t.Fatalf("Checksum = %q, want %q (Q1 of control stream)", m.Checksum, want.Checksum)
|
||||
}
|
||||
if fx.fullHit["demo-1.2.3-r0.apk"] != 0 {
|
||||
t.Fatalf("expected no full download, got %d", fx.fullHit["demo-1.2.3-r0.apk"])
|
||||
}
|
||||
if fx.rangeHit["demo-1.2.3-r0.apk"] == 0 {
|
||||
t.Fatalf("expected ranged .PKGINFO fetch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteIndexAndRedirect(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
// The per-arch index is served and triggers the initial scan.
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("APKINDEX bad: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
idx := readAPKIndex(t, rec.Body.Bytes())
|
||||
if !strings.Contains(idx, "P:demo") || !strings.Contains(idx, "A:x86_64") {
|
||||
t.Fatalf("APKINDEX missing package record: %s", idx)
|
||||
}
|
||||
if !strings.Contains(idx, "C:Q1") {
|
||||
t.Fatalf("APKINDEX missing pull checksum: %s", idx)
|
||||
}
|
||||
|
||||
// A different arch yields an empty (but valid) index.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "aarch64/APKINDEX.tar.gz", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle aarch64 APKINDEX")
|
||||
}
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("empty-arch index bad: %d", rec.Code)
|
||||
}
|
||||
if got := readAPKIndex(t, rec.Body.Bytes()); strings.Contains(got, "P:demo") {
|
||||
t.Fatalf("aarch64 index should not carry the x86_64 package: %s", got)
|
||||
}
|
||||
|
||||
// An .apk request arrives in apk's reconstructed shape
|
||||
// "<arch>/<name>-<version>.apk" (APKINDEX carries no filename), NOT as the
|
||||
// github-relative FilePath. ServeRemote must resolve it back to the stored
|
||||
// FilePath before redirecting to the backend releases_remote.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/demo-1.2.3-r0.apk", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/demo-1.2.3-r0.apk", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle .apk")
|
||||
}
|
||||
if rec.Code != http.StatusFound {
|
||||
t.Fatalf("want 302, got %d", rec.Code)
|
||||
}
|
||||
wantLoc := proxyBase + "/api/v1/remote/github/" + demoPath
|
||||
if got := rec.Header().Get("Location"); got != wantLoc {
|
||||
t.Fatalf("Location = %q, want %q (must be the stored FilePath, not the inbound path)", got, wantLoc)
|
||||
}
|
||||
}
|
||||
|
||||
// An apk download whose reconstructed "<arch>/<name>-<version>.apk" matches no
|
||||
// cached row must 404, never redirect to a bad path.
|
||||
func TestGitHubServeRemoteApkRedirectNotFound(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
// Warm the cache so the store is populated but lacks the requested package.
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/nope-9.9.9.apk", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/nope-9.9.9.apk", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle .apk")
|
||||
}
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("want 404 for unknown package, got %d (Location=%q)", rec.Code, rec.Header().Get("Location"))
|
||||
}
|
||||
}
|
||||
|
||||
// apk requests the index at "./<arch>/APKINDEX.tar.gz"; ServeRemote must collapse
|
||||
// the dot-segment and synthesize the same index as the un-prefixed request.
|
||||
func TestGitHubServeRemoteApkDotSegment(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/"+path, nil)
|
||||
if !p.ServeRemote(rec, req, remote, path, proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %q", path)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
plain, dotted := serve("x86_64/APKINDEX.tar.gz"), serve("./x86_64/APKINDEX.tar.gz")
|
||||
if plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("index: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
}
|
||||
if !bytes.Equal(plain.Body.Bytes(), dotted.Body.Bytes()) {
|
||||
t.Error("./<arch>/APKINDEX.tar.gz body differs from the un-prefixed body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRejectsNonPerArchIndex(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, fx.remote(), "APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("expected handled")
|
||||
}
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("bare APKINDEX must 404 (per-arch required), got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A canceled inbound request must still serve the warm cache (detached context),
|
||||
// not turn the metadata read into a 500.
|
||||
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil).WithContext(ctx)
|
||||
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := readAPKIndex(t, rec.Body.Bytes()); !strings.Contains(got, "P:demo") {
|
||||
t.Fatalf("expected index served from cache, got %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.ReleasesRemote = ""
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, demoPath, "https://x", store) {
|
||||
t.Fatal("expected handled")
|
||||
}
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk"); len(rows) != 1 {
|
||||
t.Fatalf("want 1 row after first scan, got %d", len(rows))
|
||||
}
|
||||
|
||||
delete(fx.apkBytes, "demo-1.2.3-r0.apk")
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("rescan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk"); len(rows) != 0 {
|
||||
t.Fatalf("want 0 rows after prune, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubAssetPatternFilter(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.Patterns = []string{`^demo-.*\.apk$`}
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
rows, _ := store.ListAlpineMetadataEntries(context.Background(), "acme-apk")
|
||||
if len(rows) != 1 || rows[0].Name != "demo" {
|
||||
t.Fatalf("pattern filter failed, rows=%+v", rows)
|
||||
}
|
||||
}
|
||||
|
||||
// Multi-arch: each asset's index record lands under its own arch bucket.
|
||||
func TestGitHubServeRemotePerArchGrouping(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.apkBytes["demo-1.2.3-r0-aarch64.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "aarch64")
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://x"
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
serve := func(arch string) string {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, arch+"/APKINDEX.tar.gz", proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %s", arch)
|
||||
}
|
||||
return readAPKIndex(t, rec.Body.Bytes())
|
||||
}
|
||||
|
||||
x86 := serve("x86_64")
|
||||
if !strings.Contains(x86, "A:x86_64") || strings.Contains(x86, "A:aarch64") {
|
||||
t.Fatalf("x86_64 index leaked another arch: %s", x86)
|
||||
}
|
||||
arm := serve("aarch64")
|
||||
if !strings.Contains(arm, "A:aarch64") || strings.Contains(arm, "A:x86_64") {
|
||||
t.Fatalf("aarch64 index leaked another arch: %s", arm)
|
||||
}
|
||||
}
|
||||
|
||||
func readAPKIndex(t *testing.T, gzBytes []byte) string {
|
||||
t.Helper()
|
||||
gz, err := gzip.NewReader(bytes.NewReader(gzBytes))
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
tr := tar.NewReader(gz)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err != nil {
|
||||
t.Fatal("APKINDEX member missing from tar.gz")
|
||||
}
|
||||
if strings.TrimPrefix(hdr.Name, "./") == "APKINDEX" {
|
||||
body, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
t.Fatalf("read APKINDEX: %v", err)
|
||||
}
|
||||
return string(body)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
syncLeaseDuration = 15 * time.Minute
|
||||
defaultSyncFreshness = 5 * time.Minute
|
||||
jobQueueDepth = 256
|
||||
)
|
||||
|
||||
// SyncStore is the persistence surface the alpine syncer needs: the metadata
|
||||
// cache it primes plus the shared sync-state coordination (remote enumeration
|
||||
// and the per-remote lease). *database.DB satisfies it.
|
||||
type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
|
||||
}
|
||||
|
||||
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
|
||||
type SyncConfig struct {
|
||||
RatePerSec float64
|
||||
Burst int
|
||||
Workers int
|
||||
PollInterval time.Duration
|
||||
}
|
||||
|
||||
type syncJob struct {
|
||||
remote models.Remote
|
||||
prime bool
|
||||
}
|
||||
|
||||
// Syncer is the single per-process background worker that keeps every
|
||||
// github_alpine remote's derived metadata fresh. It owns a deduped work queue, a
|
||||
// pool of workers, and a global token-bucket rate limiter shared across all
|
||||
// remotes and bound onto the github_alpine provider. Periodic checks are gated by
|
||||
// a shared DB lease so, across replicas, only one performs each scan.
|
||||
type Syncer struct {
|
||||
store SyncStore
|
||||
prov *GitHubProvider
|
||||
limiter *rate.Limiter
|
||||
cfg SyncConfig
|
||||
owner string
|
||||
|
||||
jobs chan syncJob
|
||||
mu sync.Mutex
|
||||
active map[string]bool
|
||||
}
|
||||
|
||||
// NewSyncer builds the syncer bound to the process-wide github_alpine provider
|
||||
// singleton. Call Run to start it.
|
||||
func NewSyncer(store SyncStore, cfg SyncConfig) *Syncer {
|
||||
return newSyncer(store, gitHubProvider, cfg)
|
||||
}
|
||||
|
||||
func newSyncer(store SyncStore, prov *GitHubProvider, cfg SyncConfig) *Syncer {
|
||||
if cfg.RatePerSec <= 0 {
|
||||
cfg.RatePerSec = 1
|
||||
}
|
||||
if cfg.Burst <= 0 {
|
||||
cfg.Burst = 5
|
||||
}
|
||||
if cfg.Workers <= 0 {
|
||||
cfg.Workers = 3
|
||||
}
|
||||
if cfg.PollInterval <= 0 {
|
||||
cfg.PollInterval = 60 * time.Second
|
||||
}
|
||||
|
||||
lim := rate.NewLimiter(rate.Limit(cfg.RatePerSec), cfg.Burst)
|
||||
s := &Syncer{
|
||||
store: store,
|
||||
prov: prov,
|
||||
limiter: lim,
|
||||
cfg: cfg,
|
||||
owner: leaseOwner(),
|
||||
jobs: make(chan syncJob, jobQueueDepth),
|
||||
active: map[string]bool{},
|
||||
}
|
||||
prov.limiter = lim
|
||||
prov.syncer = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Run starts the worker pool and the periodic scheduler and blocks until ctx is
|
||||
// canceled, at which point it drains in-flight scans and returns.
|
||||
func (s *Syncer) Run(ctx context.Context) {
|
||||
slog.Info("github_alpine syncer started",
|
||||
"rate_per_sec", s.cfg.RatePerSec, "burst", s.cfg.Burst,
|
||||
"workers", s.cfg.Workers, "poll_interval", s.cfg.PollInterval, "owner", s.owner)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < s.cfg.Workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.worker(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.cfg.PollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.schedule(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
wg.Wait()
|
||||
slog.Info("github_alpine syncer stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.schedule(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// schedule enqueues a periodic check for every github_alpine remote. The DB lease
|
||||
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
|
||||
func (s *Syncer) schedule(ctx context.Context) {
|
||||
remotes, err := s.store.ListGitHubAlpineRemotes(ctx)
|
||||
if err != nil {
|
||||
slog.Error("github_alpine syncer: list remotes", "error", err)
|
||||
return
|
||||
}
|
||||
for _, r := range remotes {
|
||||
s.enqueue(r, false)
|
||||
}
|
||||
}
|
||||
|
||||
// EnqueuePrime queues an immediate background prime for a freshly created remote.
|
||||
func (s *Syncer) EnqueuePrime(remote models.Remote) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.enqueue(remote, true)
|
||||
}
|
||||
|
||||
// enqueue adds a job unless the remote is already queued or in-flight, coalescing
|
||||
// duplicate requests down to one scan. It never blocks.
|
||||
func (s *Syncer) enqueue(remote models.Remote, prime bool) {
|
||||
s.mu.Lock()
|
||||
if s.active[remote.Name] {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.active[remote.Name] = true
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case s.jobs <- syncJob{remote: remote, prime: prime}:
|
||||
default:
|
||||
s.mu.Lock()
|
||||
delete(s.active, remote.Name)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Syncer) worker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case job := <-s.jobs:
|
||||
s.process(ctx, job)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// process claims the shared lease and, if won, runs an incremental scan. Losing
|
||||
// the claim (another replica scanning, or not yet due) is a no-op.
|
||||
func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
delete(s.active, job.remote.Name)
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
freshness := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if freshness <= 0 {
|
||||
freshness = defaultSyncFreshness
|
||||
}
|
||||
if job.prime {
|
||||
freshness = 0
|
||||
}
|
||||
|
||||
claimed, etag, err := s.store.ClaimGitHubAlpineSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
|
||||
if err != nil {
|
||||
slog.Error("github_alpine syncer: claim lease", "remote", job.remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
if !claimed {
|
||||
return
|
||||
}
|
||||
|
||||
scanCtx, cancel := context.WithTimeout(ctx, s.prov.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
|
||||
releaseEtag := etag
|
||||
if scanErr == nil {
|
||||
releaseEtag = newEtag
|
||||
} else {
|
||||
slog.Error("github_alpine syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
|
||||
}
|
||||
|
||||
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer relCancel()
|
||||
if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
|
||||
slog.Warn("github_alpine syncer: release lease", "remote", job.remote.Name, "error", err)
|
||||
}
|
||||
|
||||
if scanErr == nil && changed {
|
||||
slog.Info("github_alpine syncer: refreshed", "remote", job.remote.Name, "prime", job.prime)
|
||||
}
|
||||
}
|
||||
|
||||
// leaseOwner is a per-replica identity for the lease: hostname plus a random
|
||||
// suffix so restarts and colocated replicas never collide.
|
||||
func leaseOwner() string {
|
||||
host, _ := os.Hostname()
|
||||
var b [6]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return host + "-" + hex.EncodeToString(b[:])
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
package alpine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeSyncStore is an in-memory SyncStore: the metadata cache (via the embedded
|
||||
// fakeStore) plus the shared sync-state lease, whose claim mirrors the atomic
|
||||
// semantics of the real SQL (recency gate AND no live lease).
|
||||
type fakeSyncStore struct {
|
||||
*fakeStore
|
||||
|
||||
mu sync.Mutex
|
||||
remotes []models.Remote
|
||||
leaseOwner map[string]string
|
||||
leaseExp map[string]time.Time
|
||||
lastSynced map[string]time.Time
|
||||
etags map[string]string
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
return &fakeSyncStore{
|
||||
fakeStore: newFakeStore(),
|
||||
leaseOwner: map[string]string{},
|
||||
leaseExp: map[string]time.Time{},
|
||||
lastSynced: map[string]time.Time{},
|
||||
etags: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ListGitHubAlpineRemotes(_ context.Context) ([]models.Remote, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]models.Remote(nil), f.remotes...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
now := time.Now()
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
freshOK := !hasLS || now.Sub(ls) >= freshness
|
||||
leaseOK := !hasExp || exp.Before(now)
|
||||
if freshOK && leaseOK {
|
||||
f.leaseOwner[name] = owner
|
||||
f.leaseExp[name] = now.Add(lease)
|
||||
return true, f.etags[name], nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.leaseOwner[name] != owner {
|
||||
return nil
|
||||
}
|
||||
f.lastSynced[name] = syncedAt
|
||||
f.etags[name] = etag
|
||||
delete(f.leaseOwner, name)
|
||||
delete(f.leaseExp, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func testSyncConfig() SyncConfig {
|
||||
return SyncConfig{RatePerSec: 1000, Burst: 100, Workers: 1, PollInterval: time.Hour}
|
||||
}
|
||||
|
||||
// (a) A 304 conditional response must derive nothing: no asset fetches and
|
||||
// changed=false, so an unchanged repo is nearly free.
|
||||
func TestSyncerConditionalNotModifiedSkipsDerive(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
etag1, changed, err := p.scanWithState(context.Background(), remote, store, "")
|
||||
if err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
if !changed || etag1 != `"v1"` {
|
||||
t.Fatalf("first scan changed=%v etag=%q, want true and \"v1\"", changed, etag1)
|
||||
}
|
||||
priorRange := fx.rangeHit["demo-1.2.3-r0.apk"]
|
||||
if priorRange == 0 {
|
||||
t.Fatal("first scan should have fetched the asset .PKGINFO")
|
||||
}
|
||||
|
||||
etag2, changed2, err := p.scanWithState(context.Background(), remote, store, etag1)
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if changed2 {
|
||||
t.Fatal("304 scan must report changed=false")
|
||||
}
|
||||
if etag2 != etag1 {
|
||||
t.Fatalf("etag changed across 304: %q -> %q", etag1, etag2)
|
||||
}
|
||||
if fx.notModHit != 1 {
|
||||
t.Fatalf("want exactly one 304 releases response, got %d", fx.notModHit)
|
||||
}
|
||||
if got := fx.rangeHit["demo-1.2.3-r0.apk"]; got != priorRange {
|
||||
t.Fatalf("304 scan re-fetched asset .PKGINFO: %d -> %d", priorRange, got)
|
||||
}
|
||||
}
|
||||
|
||||
// (b) On a real change, only the newly added asset is derived.
|
||||
func TestSyncerIncrementalDerivesOnlyNewAsset(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if _, _, err := p.scanWithState(context.Background(), remote, store, ""); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
demoRange := fx.rangeHit["demo-1.2.3-r0.apk"]
|
||||
|
||||
fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
|
||||
fx.etag = `"v2"`
|
||||
|
||||
if _, changed, err := p.scanWithState(context.Background(), remote, store, `"v1"`); err != nil || !changed {
|
||||
t.Fatalf("second scan changed=%v err=%v", changed, err)
|
||||
}
|
||||
|
||||
rows, _ := store.ListAlpineMetadataEntries(context.Background(), remote.Name)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("want 2 cached rows after incremental derive, got %d", len(rows))
|
||||
}
|
||||
if got := fx.rangeHit["demo-1.2.3-r0.apk"]; got != demoRange {
|
||||
t.Fatalf("already-cached asset was re-fetched: %d -> %d", demoRange, got)
|
||||
}
|
||||
if fx.rangeHit["other-9-r0.apk"] == 0 {
|
||||
t.Fatal("newly added asset was not derived")
|
||||
}
|
||||
}
|
||||
|
||||
// (c) The shared limiter caps the request rate.
|
||||
func TestRateLimiterCapsRequestRate(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
p.limiter = rate.NewLimiter(rate.Every(120*time.Millisecond), 1)
|
||||
remote := fx.remote()
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, _, _, err := p.fetchReleases(context.Background(), remote, ""); err != nil {
|
||||
t.Fatalf("fetchReleases %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
|
||||
t.Fatalf("rate limiter did not throttle: 3 calls took %v, want >= 200ms", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// (d) Concurrent enqueues for the same remote coalesce to a single queued job.
|
||||
func TestSyncerEnqueueDedup(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-apk", PackageType: models.PackageGitHubAlpine, MutableTTL: 3600}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); s.enqueue(remote, false) }()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if got := len(s.jobs); got != 1 {
|
||||
t.Fatalf("want exactly 1 coalesced job, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// (e) Prime-on-create enqueues a prime job.
|
||||
func TestSyncerEnqueuePrime(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-apk", PackageType: models.PackageGitHubAlpine, MutableTTL: 3600}
|
||||
|
||||
s.EnqueuePrime(remote)
|
||||
select {
|
||||
case job := <-s.jobs:
|
||||
if !job.prime || job.remote.Name != "acme-apk" {
|
||||
t.Fatalf("bad prime job: %+v", job)
|
||||
}
|
||||
default:
|
||||
t.Fatal("EnqueuePrime did not enqueue a job")
|
||||
}
|
||||
}
|
||||
|
||||
// (f) A held lease prevents a second replica from scanning.
|
||||
func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
claimed, _, err := store.ClaimGitHubAlpineSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote})
|
||||
|
||||
if fx.releasesHit != 0 {
|
||||
t.Fatalf("second replica scanned while lease held: %d releases calls", fx.releasesHit)
|
||||
}
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), remote.Name); len(rows) != 0 {
|
||||
t.Fatalf("second replica derived metadata while lease held: %d rows", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
// With the syncer wired and the cache empty, an index request enqueues a prime
|
||||
// and returns a retryable 503 when it has not landed within the cold wait.
|
||||
func TestServeRemoteColdStartReturns503(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
p.coldWait = 300 * time.Millisecond
|
||||
_ = newSyncer(store, p, testSyncConfig()) // binds p.syncer, but no workers running
|
||||
remote := fx.remote()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("cold empty cache must return 503, got %d", rec.Code)
|
||||
}
|
||||
if rec.Header().Get("Retry-After") == "" {
|
||||
t.Fatal("503 should carry Retry-After")
|
||||
}
|
||||
if got := len(p.syncer.jobs); got != 1 {
|
||||
t.Fatalf("cold start did not enqueue a prime, jobs=%d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With the cache warm, the same request serves the index immediately (no 503).
|
||||
func TestServeRemoteWarmCacheServesImmediately(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
_ = newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-apk/x86_64/APKINDEX.tar.gz", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "x86_64/APKINDEX.tar.gz", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle APKINDEX")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("warm cache must serve 200, got %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A prime job (freshness 0) runs even right after a sync; a periodic job at the
|
||||
// same moment is gated by the recency window.
|
||||
func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
var _ provider.RemoteMetadataStore = store
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: true})
|
||||
if rows, _ := store.ListAlpineMetadataEntries(context.Background(), remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("prime did not derive: %d rows", len(rows))
|
||||
}
|
||||
releasesAfterPrime := fx.releasesHit
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: false})
|
||||
if fx.releasesHit != releasesAfterPrime {
|
||||
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
|
||||
}
|
||||
}
|
||||
@@ -1,462 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/auth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
provider.Register(&Provider{})
|
||||
}
|
||||
|
||||
// mutableRe marks the apt index surface (both the flat local repo and a proxied
|
||||
// Debian/Ubuntu mirror's dists/ tree) so the caching engine revalidates it
|
||||
// instead of freezing it like an immutable .deb.
|
||||
var mutableRe = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(^|/)Packages(\.gz|\.xz|\.bz2)?$`),
|
||||
regexp.MustCompile(`(^|/)Sources(\.gz|\.xz|\.bz2)?$`),
|
||||
regexp.MustCompile(`(^|/)Release$`),
|
||||
regexp.MustCompile(`(^|/)InRelease$`),
|
||||
regexp.MustCompile(`(^|/)Release\.gpg$`),
|
||||
regexp.MustCompile(`(^|/)Contents-`),
|
||||
regexp.MustCompile(`^dists/`),
|
||||
regexp.MustCompile(`/by-hash/`),
|
||||
}
|
||||
|
||||
type Provider struct{}
|
||||
|
||||
func (p *Provider) Type() models.PackageType { return models.PackageDeb }
|
||||
|
||||
func (p *Provider) Classify(path string) provider.Mutability {
|
||||
for _, re := range mutableRe {
|
||||
if re.MatchString(path) {
|
||||
return provider.Mutable
|
||||
}
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *Provider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".deb"):
|
||||
return "application/vnd.debian.binary-package"
|
||||
case strings.HasSuffix(path, ".gz"):
|
||||
return "application/gzip"
|
||||
case strings.HasSuffix(path, ".xz"):
|
||||
return "application/x-xz"
|
||||
case strings.HasSuffix(path, "Packages"), strings.HasSuffix(path, "Release"),
|
||||
strings.HasSuffix(path, "InRelease"), strings.HasSuffix(path, "Sources"):
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
|
||||
return auth.BasicHeaders(remote), nil
|
||||
}
|
||||
|
||||
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
|
||||
filename := filePath
|
||||
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
|
||||
filename = filePath[idx+1:]
|
||||
}
|
||||
|
||||
if !strings.HasSuffix(strings.ToLower(filename), ".deb") {
|
||||
return "", "", fmt.Errorf("file must be a .deb package")
|
||||
}
|
||||
|
||||
return "pool/" + filename, "application/vnd.debian.binary-package", nil
|
||||
}
|
||||
|
||||
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
|
||||
filename := strings.TrimPrefix(storagePath, "pool/")
|
||||
return map[string]any{
|
||||
"filename": filename,
|
||||
"content_hash": contentHash,
|
||||
"size_bytes": sizeBytes,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
|
||||
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
|
||||
|
||||
reader, blobSize, err := blobs.Download(ctx, s3Key)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
raw, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: read failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
control, err := extractControl(raw)
|
||||
if err != nil {
|
||||
slog.Error("deb metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
fields := parseControlFields(control)
|
||||
|
||||
sum := md5.Sum(raw)
|
||||
meta := &provider.DebMetadata{
|
||||
RepoName: repoName,
|
||||
FilePath: storagePath,
|
||||
ContentHash: contentHash,
|
||||
Name: fields["Package"],
|
||||
Version: fields["Version"],
|
||||
Architecture: fields["Architecture"],
|
||||
Control: strings.TrimRight(control, "\n"),
|
||||
Size: blobSize,
|
||||
MD5: hex.EncodeToString(sum[:]),
|
||||
SHA256: strings.TrimPrefix(contentHash, "sha256:"),
|
||||
}
|
||||
|
||||
if meta.Name == "" {
|
||||
slog.Error("deb metadata: control missing Package field", "repo", repoName, "path", storagePath)
|
||||
return
|
||||
}
|
||||
|
||||
if err := db.InsertDebMetadata(ctx, meta); err != nil {
|
||||
slog.Error("deb metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("deb metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
|
||||
}
|
||||
|
||||
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
|
||||
if err := db.DeleteDebMetadata(ctx, repoName, storagePath); err != nil {
|
||||
slog.Error("deb metadata: delete failed", "repo", repoName, "path", storagePath, "error", err)
|
||||
return err
|
||||
}
|
||||
slog.Info("deb metadata: deleted", "repo", repoName, "path", storagePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// extractControl reads a .deb (an ar archive), locates the control.tar.* member,
|
||||
// decompresses it, and returns the raw ./control paragraph. Pure Go: no dpkg.
|
||||
func extractControl(deb []byte) (string, error) {
|
||||
members, err := readAr(deb)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var name string
|
||||
var data []byte
|
||||
for _, m := range members {
|
||||
if strings.HasPrefix(m.name, "control.tar") {
|
||||
name = m.name
|
||||
data = m.data
|
||||
break
|
||||
}
|
||||
}
|
||||
if data == nil {
|
||||
return "", errors.New("no control.tar member in .deb")
|
||||
}
|
||||
|
||||
tarBytes, err := decompress(name, data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return readControlParagraph(tarBytes)
|
||||
}
|
||||
|
||||
// readControlParagraph scans a decompressed control.tar and returns the raw
|
||||
// ./control paragraph. Shared by the local upload path (extractControl) and the
|
||||
// github_deb ranged-prefix parser.
|
||||
func readControlParagraph(controlTar []byte) (string, error) {
|
||||
tr := tar.NewReader(bytes.NewReader(controlTar))
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read control.tar: %w", err)
|
||||
}
|
||||
clean := strings.TrimPrefix(hdr.Name, "./")
|
||||
if clean == "control" {
|
||||
b, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read control file: %w", err)
|
||||
}
|
||||
return string(b), nil
|
||||
}
|
||||
}
|
||||
return "", errors.New("no ./control in control.tar")
|
||||
}
|
||||
|
||||
func decompress(name string, data []byte) ([]byte, error) {
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".gz"):
|
||||
zr, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zr.Close()
|
||||
return io.ReadAll(zr)
|
||||
case strings.HasSuffix(name, ".xz"):
|
||||
xr, err := xz.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(xr)
|
||||
case strings.HasSuffix(name, ".zst"):
|
||||
zr, err := zstd.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer zr.Close()
|
||||
return io.ReadAll(zr)
|
||||
case strings.HasSuffix(name, ".tar"):
|
||||
return data, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported control.tar compression: %s", name)
|
||||
}
|
||||
|
||||
type arMember struct {
|
||||
name string
|
||||
data []byte
|
||||
}
|
||||
|
||||
// readAr parses the (trivial) Unix ar archive that wraps a .deb. Each member has
|
||||
// a 60-byte header; the size field is decimal ASCII and data is padded to an
|
||||
// even offset.
|
||||
func readAr(data []byte) ([]arMember, error) {
|
||||
const magic = "!<arch>\n"
|
||||
if len(data) < len(magic) || string(data[:len(magic)]) != magic {
|
||||
return nil, errors.New("not an ar archive")
|
||||
}
|
||||
off := len(magic)
|
||||
|
||||
var members []arMember
|
||||
for off+60 <= len(data) {
|
||||
hdr := data[off : off+60]
|
||||
off += 60
|
||||
|
||||
name := strings.TrimRight(string(hdr[0:16]), " ")
|
||||
name = strings.TrimSuffix(name, "/")
|
||||
size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad ar size for %q: %w", name, err)
|
||||
}
|
||||
if off+int(size) > len(data) {
|
||||
return nil, fmt.Errorf("truncated ar member %q", name)
|
||||
}
|
||||
members = append(members, arMember{name: name, data: data[off : off+int(size)]})
|
||||
off += int(size)
|
||||
if size%2 == 1 {
|
||||
off++
|
||||
}
|
||||
}
|
||||
return members, nil
|
||||
}
|
||||
|
||||
// parseControlFields reads the single-line fields of an RFC822-style control
|
||||
// paragraph. Continuation lines (leading whitespace) belong to the previous
|
||||
// field and are ignored here since only Package/Version/Architecture are read.
|
||||
func parseControlFields(control string) map[string]string {
|
||||
fields := map[string]string{}
|
||||
sc := bufio.NewScanner(strings.NewReader(control))
|
||||
sc.Buffer(make([]byte, 0, 64*1024), 1024*1024)
|
||||
for sc.Scan() {
|
||||
line := sc.Text()
|
||||
if line == "" || line[0] == ' ' || line[0] == '\t' {
|
||||
continue
|
||||
}
|
||||
idx := strings.IndexByte(line, ':')
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
key := strings.TrimSpace(line[:idx])
|
||||
if _, seen := fields[key]; seen {
|
||||
continue
|
||||
}
|
||||
fields[key] = strings.TrimSpace(line[idx+1:])
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
// normalizeIndexPath collapses apt's verbatim dist prefix from a flat-repo
|
||||
// request. For `deb ... <repo>/ ./`, apt appends the "./" dist literally and asks
|
||||
// for "./Packages" (and "./Release", "./InRelease"); dot-segments must be
|
||||
// collapsed so the index matcher sees "Packages". A no-op for pool/*.deb paths.
|
||||
func normalizeIndexPath(p string) string {
|
||||
return strings.TrimPrefix(path.Clean("/"+p), "/")
|
||||
}
|
||||
|
||||
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
|
||||
path := normalizeIndexPath(reqPath)
|
||||
switch path {
|
||||
case "Packages", "Packages.gz", "Release":
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
reader, ok := files.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "deb metadata not available", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
metas, err := reader.ListDebMetadataEntries(r.Context(), repoName)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
slog.Warn("deb: metadata read canceled", "repo", repoName, "error", err)
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return true
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
|
||||
switch path {
|
||||
case "Packages":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePackages(metas))
|
||||
case "Packages.gz":
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(gzipBytes(generatePackages(metas)))
|
||||
case "Release":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateRelease(metas))
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("deb local index generation for virtual repos not supported")
|
||||
}
|
||||
|
||||
// generatePackages emits the flat-repo Packages file: each package's raw control
|
||||
// stanza followed by the apt-required Filename/Size/MD5sum/SHA256 fields, blank
|
||||
// line separated.
|
||||
func generatePackages(metas []provider.DebMetadata) []byte {
|
||||
var b bytes.Buffer
|
||||
for _, m := range metas {
|
||||
b.WriteString(strings.TrimRight(m.Control, "\n"))
|
||||
b.WriteString("\n")
|
||||
fmt.Fprintf(&b, "Filename: %s\n", m.FilePath)
|
||||
fmt.Fprintf(&b, "Size: %d\n", m.Size)
|
||||
if m.MD5 != "" {
|
||||
fmt.Fprintf(&b, "MD5sum: %s\n", m.MD5)
|
||||
}
|
||||
if m.SHA256 != "" {
|
||||
fmt.Fprintf(&b, "SHA256: %s\n", m.SHA256)
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func generateRelease(metas []provider.DebMetadata) []byte {
|
||||
packages := generatePackages(metas)
|
||||
packagesGz := gzipBytes(packages)
|
||||
|
||||
arches := uniqueArches(metas)
|
||||
|
||||
var b bytes.Buffer
|
||||
fmt.Fprintf(&b, "Date: %s\n", releaseDate(metas).Format(time.RFC1123Z))
|
||||
fmt.Fprintf(&b, "Architectures: %s\n", strings.Join(arches, " "))
|
||||
b.WriteString("Acquire-By-Hash: no\n")
|
||||
|
||||
b.WriteString("MD5Sum:\n")
|
||||
writeReleaseEntry(&b, md5Hex(packages), len(packages), "Packages")
|
||||
writeReleaseEntry(&b, md5Hex(packagesGz), len(packagesGz), "Packages.gz")
|
||||
|
||||
b.WriteString("SHA256:\n")
|
||||
writeReleaseEntry(&b, sha256Hex(packages), len(packages), "Packages")
|
||||
writeReleaseEntry(&b, sha256Hex(packagesGz), len(packagesGz), "Packages.gz")
|
||||
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
// releaseDate derives the Release Date: from the newest package's persisted
|
||||
// created_at (in UTC) so the file is byte-identical across the no-affinity
|
||||
// replicas and across regenerations (issue #117); an empty repo falls back to
|
||||
// the Unix epoch. This never uses wall clock, which also keeps Date: from
|
||||
// running ahead of any Valid-Until logic.
|
||||
func releaseDate(metas []provider.DebMetadata) time.Time {
|
||||
newest := time.Unix(0, 0)
|
||||
for _, m := range metas {
|
||||
if m.CreatedAt.After(newest) {
|
||||
newest = m.CreatedAt
|
||||
}
|
||||
}
|
||||
return newest.UTC()
|
||||
}
|
||||
|
||||
func writeReleaseEntry(b *bytes.Buffer, hash string, size int, name string) {
|
||||
fmt.Fprintf(b, " %s %d %s\n", hash, size, name)
|
||||
}
|
||||
|
||||
func uniqueArches(metas []provider.DebMetadata) []string {
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
for _, m := range metas {
|
||||
a := m.Architecture
|
||||
if a == "" || seen[a] {
|
||||
continue
|
||||
}
|
||||
seen[a] = true
|
||||
out = append(out, a)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func md5Hex(data []byte) string {
|
||||
h := md5.Sum(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
|
||||
func sha256Hex(data []byte) string {
|
||||
h := sha256.Sum256(data)
|
||||
return hex.EncodeToString(h[:])
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
// debFixture returns a fixed set of rows with persisted created_at values, in
|
||||
// the total order ListDebMetadataEntries produces (name, version, arch,
|
||||
// file_path), so the generators are exercised on a stable input.
|
||||
func debFixture() []provider.DebMetadata {
|
||||
t1 := time.Date(2026, 3, 1, 8, 30, 0, 0, time.UTC)
|
||||
t2 := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC) // newest
|
||||
return []provider.DebMetadata{
|
||||
{
|
||||
RepoName: "r", FilePath: "pool/aaa_1.0_amd64.deb", ContentHash: "sha256:aa",
|
||||
Name: "aaa", Version: "1.0", Architecture: "amd64",
|
||||
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64",
|
||||
Size: 100, MD5: "d41d8cd98f00b204e9800998ecf8427e", SHA256: "aa", CreatedAt: t1,
|
||||
},
|
||||
{
|
||||
RepoName: "r", FilePath: "pool/bbb_2.0_arm64.deb", ContentHash: "sha256:bb",
|
||||
Name: "bbb", Version: "2.0", Architecture: "arm64",
|
||||
Control: "Package: bbb\nVersion: 2.0\nArchitecture: arm64",
|
||||
Size: 200, MD5: "0cc175b9c0f1b6a831c399e269772661", SHA256: "bb", CreatedAt: t2,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebGeneratorsDeterministic asserts the served bytes are a pure function of
|
||||
// DB state: Packages, Packages.gz and Release are byte-identical across two
|
||||
// generations separated by wall-clock time. Fails against the old
|
||||
// time.Now()-stamped Release Date:.
|
||||
func TestDebGeneratorsDeterministic(t *testing.T) {
|
||||
metas := debFixture()
|
||||
|
||||
pkgs1 := generatePackages(metas)
|
||||
rel1 := generateRelease(metas)
|
||||
gz1 := gzipBytes(pkgs1)
|
||||
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
|
||||
pkgs2 := generatePackages(metas)
|
||||
rel2 := generateRelease(metas)
|
||||
gz2 := gzipBytes(pkgs2)
|
||||
|
||||
if !bytes.Equal(pkgs1, pkgs2) {
|
||||
t.Error("Packages differs across generations")
|
||||
}
|
||||
if !bytes.Equal(gz1, gz2) {
|
||||
t.Error("Packages.gz differs across generations")
|
||||
}
|
||||
if !bytes.Equal(rel1, rel2) {
|
||||
t.Errorf("Release differs across generations:\n--- first ---\n%s\n--- second ---\n%s", rel1, rel2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebReleaseDateUsesPersistedCreatedAt pins the Release Date: to the newest
|
||||
// persisted created_at (RFC1123Z, UTC), not wall clock. Fails against the old
|
||||
// time.Now() code.
|
||||
func TestDebReleaseDateUsesPersistedCreatedAt(t *testing.T) {
|
||||
metas := debFixture()
|
||||
want := time.Date(2026, 4, 15, 12, 0, 0, 0, time.UTC).Format(time.RFC1123Z)
|
||||
|
||||
rel := string(generateRelease(metas))
|
||||
var got string
|
||||
for _, line := range strings.Split(rel, "\n") {
|
||||
if strings.HasPrefix(line, "Date:") {
|
||||
got = strings.TrimSpace(strings.TrimPrefix(line, "Date:"))
|
||||
break
|
||||
}
|
||||
}
|
||||
if got != want {
|
||||
t.Errorf("Release Date: = %q, want %q (newest created_at)", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebReleaseDateEmptyRepoIsEpoch guards the fallback: an empty repo yields a
|
||||
// deterministic epoch Date: rather than wall clock.
|
||||
func TestDebReleaseDateEmptyRepoIsEpoch(t *testing.T) {
|
||||
want := time.Unix(0, 0).UTC().Format(time.RFC1123Z)
|
||||
rel := string(generateRelease(nil))
|
||||
if !strings.Contains(rel, "Date: "+want+"\n") {
|
||||
t.Errorf("empty-repo Release missing epoch Date: %q\n%s", want, rel)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDebReleaseChecksumsMatchServedBytes is the exact apt invariant: the
|
||||
// sha256/size (and md5/size) advertised for Packages and Packages.gz in Release
|
||||
// equal the sha256/size of the actual bytes ServeLocalIndex serves. apt rejects
|
||||
// any mismatch.
|
||||
func TestDebReleaseChecksumsMatchServedBytes(t *testing.T) {
|
||||
metas := debFixture()
|
||||
|
||||
packages := generatePackages(metas)
|
||||
packagesGz := gzipBytes(packages)
|
||||
rel := string(generateRelease(metas))
|
||||
|
||||
wantSHA := map[string]struct {
|
||||
hash string
|
||||
size int
|
||||
}{
|
||||
"Packages": {sha256Hex(packages), len(packages)},
|
||||
"Packages.gz": {sha256Hex(packagesGz), len(packagesGz)},
|
||||
}
|
||||
wantMD5 := map[string]struct {
|
||||
hash string
|
||||
size int
|
||||
}{
|
||||
"Packages": {md5Hex(packages), len(packages)},
|
||||
"Packages.gz": {md5Hex(packagesGz), len(packagesGz)},
|
||||
}
|
||||
|
||||
sha := parseReleaseSection(rel, "SHA256:")
|
||||
md5s := parseReleaseSection(rel, "MD5Sum:")
|
||||
|
||||
for name, w := range wantSHA {
|
||||
got, ok := sha[name]
|
||||
if !ok {
|
||||
t.Fatalf("Release SHA256 section missing %q", name)
|
||||
}
|
||||
if got.hash != w.hash || got.size != w.size {
|
||||
t.Errorf("Release SHA256 %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size)
|
||||
}
|
||||
}
|
||||
for name, w := range wantMD5 {
|
||||
got, ok := md5s[name]
|
||||
if !ok {
|
||||
t.Fatalf("Release MD5Sum section missing %q", name)
|
||||
}
|
||||
if got.hash != w.hash || got.size != w.size {
|
||||
t.Errorf("Release MD5Sum %s = (%s, %d), served bytes are (%s, %d)", name, got.hash, got.size, w.hash, w.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type releaseEntry struct {
|
||||
hash string
|
||||
size int
|
||||
}
|
||||
|
||||
// parseReleaseSection reads the indented " <hash> <size> <name>" lines that
|
||||
// follow a "SHA256:" / "MD5Sum:" header until the next non-indented line.
|
||||
func parseReleaseSection(release, header string) map[string]releaseEntry {
|
||||
out := map[string]releaseEntry{}
|
||||
lines := strings.Split(release, "\n")
|
||||
in := false
|
||||
for _, line := range lines {
|
||||
if line == header {
|
||||
in = true
|
||||
continue
|
||||
}
|
||||
if !in {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(line, " ") {
|
||||
break
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 3 {
|
||||
continue
|
||||
}
|
||||
size, _ := strconv.Atoi(fields[1])
|
||||
out[fields[2]] = releaseEntry{hash: fields[0], size: size}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -1,408 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/ulikunitz/xz"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type fakeBlobReader struct{ data []byte }
|
||||
|
||||
func (f fakeBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
|
||||
}
|
||||
|
||||
type errBlobReader struct{}
|
||||
|
||||
func (errBlobReader) Download(_ context.Context, _ string) (io.ReadCloser, int64, error) {
|
||||
return nil, 0, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
// fakeDebStore satisfies provider.MetadataStore (both insert methods) and
|
||||
// records the deb row that AfterUpload writes.
|
||||
type fakeDebStore struct{ inserted *provider.DebMetadata }
|
||||
|
||||
func (f *fakeDebStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeDebStore) InsertDebMetadata(_ context.Context, m *provider.DebMetadata) error {
|
||||
f.inserted = m
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeDebReader struct{ metas []provider.DebMetadata }
|
||||
|
||||
func (f fakeDebReader) ListDebMetadataEntries(context.Context, string) ([]provider.DebMetadata, error) {
|
||||
return f.metas, nil
|
||||
}
|
||||
func (f fakeDebReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (f fakeDebReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
type errDebReader struct{}
|
||||
|
||||
func (errDebReader) ListDebMetadataEntries(context.Context, string) ([]provider.DebMetadata, error) {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
func (errDebReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (errDebReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
|
||||
|
||||
func TestDebPureFuncs(t *testing.T) {
|
||||
p := &Provider{}
|
||||
if p.Type() != models.PackageDeb {
|
||||
t.Errorf("type = %q", p.Type())
|
||||
}
|
||||
if out, _ := p.RewriteResponse(nil, models.Remote{}, "http://p"); out != nil {
|
||||
t.Error("deb never rewrites")
|
||||
}
|
||||
if got := p.UpstreamURL(models.Remote{BaseURL: "https://mirror/"}, "/dists/bookworm/Release"); got != "https://mirror/dists/bookworm/Release" {
|
||||
t.Errorf("upstream url %q", got)
|
||||
}
|
||||
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "u", Password: "p"})
|
||||
if h.Get("Authorization") == "" {
|
||||
t.Error("auth header")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebClassify(t *testing.T) {
|
||||
p := &Provider{}
|
||||
tests := []struct {
|
||||
path string
|
||||
want provider.Mutability
|
||||
}{
|
||||
{"pool/foo_1.0_amd64.deb", provider.Immutable},
|
||||
{"Packages", provider.Mutable},
|
||||
{"Packages.gz", provider.Mutable},
|
||||
{"Release", provider.Mutable},
|
||||
{"InRelease", provider.Mutable},
|
||||
{"Release.gpg", provider.Mutable},
|
||||
{"dists/bookworm/main/binary-amd64/Packages", provider.Mutable},
|
||||
{"dists/bookworm/Release", provider.Mutable},
|
||||
{"dists/bookworm/main/by-hash/SHA256/abc", provider.Mutable},
|
||||
{"dists/bookworm/main/Contents-amd64.gz", provider.Mutable},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := p.Classify(tt.path); got != tt.want {
|
||||
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebContentType(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for path, want := range map[string]string{
|
||||
"pool/foo_1.0_amd64.deb": "application/vnd.debian.binary-package",
|
||||
"dists/bookworm/main/bin/Packages.gz": "application/gzip",
|
||||
"dists/bookworm/main/bin/Packages.xz": "application/x-xz",
|
||||
"Packages": "text/plain",
|
||||
"Release": "text/plain",
|
||||
"InRelease": "text/plain",
|
||||
"pool/other": "application/octet-stream",
|
||||
} {
|
||||
if got := p.ContentType(path); got != want {
|
||||
t.Errorf("ContentType(%q) = %q, want %q", path, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebValidateUpload(t *testing.T) {
|
||||
p := &Provider{}
|
||||
sp, ct, err := p.ValidateUpload("dir/foo_1.0_amd64.deb")
|
||||
if err != nil || sp != "pool/foo_1.0_amd64.deb" || ct != "application/vnd.debian.binary-package" {
|
||||
t.Errorf("sp=%q ct=%q err=%v", sp, ct, err)
|
||||
}
|
||||
if _, _, err := p.ValidateUpload("foo.rpm"); err == nil {
|
||||
t.Error("expected error for non-deb")
|
||||
}
|
||||
resp := p.UploadResponse("pool/foo_1.0_amd64.deb", "sha256:abc", 42)
|
||||
if resp["filename"] != "foo_1.0_amd64.deb" || resp["content_hash"] != "sha256:abc" || resp["size_bytes"] != int64(42) {
|
||||
t.Errorf("upload response %v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebAfterUpload(t *testing.T) {
|
||||
data := testsupport.MinimalDeb("e2e-testpkg", "1.2.3", "amd64")
|
||||
store := &fakeDebStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "myrepo", "pool/e2e-testpkg_1.2.3_amd64.deb",
|
||||
"sha256:deadbeef", fakeBlobReader{data: data}, store)
|
||||
|
||||
m := store.inserted
|
||||
if m == nil {
|
||||
t.Fatal("no metadata inserted")
|
||||
}
|
||||
if m.Name != "e2e-testpkg" || m.Version != "1.2.3" || m.Architecture != "amd64" {
|
||||
t.Errorf("unexpected metadata: %+v", m)
|
||||
}
|
||||
if m.Size != int64(len(data)) {
|
||||
t.Errorf("Size = %d, want %d", m.Size, len(data))
|
||||
}
|
||||
if m.SHA256 != "deadbeef" {
|
||||
t.Errorf("SHA256 = %q, want deadbeef", m.SHA256)
|
||||
}
|
||||
if m.MD5 == "" {
|
||||
t.Error("MD5 not computed")
|
||||
}
|
||||
if !strings.Contains(m.Control, "Package: e2e-testpkg") {
|
||||
t.Errorf("raw control not stored: %q", m.Control)
|
||||
}
|
||||
// The raw stanza is stored verbatim (no trailing newline) so Packages can
|
||||
// reproduce it faithfully.
|
||||
if strings.HasSuffix(m.Control, "\n") {
|
||||
t.Error("control should be trimmed of trailing newline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebAfterUploadErrors(t *testing.T) {
|
||||
// Download failure: no insert, no panic.
|
||||
store := &fakeDebStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", errBlobReader{}, store)
|
||||
if store.inserted != nil {
|
||||
t.Error("no metadata should be inserted on download error")
|
||||
}
|
||||
// Not a .deb (ar) archive.
|
||||
store2 := &fakeDebStore{}
|
||||
(&Provider{}).AfterUpload(context.Background(), "r", "p", "sha256:x", fakeBlobReader{data: []byte("not a deb")}, store2)
|
||||
if store2.inserted != nil {
|
||||
t.Error("no metadata should be inserted on parse error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebControlDecompression(t *testing.T) {
|
||||
// The control tarball may be gzip, xz, or zstd (goreleaser/nfpm emit gzip or
|
||||
// xz); each must round-trip to the same control stanza.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
member string
|
||||
comp func([]byte) []byte
|
||||
}{
|
||||
{"gzip", "control.tar.gz", gzipBytes},
|
||||
{"xz", "control.tar.xz", xzBytes},
|
||||
{"zstd", "control.tar.zst", zstdBytes},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
deb := buildDeb("pkg", "9.9", "arm64", tc.member, tc.comp)
|
||||
control, err := extractControl(deb)
|
||||
if err != nil {
|
||||
t.Fatalf("extractControl: %v", err)
|
||||
}
|
||||
fields := parseControlFields(control)
|
||||
if fields["Package"] != "pkg" || fields["Version"] != "9.9" || fields["Architecture"] != "arm64" {
|
||||
t.Errorf("fields = %v", fields)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebParseControlContinuationLines(t *testing.T) {
|
||||
control := "Package: p\nVersion: 1\n" +
|
||||
"Description: short\n very long\n .\n more\n" +
|
||||
"Architecture: all\n"
|
||||
f := parseControlFields(control)
|
||||
if f["Package"] != "p" || f["Version"] != "1" || f["Architecture"] != "all" {
|
||||
t.Errorf("continuation lines corrupted parse: %v", f)
|
||||
}
|
||||
if f["Description"] != "short" {
|
||||
t.Errorf("Description folded continuation into value: %q", f["Description"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebServeLocalIndex(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeDebReader{metas: []provider.DebMetadata{
|
||||
{Name: "aaa", Version: "1.0", Architecture: "amd64", FilePath: "pool/aaa_1.0_amd64.deb",
|
||||
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64", Size: 100, MD5: "md5aaa", SHA256: "sha256aaa"},
|
||||
{Name: "bbb", Version: "2.0", Architecture: "arm64", FilePath: "pool/bbb_2.0_arm64.deb",
|
||||
Control: "Package: bbb\nVersion: 2.0\nArchitecture: arm64", Size: 200, MD5: "md5bbb", SHA256: "sha256bbb"},
|
||||
}}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// Packages lists both packages with their apt fields.
|
||||
w := serve("Packages")
|
||||
body := w.Body.String()
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("Packages code %d", w.Code)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"Package: aaa", "Package: bbb",
|
||||
"Filename: pool/aaa_1.0_amd64.deb", "Size: 100", "MD5sum: md5aaa", "SHA256: sha256aaa",
|
||||
"Filename: pool/bbb_2.0_arm64.deb", "Size: 200",
|
||||
} {
|
||||
if !strings.Contains(body, want) {
|
||||
t.Errorf("Packages missing %q:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
// Stanzas are blank-line separated.
|
||||
if !strings.Contains(body, "SHA256: sha256aaa\n\n") {
|
||||
t.Errorf("stanzas not blank-line separated:\n%s", body)
|
||||
}
|
||||
|
||||
// Packages.gz decompresses to exactly the plain Packages bytes.
|
||||
w = serve("Packages.gz")
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("Packages.gz code %d", w.Code)
|
||||
}
|
||||
zr, err := gzip.NewReader(bytes.NewReader(w.Body.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatalf("Packages.gz not gzip: %v", err)
|
||||
}
|
||||
plain, _ := io.ReadAll(zr)
|
||||
if !bytes.Equal(plain, []byte(body)) {
|
||||
t.Error("Packages.gz does not decompress to Packages")
|
||||
}
|
||||
|
||||
// Release lists arches and both index files under MD5Sum/SHA256.
|
||||
w = serve("Release")
|
||||
rel := w.Body.String()
|
||||
if w.Code != 200 {
|
||||
t.Fatalf("Release code %d", w.Code)
|
||||
}
|
||||
for _, want := range []string{"Date:", "Architectures: amd64 arm64", "Acquire-By-Hash: no", "MD5Sum:", "SHA256:", " Packages\n", " Packages.gz\n"} {
|
||||
if !strings.Contains(rel, want) {
|
||||
t.Errorf("Release missing %q:\n%s", want, rel)
|
||||
}
|
||||
}
|
||||
|
||||
// Unsigned trust model: no InRelease / Release.gpg served here.
|
||||
for _, path := range []string{"InRelease", "Release.gpg", "pool/aaa_1.0_amd64.deb"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Errorf("ServeLocalIndex should return false for %q", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Real apt appends the flat-repo dist "./" verbatim, so it requests "./Packages"
|
||||
// / "./Release" (curl pre-normalizes /./ which masks this). The handler must
|
||||
// collapse the dot-segment and return the same bytes as the un-prefixed request.
|
||||
func TestDebServeLocalIndexAptDotSegment(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeDebReader{metas: []provider.DebMetadata{
|
||||
{Name: "aaa", Version: "1.0", Architecture: "amd64", FilePath: "pool/aaa_1.0_amd64.deb",
|
||||
Control: "Package: aaa\nVersion: 1.0\nArchitecture: amd64", Size: 100, MD5: "md5aaa", SHA256: "sha256aaa"},
|
||||
}}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "myrepo", path) {
|
||||
t.Fatalf("ServeLocalIndex returned false for %q", path)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// Packages is deterministic: require exact byte identity.
|
||||
if plain, dotted := serve("Packages"), serve("./Packages"); plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("Packages: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
} else if !bytes.Equal(plain.Body.Bytes(), dotted.Body.Bytes()) {
|
||||
t.Error("./Packages body differs from Packages body")
|
||||
}
|
||||
|
||||
// Release carries a Date: header stamped from time.Now(); compare the rest.
|
||||
plain, dotted := serve("Release"), serve("./Release")
|
||||
if plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("Release: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
}
|
||||
if stripDate(plain.Body.String()) != stripDate(dotted.Body.String()) {
|
||||
t.Error("./Release body differs from Release body (ignoring Date)")
|
||||
}
|
||||
}
|
||||
|
||||
func stripDate(s string) string {
|
||||
var out []string
|
||||
for _, line := range strings.Split(s, "\n") {
|
||||
if strings.HasPrefix(line, "Date:") {
|
||||
continue
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
func TestDebServeMetadataError(t *testing.T) {
|
||||
p := &Provider{}
|
||||
for _, path := range []string{"Packages", "Packages.gz", "Release"} {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
p.ServeLocalIndex(w, r, errDebReader{}, "repo", path)
|
||||
if w.Code != 500 {
|
||||
t.Errorf("%s with failing reader = %d, want 500", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDebGenerateLocalIndexUnsupported(t *testing.T) {
|
||||
if _, err := (&Provider{}).GenerateLocalIndex(context.Background(), fakeDebReader{}, "r", "Packages"); err == nil {
|
||||
t.Error("expected unsupported error")
|
||||
}
|
||||
}
|
||||
|
||||
// buildDeb assembles an ar .deb whose control member uses the given name and
|
||||
// compressor, so the decompression branches can be exercised directly.
|
||||
func buildDeb(name, version, arch, member string, comp func([]byte) []byte) []byte {
|
||||
control := "Package: " + name + "\nVersion: " + version + "\nArchitecture: " + arch + "\n"
|
||||
controlTar := comp(tarSingle("./control", []byte(control)))
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString("!<arch>\n")
|
||||
arWrite(&buf, "debian-binary", []byte("2.0\n"))
|
||||
arWrite(&buf, member, controlTar)
|
||||
arWrite(&buf, "data.tar.gz", gzipBytes(tarSingle("./x", []byte("x"))))
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func tarSingle(name string, data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg})
|
||||
tw.Write(data)
|
||||
tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func arWrite(buf *bytes.Buffer, name string, data []byte) {
|
||||
fmt.Fprintf(buf, "%-16s%-12s%-6s%-6s%-8s%-10d`\n", name, "0", "0", "0", "100644", len(data))
|
||||
buf.Write(data)
|
||||
if len(data)%2 == 1 {
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
func xzBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
w, _ := xz.NewWriter(&buf)
|
||||
w.Write(data)
|
||||
w.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func zstdBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
w, _ := zstd.NewWriter(&buf)
|
||||
w.Write(data)
|
||||
w.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -1,724 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// gitHubProvider is the process-wide singleton for github_deb. The background
|
||||
// Syncer binds its shared rate limiter and work queue onto this instance so the
|
||||
// request path and the syncer drive the same derive machinery.
|
||||
var gitHubProvider = newGitHubProvider()
|
||||
|
||||
func init() {
|
||||
provider.Register(gitHubProvider)
|
||||
}
|
||||
|
||||
// Tuning knobs for the no-precache control fetch. A .deb is an ar archive whose
|
||||
// control.tar member sits right after the tiny debian-binary member, so a small
|
||||
// front prefix reliably covers it.
|
||||
const (
|
||||
defaultHeaderRangeInitial = 32 << 10 // 32 KiB — covers control.tar of almost every .deb
|
||||
defaultHeaderRangeMax = 16 << 20 // 16 MiB — give up past this and skip the asset
|
||||
defaultReleasePageCap = 10 // 100 releases/page * 10 pages
|
||||
|
||||
defaultScanTimeout = 10 * time.Minute
|
||||
defaultServeTimeout = 30 * time.Second
|
||||
defaultColdWait = 8 * time.Second
|
||||
)
|
||||
|
||||
// GitHubProvider is a metadata-only remote: it scans a GitHub repo's releases
|
||||
// for .deb assets, derives per-asset control metadata via a ranged prefix fetch
|
||||
// (never downloading whole packages), synthesizes a flat apt repository from that
|
||||
// cached metadata, and redirects package downloads to a backend "releases_remote"
|
||||
// (the generic github.com remote) that serves the actual bytes.
|
||||
type GitHubProvider struct {
|
||||
client *http.Client
|
||||
|
||||
headerInitial int64
|
||||
headerMax int64
|
||||
pageCap int
|
||||
scanTimeout time.Duration
|
||||
serveTimeout time.Duration
|
||||
coldWait time.Duration
|
||||
|
||||
limiter *rate.Limiter
|
||||
syncer *Syncer
|
||||
|
||||
serverCred githubauth.Credential
|
||||
|
||||
mu sync.Mutex
|
||||
scanning map[string]bool
|
||||
lastScan map[string]time.Time
|
||||
}
|
||||
|
||||
func newGitHubProvider() *GitHubProvider {
|
||||
return &GitHubProvider{
|
||||
client: &http.Client{},
|
||||
headerInitial: defaultHeaderRangeInitial,
|
||||
headerMax: defaultHeaderRangeMax,
|
||||
pageCap: defaultReleasePageCap,
|
||||
scanTimeout: defaultScanTimeout,
|
||||
serveTimeout: defaultServeTimeout,
|
||||
coldWait: defaultColdWait,
|
||||
scanning: map[string]bool{},
|
||||
lastScan: map[string]time.Time{},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) limiterWait(ctx context.Context) error {
|
||||
if p.limiter == nil {
|
||||
return nil
|
||||
}
|
||||
return p.limiter.Wait(ctx)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) Type() models.PackageType { return models.PackageGitHubDeb }
|
||||
|
||||
func (p *GitHubProvider) Classify(path string) provider.Mutability {
|
||||
switch path {
|
||||
case "Packages", "Packages.gz", "Release", "InRelease", "Release.gpg":
|
||||
return provider.Mutable
|
||||
}
|
||||
return provider.Immutable
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) ContentType(path string) string {
|
||||
switch {
|
||||
case strings.HasSuffix(path, ".deb"):
|
||||
return "application/vnd.debian.binary-package"
|
||||
case strings.HasSuffix(path, ".gz"):
|
||||
return "application/gzip"
|
||||
case path == "Packages" || path == "Release" || path == "InRelease":
|
||||
return "text/plain"
|
||||
}
|
||||
return "application/octet-stream"
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) UpstreamURL(remote models.Remote, path string) string {
|
||||
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) {
|
||||
return p.githubHeaders(ctx, remote, false)
|
||||
}
|
||||
|
||||
// ServeRemote answers a request against a github_deb remote. It refreshes the
|
||||
// derived metadata (bounded by mutable_ttl), serves a synthesized flat apt repo
|
||||
// (Packages/Packages.gz/Release), 404s the signed index variants (the repo is
|
||||
// consumed via [trusted=yes]), and 302-redirects .deb downloads to the backend
|
||||
// releases_remote. Returns false only for paths it does not own.
|
||||
func (p *GitHubProvider) ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, reqPath, proxyBaseURL string, store provider.RemoteMetadataStore) bool {
|
||||
p.onRequest(remote, store)
|
||||
|
||||
// apt appends the flat-repo dist "./" verbatim, so it asks for "./Packages"
|
||||
// etc.; collapse the dot-segment before matching the synthesized index.
|
||||
path := normalizeIndexPath(reqPath)
|
||||
|
||||
switch path {
|
||||
case "Packages", "Packages.gz", "Release":
|
||||
p.serveIndex(w, r, remote, path, store)
|
||||
return true
|
||||
case "InRelease", "Release.gpg":
|
||||
// Unsigned flat repo: apt consumes it with [trusted=yes]. Signal absence
|
||||
// so apt falls back to the plain Release without waiting on a signature.
|
||||
http.Error(w, "not found", http.StatusNotFound)
|
||||
return true
|
||||
}
|
||||
|
||||
if strings.HasSuffix(path, ".deb") {
|
||||
if remote.ReleasesRemote == "" {
|
||||
http.Error(w, "github_deb remote has no releases_remote configured for downloads", http.StatusInternalServerError)
|
||||
return true
|
||||
}
|
||||
loc := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.ReleasesRemote + "/" + strings.TrimLeft(path, "/")
|
||||
http.Redirect(w, r, loc, http.StatusFound)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serveIndex(w http.ResponseWriter, r *http.Request, remote models.Remote, path string, store provider.RemoteMetadataStore) {
|
||||
// Serve on a context detached from the inbound request so a client disconnect
|
||||
// never cancels the metadata DB read and surfaces as a 500.
|
||||
sctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), p.serveTimeout)
|
||||
defer cancel()
|
||||
|
||||
if p.syncer != nil && !p.ensurePrimed(sctx, remote, store) {
|
||||
w.Header().Set("Retry-After", "5")
|
||||
http.Error(w, "metadata is being prepared, retry shortly", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
http.Error(w, "deb metadata not available", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
metas, err := reader.ListDebMetadataEntries(sctx, remote.Name)
|
||||
if err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
http.Error(w, "metadata read canceled", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
switch path {
|
||||
case "Packages":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generatePackages(metas))
|
||||
case "Packages.gz":
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(gzipBytes(generatePackages(metas)))
|
||||
case "Release":
|
||||
w.Header().Set("Content-Type", "text/plain")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write(generateRelease(metas))
|
||||
}
|
||||
}
|
||||
|
||||
// onRequest keeps a remote's derived metadata fresh off the request path.
|
||||
func (p *GitHubProvider) onRequest(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, false)
|
||||
return
|
||||
}
|
||||
p.refresh(remote, store)
|
||||
}
|
||||
|
||||
// ensurePrimed returns true once the remote has at least one cached row. On an
|
||||
// empty cache it enqueues a prime and polls briefly for it to land.
|
||||
func (p *GitHubProvider) ensurePrimed(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) bool {
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
if p.syncer != nil {
|
||||
p.syncer.enqueue(remote, true)
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(p.coldWait)
|
||||
for time.Now().Before(deadline) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-time.After(400 * time.Millisecond):
|
||||
}
|
||||
if !p.cacheEmpty(ctx, store, remote.Name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) cacheEmpty(ctx context.Context, store provider.RemoteMetadataStore, name string) bool {
|
||||
reader, ok := store.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
rows, err := reader.ListDebMetadataEntries(ctx, name)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return len(rows) == 0
|
||||
}
|
||||
|
||||
// refresh brings the derived metadata up to date without coupling the scan to
|
||||
// the inbound request (legacy inline path used without a syncer / in unit tests).
|
||||
func (p *GitHubProvider) refresh(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
ttl := time.Duration(remote.MutableTTL) * time.Second
|
||||
if ttl <= 0 {
|
||||
ttl = 5 * time.Minute
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
last, ok := p.lastScan[remote.Name]
|
||||
fresh := ok && time.Since(last) < ttl
|
||||
if fresh || p.scanning[remote.Name] {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.scanning[remote.Name] = true
|
||||
p.mu.Unlock()
|
||||
|
||||
if p.cacheEmpty(context.Background(), store, remote.Name) {
|
||||
p.runScan(remote, store)
|
||||
return
|
||||
}
|
||||
go p.runScan(remote, store)
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) runScan(remote models.Remote, store provider.RemoteMetadataStore) {
|
||||
defer func() {
|
||||
p.mu.Lock()
|
||||
delete(p.scanning, remote.Name)
|
||||
p.mu.Unlock()
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), p.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
if err := p.scan(ctx, remote, store); err != nil {
|
||||
slog.Error("github_deb: release scan failed", "remote", remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
// scan runs a full unconditional derive. Retained for the legacy inline refresh
|
||||
// path and existing tests; the syncer uses scanWithState.
|
||||
func (p *GitHubProvider) scan(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore) error {
|
||||
_, _, err := p.scanWithState(ctx, remote, store, "")
|
||||
return err
|
||||
}
|
||||
|
||||
// scanWithState derives metadata incrementally. It sends the prior releases-list
|
||||
// ETag as a conditional request: a 304 means nothing changed. On a 200 it diffs
|
||||
// the release assets against the cache, derives only new/changed assets, prunes
|
||||
// assets that disappeared, and returns the new ETag.
|
||||
func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote, store provider.RemoteMetadataStore, etag string) (newEtag string, changed bool, err error) {
|
||||
releases, newEtag, notModified, err := p.fetchReleases(ctx, remote, etag)
|
||||
if err != nil {
|
||||
return etag, false, err
|
||||
}
|
||||
if notModified {
|
||||
return etag, false, nil
|
||||
}
|
||||
|
||||
reader, ok := store.(provider.DebMetadataReader)
|
||||
if !ok {
|
||||
return newEtag, false, errors.New("store does not support deb metadata reads")
|
||||
}
|
||||
existing, err := reader.ListDebMetadataEntries(ctx, remote.Name)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
existingByPath := make(map[string]provider.DebMetadata, len(existing))
|
||||
for _, m := range existing {
|
||||
existingByPath[m.FilePath] = m
|
||||
}
|
||||
|
||||
allow, err := compilePatterns(remote.Patterns)
|
||||
if err != nil {
|
||||
return newEtag, false, err
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range releases {
|
||||
if rel.Draft {
|
||||
continue
|
||||
}
|
||||
for _, asset := range rel.Assets {
|
||||
if !strings.HasSuffix(strings.ToLower(asset.Name), ".deb") {
|
||||
continue
|
||||
}
|
||||
if !matchesAny(allow, asset.Name) {
|
||||
continue
|
||||
}
|
||||
fp := assetPath(asset)
|
||||
if fp == "" {
|
||||
continue
|
||||
}
|
||||
seen[fp] = true
|
||||
|
||||
if cur, ok := existingByPath[fp]; ok {
|
||||
if asset.Digest == "" || cur.ContentHash == asset.Digest {
|
||||
continue
|
||||
}
|
||||
_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
|
||||
meta, err := p.deriveAsset(ctx, remote, asset, fp)
|
||||
if err != nil {
|
||||
slog.Warn("github_deb: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
if err := store.InsertDebMetadata(ctx, meta); err != nil {
|
||||
slog.Error("github_deb: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
|
||||
continue
|
||||
}
|
||||
slog.Info("github_deb: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
|
||||
}
|
||||
}
|
||||
|
||||
for fp := range existingByPath {
|
||||
if !seen[fp] {
|
||||
_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
|
||||
}
|
||||
}
|
||||
return newEtag, true, nil
|
||||
}
|
||||
|
||||
type ghRelease struct {
|
||||
TagName string `json:"tag_name"`
|
||||
Draft bool `json:"draft"`
|
||||
Assets []ghAsset `json:"assets"`
|
||||
}
|
||||
|
||||
type ghAsset struct {
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
BrowserDownloadURL string `json:"browser_download_url"`
|
||||
Digest string `json:"digest"`
|
||||
}
|
||||
|
||||
// fetchReleases lists a repo's releases, sending the prior ETag as If-None-Match
|
||||
// on page 1 so an unchanged repo short-circuits to notModified. Every call waits
|
||||
// on the shared limiter first.
|
||||
func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote, etag string) (all []ghRelease, newEtag string, notModified bool, err error) {
|
||||
base := strings.TrimRight(remote.BaseURL, "/") + "/releases"
|
||||
for page := 1; page <= p.pageCap; page++ {
|
||||
u := fmt.Sprintf("%s?per_page=100&page=%d", base, page)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
if page == 1 && etag != "" {
|
||||
req.Header.Set("If-None-Match", etag)
|
||||
}
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if page == 1 && resp.StatusCode == http.StatusNotModified {
|
||||
io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
return nil, etag, true, nil
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
respEtag := resp.Header.Get("ETag")
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
|
||||
}
|
||||
if page == 1 {
|
||||
newEtag = respEtag
|
||||
}
|
||||
var releases []ghRelease
|
||||
if err := json.Unmarshal(body, &releases); err != nil {
|
||||
return nil, "", false, fmt.Errorf("decode releases: %w", err)
|
||||
}
|
||||
if len(releases) == 0 {
|
||||
break
|
||||
}
|
||||
all = append(all, releases...)
|
||||
if len(releases) < 100 {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, newEtag, false, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote, asset ghAsset, fp string) (*provider.DebMetadata, error) {
|
||||
control, err := p.fetchControl(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fields := parseControlFields(control)
|
||||
|
||||
meta := &provider.DebMetadata{
|
||||
RepoName: remote.Name,
|
||||
FilePath: fp,
|
||||
Name: fields["Package"],
|
||||
Version: fields["Version"],
|
||||
Architecture: fields["Architecture"],
|
||||
Control: strings.TrimRight(control, "\n"),
|
||||
Size: asset.Size,
|
||||
}
|
||||
if meta.Name == "" {
|
||||
return nil, errors.New("control missing Package field")
|
||||
}
|
||||
|
||||
// The Packages SHA256 must be the sha256 of the whole .deb. Prefer GitHub's
|
||||
// asset digest so we never download the body; only when it is absent (or not
|
||||
// sha256) do we stream the asset once. MD5sum is left unset — apt verifies the
|
||||
// download against SHA256 alone under [trusted=yes].
|
||||
if h, ok := sha256FromDigest(asset.Digest); ok {
|
||||
meta.ContentHash = "sha256:" + h
|
||||
meta.SHA256 = h
|
||||
} else {
|
||||
h, err := p.computeSHA256(ctx, remote, asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("compute sha256: %w", err)
|
||||
}
|
||||
meta.ContentHash = "sha256:" + h
|
||||
meta.SHA256 = h
|
||||
}
|
||||
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// fetchControl pulls only the front of the .deb with a ranged GET and extracts
|
||||
// the control paragraph from it. control.tar sits right after the tiny
|
||||
// debian-binary member, so a small prefix suffices; a prefix that truncates the
|
||||
// control member doubles the range and retries.
|
||||
func (p *GitHubProvider) fetchControl(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
|
||||
n := p.headerInitial
|
||||
for {
|
||||
body, full, err := p.rangeGet(ctx, remote, downloadURL, n)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
control, complete, perr := controlFromPrefix(body)
|
||||
if perr != nil {
|
||||
return "", fmt.Errorf("parse deb control: %w", perr)
|
||||
}
|
||||
if complete {
|
||||
return control, nil
|
||||
}
|
||||
if full || n >= p.headerMax {
|
||||
return "", fmt.Errorf("control.tar not found within %d bytes of %s", n, downloadURL)
|
||||
}
|
||||
n *= 2
|
||||
if n > p.headerMax {
|
||||
n = p.headerMax
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// controlFromPrefix parses the ar members present in a front prefix of a .deb.
|
||||
// It returns the ./control paragraph once control.tar.* is fully covered
|
||||
// (complete=true); a prefix too short to cover it returns complete=false so the
|
||||
// caller can widen the range. Later members (data.tar.*) are ignored.
|
||||
func controlFromPrefix(prefix []byte) (control string, complete bool, err error) {
|
||||
const magic = "!<arch>\n"
|
||||
if len(prefix) < len(magic) {
|
||||
return "", false, nil
|
||||
}
|
||||
if string(prefix[:len(magic)]) != magic {
|
||||
return "", false, errors.New("not an ar archive")
|
||||
}
|
||||
off := len(magic)
|
||||
for {
|
||||
if off+60 > len(prefix) {
|
||||
return "", false, nil
|
||||
}
|
||||
hdr := prefix[off : off+60]
|
||||
off += 60
|
||||
name := strings.TrimSuffix(strings.TrimRight(string(hdr[0:16]), " "), "/")
|
||||
size, err := strconv.ParseInt(strings.TrimSpace(string(hdr[48:58])), 10, 64)
|
||||
if err != nil {
|
||||
return "", false, fmt.Errorf("bad ar size for %q: %w", name, err)
|
||||
}
|
||||
if strings.HasPrefix(name, "control.tar") {
|
||||
if off+int(size) > len(prefix) {
|
||||
return "", false, nil
|
||||
}
|
||||
tarBytes, err := decompress(name, prefix[off:off+int(size)])
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
c, err := readControlParagraph(tarBytes)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
}
|
||||
return c, true, nil
|
||||
}
|
||||
if off+int(size) > len(prefix) {
|
||||
return "", false, nil
|
||||
}
|
||||
off += int(size)
|
||||
if size%2 == 1 {
|
||||
off++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rangeGet returns the first n bytes of downloadURL. full is true when the
|
||||
// response body was shorter than n (i.e. we already have the whole object).
|
||||
func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, false)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=0-%d", n-1))
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, n))
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
full := int64(len(body)) < n
|
||||
return body, full, nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
hdr, err := p.githubHeaders(ctx, remote, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
copyHeaders(req, hdr)
|
||||
|
||||
if err := p.limiterWait(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
resp, err := p.client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, resp.Body); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
// assetPath is the package's location relative to github.com — the path the
|
||||
// backend releases_remote (base https://github.com) proxies. It doubles as the
|
||||
// deb_metadata key and the Filename field in the Packages index, so a .deb
|
||||
// download resolves back to this remote and redirects to the backend.
|
||||
func assetPath(asset ghAsset) string {
|
||||
u, err := url.Parse(asset.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimPrefix(u.Path, "/")
|
||||
}
|
||||
|
||||
func sha256FromDigest(digest string) (string, bool) {
|
||||
if strings.HasPrefix(digest, "sha256:") {
|
||||
return strings.TrimPrefix(digest, "sha256:"), true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// githubHeaders builds the outbound headers for a GitHub request, attaching a
|
||||
// bearer credential when one is available. A per-remote credential wins; absent
|
||||
// that, the process-wide server credential is used; absent both, the request is
|
||||
// unauthenticated.
|
||||
func (p *GitHubProvider) githubHeaders(ctx context.Context, remote models.Remote, api bool) (http.Header, error) {
|
||||
h := http.Header{}
|
||||
if api {
|
||||
h.Set("Accept", "application/vnd.github+json")
|
||||
h.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
}
|
||||
tok, err := p.githubToken(ctx, remote)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tok != "" {
|
||||
h.Set("Authorization", "Bearer "+tok)
|
||||
}
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// githubToken resolves the bearer token for a remote. Precedence: a per-remote
|
||||
// credential (password, then username) overrides the server credential.
|
||||
func (p *GitHubProvider) githubToken(ctx context.Context, remote models.Remote) (string, error) {
|
||||
if remote.Password != "" {
|
||||
return remote.Password, nil
|
||||
}
|
||||
if remote.Username != "" {
|
||||
return remote.Username, nil
|
||||
}
|
||||
if c := p.serverCredential(); c != nil {
|
||||
return c.Token(ctx)
|
||||
}
|
||||
return "", nil
|
||||
}
|
||||
|
||||
func (p *GitHubProvider) serverCredential() githubauth.Credential {
|
||||
if p.serverCred != nil {
|
||||
return p.serverCred
|
||||
}
|
||||
return githubauth.Server()
|
||||
}
|
||||
|
||||
func copyHeaders(req *http.Request, h http.Header) {
|
||||
for k, vals := range h {
|
||||
for _, v := range vals {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func compilePatterns(patterns []string) ([]*regexp.Regexp, error) {
|
||||
var out []*regexp.Regexp
|
||||
for _, p := range patterns {
|
||||
re, err := regexp.Compile(p)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid pattern %q: %w", p, err)
|
||||
}
|
||||
out = append(out, re)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func matchesAny(res []*regexp.Regexp, s string) bool {
|
||||
if len(res) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, re := range res {
|
||||
if re.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,462 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeStore is an in-memory provider.RemoteMetadataStore + DebMetadataReader
|
||||
// keyed by file_path, mirroring the (repo_name, file_path) uniqueness of the
|
||||
// real deb_metadata table.
|
||||
type fakeStore struct {
|
||||
mu sync.Mutex
|
||||
rows map[string]provider.DebMetadata
|
||||
}
|
||||
|
||||
func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.DebMetadata{}} }
|
||||
|
||||
func (f *fakeStore) InsertDebMetadata(_ context.Context, m *provider.DebMetadata) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if _, ok := f.rows[m.FilePath]; ok {
|
||||
return nil // ON CONFLICT DO NOTHING
|
||||
}
|
||||
f.rows[m.FilePath] = *m
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) DeleteDebMetadata(_ context.Context, _, filePath string) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.rows, filePath)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
|
||||
func (f *fakeStore) DeleteRPMMetadata(context.Context, string, string) error { return nil }
|
||||
func (f *fakeStore) ListRPMMetadataEntries(context.Context, string) ([]provider.RPMMetadata, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) ListDebMetadataEntries(ctx context.Context, _ string) ([]provider.DebMetadata, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
out := make([]provider.DebMetadata, 0, len(f.rows))
|
||||
for _, m := range f.rows {
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// githubFixture serves the releases API and the .deb asset downloads (with Range
|
||||
// support) for a set of packages. digest controls whether the asset carries a
|
||||
// sha256 digest (no-download path) or not (compute path).
|
||||
type githubFixture struct {
|
||||
srv *httptest.Server
|
||||
debBytes map[string][]byte
|
||||
rangeHit map[string]int
|
||||
fullHit map[string]int
|
||||
etag string
|
||||
releasesHit int
|
||||
notModHit int
|
||||
releaseAuth string
|
||||
assetAuth string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
|
||||
t.Helper()
|
||||
f := &githubFixture{
|
||||
debBytes: map[string][]byte{},
|
||||
rangeHit: map[string]int{},
|
||||
fullHit: map[string]int{},
|
||||
}
|
||||
f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/repos/acme/tools/releases", func(w http.ResponseWriter, r *http.Request) {
|
||||
page := r.URL.Query().Get("page")
|
||||
if page != "" && page != "1" {
|
||||
w.Write([]byte("[]"))
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
f.releasesHit++
|
||||
f.releaseAuth = r.Header.Get("Authorization")
|
||||
etag := f.etag
|
||||
if etag != "" && r.Header.Get("If-None-Match") == etag {
|
||||
f.notModHit++
|
||||
f.mu.Unlock()
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if etag != "" {
|
||||
w.Header().Set("ETag", etag)
|
||||
}
|
||||
var assets []map[string]any
|
||||
for name := range f.debBytes {
|
||||
a := map[string]any{
|
||||
"name": name,
|
||||
"size": len(f.debBytes[name]),
|
||||
"browser_download_url": f.srv.URL + "/acme/tools/releases/download/v1.2-3/" + name,
|
||||
}
|
||||
if withDigest {
|
||||
sum := sha256.Sum256(f.debBytes[name])
|
||||
a["digest"] = "sha256:" + hex.EncodeToString(sum[:])
|
||||
}
|
||||
assets = append(assets, a)
|
||||
}
|
||||
rel := []map[string]any{{"tag_name": "v1.2-3", "draft": false, "assets": assets}}
|
||||
json.NewEncoder(w).Encode(rel)
|
||||
})
|
||||
mux.HandleFunc("/acme/tools/releases/download/", func(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:]
|
||||
body, ok := f.debBytes[name]
|
||||
if !ok {
|
||||
http.Error(w, "not found", 404)
|
||||
return
|
||||
}
|
||||
rng := r.Header.Get("Range")
|
||||
f.mu.Lock()
|
||||
f.assetAuth = r.Header.Get("Authorization")
|
||||
if rng != "" {
|
||||
f.rangeHit[name]++
|
||||
} else {
|
||||
f.fullHit[name]++
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
if rng == "" {
|
||||
w.WriteHeader(200)
|
||||
w.Write(body)
|
||||
return
|
||||
}
|
||||
var end int
|
||||
fmt.Sscanf(rng, "bytes=0-%d", &end)
|
||||
if end >= len(body)-1 {
|
||||
end = len(body) - 1
|
||||
}
|
||||
w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
|
||||
w.Header().Set("Content-Length", strconv.Itoa(end+1))
|
||||
w.WriteHeader(http.StatusPartialContent)
|
||||
w.Write(body[:end+1])
|
||||
})
|
||||
f.srv = httptest.NewServer(mux)
|
||||
t.Cleanup(f.srv.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *githubFixture) remote() models.Remote {
|
||||
return models.Remote{
|
||||
Name: "acme-deb",
|
||||
PackageType: models.PackageGitHubDeb,
|
||||
BaseURL: f.srv.URL + "/repos/acme/tools",
|
||||
ReleasesRemote: "github",
|
||||
MutableTTL: 3600,
|
||||
}
|
||||
}
|
||||
|
||||
func newTestProvider() *GitHubProvider {
|
||||
p := newGitHubProvider()
|
||||
p.headerInitial = 32 // force the ranged-fetch retry loop against the tiny fixture
|
||||
p.headerMax = 1 << 20
|
||||
return p
|
||||
}
|
||||
|
||||
const demoPath = "acme/tools/releases/download/v1.2-3/demo_1.2-3_amd64.deb"
|
||||
|
||||
func TestGitHubScanDerivesControlFromPrefixAndDigest(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
|
||||
metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 metadata row, got %d", len(metas))
|
||||
}
|
||||
m := metas[0]
|
||||
if m.Name != "demo" || m.Version != "1.2-3" || m.Architecture != "amd64" {
|
||||
t.Fatalf("bad control fields: %+v", m)
|
||||
}
|
||||
if m.FilePath != demoPath {
|
||||
t.Fatalf("FilePath = %q, want %q", m.FilePath, demoPath)
|
||||
}
|
||||
if int(m.Size) != len(fx.debBytes["demo_1.2-3_amd64.deb"]) {
|
||||
t.Fatalf("Size = %d, want %d", m.Size, len(fx.debBytes["demo_1.2-3_amd64.deb"]))
|
||||
}
|
||||
sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
|
||||
if m.SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("SHA256 = %q, want digest", m.SHA256)
|
||||
}
|
||||
if m.ContentHash != "sha256:"+hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("ContentHash = %q", m.ContentHash)
|
||||
}
|
||||
if m.MD5 != "" {
|
||||
t.Fatalf("MD5 should be unset for metadata-only derive, got %q", m.MD5)
|
||||
}
|
||||
if fx.fullHit["demo_1.2-3_amd64.deb"] != 0 {
|
||||
t.Fatalf("expected no full download when digest present, got %d", fx.fullHit["demo_1.2-3_amd64.deb"])
|
||||
}
|
||||
if fx.rangeHit["demo_1.2-3_amd64.deb"] == 0 {
|
||||
t.Fatalf("expected ranged control fetch")
|
||||
}
|
||||
if !strings.Contains(m.Control, "Package: demo") {
|
||||
t.Fatalf("raw control not captured: %q", m.Control)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubChecksumComputedWhenDigestAbsent(t *testing.T) {
|
||||
fx := newGitHubFixture(t, false)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
metas, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
||||
if len(metas) != 1 {
|
||||
t.Fatalf("want 1 row, got %d", len(metas))
|
||||
}
|
||||
sum := sha256.Sum256(fx.debBytes["demo_1.2-3_amd64.deb"])
|
||||
if metas[0].SHA256 != hex.EncodeToString(sum[:]) {
|
||||
t.Fatalf("computed checksum mismatch: %q", metas[0].SHA256)
|
||||
}
|
||||
if fx.fullHit["demo_1.2-3_amd64.deb"] == 0 {
|
||||
t.Fatalf("expected a full download to compute sha256 when digest absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteIndexAndRedirect(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
// Release is served and triggers the initial scan.
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Release", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Release", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle Release")
|
||||
}
|
||||
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "Architectures:") {
|
||||
t.Fatalf("Release bad: code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "amd64") {
|
||||
t.Fatalf("Release missing arch: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
// Packages carries the package with a Filename that is the github-relative
|
||||
// download path (so it resolves back to this remote and redirects).
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
pkgs := rec.Body.String()
|
||||
if !strings.Contains(pkgs, "Package: demo") {
|
||||
t.Fatalf("Packages missing package: %s", pkgs)
|
||||
}
|
||||
if !strings.Contains(pkgs, "Filename: "+demoPath) {
|
||||
t.Fatalf("Packages missing/incorrect Filename: %s", pkgs)
|
||||
}
|
||||
if !strings.Contains(pkgs, "SHA256: ") {
|
||||
t.Fatalf("Packages missing SHA256: %s", pkgs)
|
||||
}
|
||||
if strings.Contains(pkgs, "MD5sum:") {
|
||||
t.Fatalf("Packages should omit empty MD5sum: %s", pkgs)
|
||||
}
|
||||
|
||||
// Packages.gz decompresses to the same content.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages.gz", proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle Packages.gz")
|
||||
}
|
||||
gz, err := gzip.NewReader(rec.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("gzip: %v", err)
|
||||
}
|
||||
unz, _ := io.ReadAll(gz)
|
||||
if !strings.Contains(string(unz), "Package: demo") {
|
||||
t.Fatalf("Packages.gz missing package: %s", unz)
|
||||
}
|
||||
|
||||
// InRelease/Release.gpg 404 (unsigned, consumed via [trusted=yes]).
|
||||
for _, sp := range []string{"InRelease", "Release.gpg"} {
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, sp, proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %s", sp)
|
||||
}
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("%s want 404, got %d", sp, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// A .deb request redirects to the backend releases_remote.
|
||||
rec = httptest.NewRecorder()
|
||||
req = httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/"+demoPath, nil)
|
||||
if !p.ServeRemote(rec, req, remote, demoPath, proxyBase, store) {
|
||||
t.Fatal("ServeRemote did not handle .deb")
|
||||
}
|
||||
if rec.Code != http.StatusFound {
|
||||
t.Fatalf("want 302, got %d", rec.Code)
|
||||
}
|
||||
wantLoc := proxyBase + "/api/v1/remote/github/" + demoPath
|
||||
if got := rec.Header().Get("Location"); got != wantLoc {
|
||||
t.Fatalf("Location = %q, want %q", got, wantLoc)
|
||||
}
|
||||
}
|
||||
|
||||
// Real apt appends the flat-repo dist "./" verbatim, so the metadata-only remote
|
||||
// receives "./Packages" / "./Release"; ServeRemote must collapse the dot-segment
|
||||
// and synthesize the same index as the un-prefixed request.
|
||||
func TestGitHubServeRemoteAptDotSegment(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
const proxyBase = "https://artifactapi.example"
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/"+path, nil)
|
||||
if !p.ServeRemote(rec, req, remote, path, proxyBase, store) {
|
||||
t.Fatalf("ServeRemote did not handle %q", path)
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
// Packages is deterministic: byte-identical to the un-prefixed request.
|
||||
plain, dotted := serve("Packages"), serve("./Packages")
|
||||
if plain.Code != 200 || dotted.Code != 200 {
|
||||
t.Fatalf("Packages: plain=%d dotted=%d, want 200/200", plain.Code, dotted.Code)
|
||||
}
|
||||
if !strings.Contains(dotted.Body.String(), "Package: demo") {
|
||||
t.Fatalf("./Packages missing synthesized body: %s", dotted.Body.String())
|
||||
}
|
||||
if !bytes.Equal(plain.Body.Bytes(), dotted.Body.Bytes()) {
|
||||
t.Error("./Packages body differs from Packages body")
|
||||
}
|
||||
|
||||
// Release carries a time.Now() Date: header; compare the rest.
|
||||
rPlain, rDotted := serve("Release"), serve("./Release")
|
||||
if rPlain.Code != 200 || rDotted.Code != 200 {
|
||||
t.Fatalf("Release: plain=%d dotted=%d, want 200/200", rPlain.Code, rDotted.Code)
|
||||
}
|
||||
if stripDate(rPlain.Body.String()) != stripDate(rDotted.Body.String()) {
|
||||
t.Error("./Release body differs from Release body (ignoring Date)")
|
||||
}
|
||||
}
|
||||
|
||||
// A canceled inbound request must still serve the warm cache (detached context),
|
||||
// not turn the metadata read into a 500.
|
||||
func TestGitHubServeRemoteCanceledRequestServesCache(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
p.mu.Lock()
|
||||
p.lastScan[remote.Name] = time.Now()
|
||||
p.mu.Unlock()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil).WithContext(ctx)
|
||||
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("canceled request must serve cache, not error; got code=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "Package: demo") {
|
||||
t.Fatalf("expected Packages served from cache, got %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubServeRemoteRedirectRequiresReleasesRemote(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.ReleasesRemote = ""
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/x", nil)
|
||||
if !p.ServeRemote(rec, req, remote, demoPath, "https://x", store) {
|
||||
t.Fatal("expected handled")
|
||||
}
|
||||
if rec.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("want 500 when releases_remote unset, got %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubScanPrunesRemovedAssets(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 1 {
|
||||
t.Fatalf("want 1 row after first scan, got %d", len(rows))
|
||||
}
|
||||
|
||||
delete(fx.debBytes, "demo_1.2-3_amd64.deb")
|
||||
if err := p.scan(context.Background(), fx.remote(), store); err != nil {
|
||||
t.Fatalf("rescan: %v", err)
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb"); len(rows) != 0 {
|
||||
t.Fatalf("want 0 rows after prune, got %d", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGitHubAssetPatternFilter(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.debBytes["other_9_arm64.deb"] = testsupport.MinimalDeb("other", "9", "arm64")
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
remote.Patterns = []string{`^demo_.*_amd64\.deb$`}
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
rows, _ := store.ListDebMetadataEntries(context.Background(), "acme-deb")
|
||||
if len(rows) != 1 || rows[0].Name != "demo" {
|
||||
t.Fatalf("pattern filter failed, rows=%+v", rows)
|
||||
}
|
||||
}
|
||||
@@ -1,238 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
const (
|
||||
syncLeaseDuration = 15 * time.Minute
|
||||
defaultSyncFreshness = 5 * time.Minute
|
||||
jobQueueDepth = 256
|
||||
)
|
||||
|
||||
// SyncStore is the persistence surface the deb syncer needs: the metadata cache
|
||||
// it primes plus the shared sync-state coordination (remote enumeration and the
|
||||
// per-remote lease). *database.DB satisfies it.
|
||||
type SyncStore interface {
|
||||
provider.RemoteMetadataStore
|
||||
ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
|
||||
ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
|
||||
ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
|
||||
}
|
||||
|
||||
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
|
||||
type SyncConfig struct {
|
||||
RatePerSec float64
|
||||
Burst int
|
||||
Workers int
|
||||
PollInterval time.Duration
|
||||
}
|
||||
|
||||
type syncJob struct {
|
||||
remote models.Remote
|
||||
prime bool
|
||||
}
|
||||
|
||||
// Syncer is the single per-process background worker that keeps every github_deb
|
||||
// remote's derived metadata fresh. It owns a deduped work queue, a pool of
|
||||
// workers, and a global token-bucket rate limiter shared across all remotes and
|
||||
// bound onto the github_deb provider. Periodic checks are gated by a shared DB
|
||||
// lease so, across replicas, only one performs each scan.
|
||||
type Syncer struct {
|
||||
store SyncStore
|
||||
prov *GitHubProvider
|
||||
limiter *rate.Limiter
|
||||
cfg SyncConfig
|
||||
owner string
|
||||
|
||||
jobs chan syncJob
|
||||
mu sync.Mutex
|
||||
active map[string]bool
|
||||
}
|
||||
|
||||
// NewSyncer builds the syncer bound to the process-wide github_deb provider
|
||||
// singleton. Call Run to start it.
|
||||
func NewSyncer(store SyncStore, cfg SyncConfig) *Syncer {
|
||||
return newSyncer(store, gitHubProvider, cfg)
|
||||
}
|
||||
|
||||
func newSyncer(store SyncStore, prov *GitHubProvider, cfg SyncConfig) *Syncer {
|
||||
if cfg.RatePerSec <= 0 {
|
||||
cfg.RatePerSec = 1
|
||||
}
|
||||
if cfg.Burst <= 0 {
|
||||
cfg.Burst = 5
|
||||
}
|
||||
if cfg.Workers <= 0 {
|
||||
cfg.Workers = 3
|
||||
}
|
||||
if cfg.PollInterval <= 0 {
|
||||
cfg.PollInterval = 60 * time.Second
|
||||
}
|
||||
|
||||
lim := rate.NewLimiter(rate.Limit(cfg.RatePerSec), cfg.Burst)
|
||||
s := &Syncer{
|
||||
store: store,
|
||||
prov: prov,
|
||||
limiter: lim,
|
||||
cfg: cfg,
|
||||
owner: leaseOwner(),
|
||||
jobs: make(chan syncJob, jobQueueDepth),
|
||||
active: map[string]bool{},
|
||||
}
|
||||
prov.limiter = lim
|
||||
prov.syncer = s
|
||||
return s
|
||||
}
|
||||
|
||||
// Run starts the worker pool and the periodic scheduler and blocks until ctx is
|
||||
// canceled, at which point it drains in-flight scans and returns.
|
||||
func (s *Syncer) Run(ctx context.Context) {
|
||||
slog.Info("github_deb syncer started",
|
||||
"rate_per_sec", s.cfg.RatePerSec, "burst", s.cfg.Burst,
|
||||
"workers", s.cfg.Workers, "poll_interval", s.cfg.PollInterval, "owner", s.owner)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < s.cfg.Workers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
s.worker(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(s.cfg.PollInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.schedule(ctx)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
wg.Wait()
|
||||
slog.Info("github_deb syncer stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.schedule(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// schedule enqueues a periodic check for every github_deb remote. The DB lease
|
||||
// enforces the per-remote mutable_ttl cadence and cross-replica coordination.
|
||||
func (s *Syncer) schedule(ctx context.Context) {
|
||||
remotes, err := s.store.ListGitHubDebRemotes(ctx)
|
||||
if err != nil {
|
||||
slog.Error("github_deb syncer: list remotes", "error", err)
|
||||
return
|
||||
}
|
||||
for _, r := range remotes {
|
||||
s.enqueue(r, false)
|
||||
}
|
||||
}
|
||||
|
||||
// EnqueuePrime queues an immediate background prime for a freshly created remote.
|
||||
func (s *Syncer) EnqueuePrime(remote models.Remote) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.enqueue(remote, true)
|
||||
}
|
||||
|
||||
// enqueue adds a job unless the remote is already queued or in-flight, coalescing
|
||||
// duplicate requests down to one scan. It never blocks.
|
||||
func (s *Syncer) enqueue(remote models.Remote, prime bool) {
|
||||
s.mu.Lock()
|
||||
if s.active[remote.Name] {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.active[remote.Name] = true
|
||||
s.mu.Unlock()
|
||||
|
||||
select {
|
||||
case s.jobs <- syncJob{remote: remote, prime: prime}:
|
||||
default:
|
||||
s.mu.Lock()
|
||||
delete(s.active, remote.Name)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Syncer) worker(ctx context.Context) {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case job := <-s.jobs:
|
||||
s.process(ctx, job)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// process claims the shared lease and, if won, runs an incremental scan. Losing
|
||||
// the claim (another replica scanning, or not yet due) is a no-op.
|
||||
func (s *Syncer) process(ctx context.Context, job syncJob) {
|
||||
defer func() {
|
||||
s.mu.Lock()
|
||||
delete(s.active, job.remote.Name)
|
||||
s.mu.Unlock()
|
||||
}()
|
||||
|
||||
freshness := time.Duration(job.remote.MutableTTL) * time.Second
|
||||
if freshness <= 0 {
|
||||
freshness = defaultSyncFreshness
|
||||
}
|
||||
if job.prime {
|
||||
freshness = 0
|
||||
}
|
||||
|
||||
claimed, etag, err := s.store.ClaimGitHubDebSyncLease(ctx, job.remote.Name, s.owner, freshness, syncLeaseDuration)
|
||||
if err != nil {
|
||||
slog.Error("github_deb syncer: claim lease", "remote", job.remote.Name, "error", err)
|
||||
return
|
||||
}
|
||||
if !claimed {
|
||||
return
|
||||
}
|
||||
|
||||
scanCtx, cancel := context.WithTimeout(ctx, s.prov.scanTimeout)
|
||||
defer cancel()
|
||||
|
||||
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
|
||||
releaseEtag := etag
|
||||
if scanErr == nil {
|
||||
releaseEtag = newEtag
|
||||
} else {
|
||||
slog.Error("github_deb syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
|
||||
}
|
||||
|
||||
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
|
||||
defer relCancel()
|
||||
if err := s.store.ReleaseGitHubDebSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
|
||||
slog.Warn("github_deb syncer: release lease", "remote", job.remote.Name, "error", err)
|
||||
}
|
||||
|
||||
if scanErr == nil && changed {
|
||||
slog.Info("github_deb syncer: refreshed", "remote", job.remote.Name, "prime", job.prime)
|
||||
}
|
||||
}
|
||||
|
||||
// leaseOwner is a per-replica identity for the lease: hostname plus a random
|
||||
// suffix so restarts and colocated replicas never collide.
|
||||
func leaseOwner() string {
|
||||
host, _ := os.Hostname()
|
||||
var b [6]byte
|
||||
_, _ = rand.Read(b[:])
|
||||
return host + "-" + hex.EncodeToString(b[:])
|
||||
}
|
||||
@@ -1,300 +0,0 @@
|
||||
package deb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"golang.org/x/time/rate"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/testsupport"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
// fakeSyncStore is an in-memory SyncStore: the metadata cache (via the embedded
|
||||
// fakeStore) plus the shared sync-state lease, whose claim mirrors the atomic
|
||||
// semantics of the real SQL (recency gate AND no live lease).
|
||||
type fakeSyncStore struct {
|
||||
*fakeStore
|
||||
|
||||
mu sync.Mutex
|
||||
remotes []models.Remote
|
||||
leaseOwner map[string]string
|
||||
leaseExp map[string]time.Time
|
||||
lastSynced map[string]time.Time
|
||||
etags map[string]string
|
||||
}
|
||||
|
||||
func newFakeSyncStore() *fakeSyncStore {
|
||||
return &fakeSyncStore{
|
||||
fakeStore: newFakeStore(),
|
||||
leaseOwner: map[string]string{},
|
||||
leaseExp: map[string]time.Time{},
|
||||
lastSynced: map[string]time.Time{},
|
||||
etags: map[string]string{},
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ListGitHubDebRemotes(_ context.Context) ([]models.Remote, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]models.Remote(nil), f.remotes...), nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner string, freshness, lease time.Duration) (bool, string, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
now := time.Now()
|
||||
ls, hasLS := f.lastSynced[name]
|
||||
exp, hasExp := f.leaseExp[name]
|
||||
freshOK := !hasLS || now.Sub(ls) >= freshness
|
||||
leaseOK := !hasExp || exp.Before(now)
|
||||
if freshOK && leaseOK {
|
||||
f.leaseOwner[name] = owner
|
||||
f.leaseExp[name] = now.Add(lease)
|
||||
return true, f.etags[name], nil
|
||||
}
|
||||
return false, "", nil
|
||||
}
|
||||
|
||||
func (f *fakeSyncStore) ReleaseGitHubDebSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.leaseOwner[name] != owner {
|
||||
return nil
|
||||
}
|
||||
f.lastSynced[name] = syncedAt
|
||||
f.etags[name] = etag
|
||||
delete(f.leaseOwner, name)
|
||||
delete(f.leaseExp, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func testSyncConfig() SyncConfig {
|
||||
return SyncConfig{RatePerSec: 1000, Burst: 100, Workers: 1, PollInterval: time.Hour}
|
||||
}
|
||||
|
||||
// (a) A 304 conditional response must derive nothing: no asset fetches and
|
||||
// changed=false, so an unchanged repo is nearly free.
|
||||
func TestSyncerConditionalNotModifiedSkipsDerive(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
etag1, changed, err := p.scanWithState(context.Background(), remote, store, "")
|
||||
if err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
if !changed || etag1 != `"v1"` {
|
||||
t.Fatalf("first scan changed=%v etag=%q, want true and \"v1\"", changed, etag1)
|
||||
}
|
||||
priorRange := fx.rangeHit["demo_1.2-3_amd64.deb"]
|
||||
if priorRange == 0 {
|
||||
t.Fatal("first scan should have fetched the asset control")
|
||||
}
|
||||
|
||||
etag2, changed2, err := p.scanWithState(context.Background(), remote, store, etag1)
|
||||
if err != nil {
|
||||
t.Fatalf("second scan: %v", err)
|
||||
}
|
||||
if changed2 {
|
||||
t.Fatal("304 scan must report changed=false")
|
||||
}
|
||||
if etag2 != etag1 {
|
||||
t.Fatalf("etag changed across 304: %q -> %q", etag1, etag2)
|
||||
}
|
||||
if fx.notModHit != 1 {
|
||||
t.Fatalf("want exactly one 304 releases response, got %d", fx.notModHit)
|
||||
}
|
||||
if got := fx.rangeHit["demo_1.2-3_amd64.deb"]; got != priorRange {
|
||||
t.Fatalf("304 scan re-fetched asset control: %d -> %d", priorRange, got)
|
||||
}
|
||||
}
|
||||
|
||||
// (b) On a real change, only the newly added asset is derived.
|
||||
func TestSyncerIncrementalDerivesOnlyNewAsset(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
p := newTestProvider()
|
||||
store := newFakeStore()
|
||||
remote := fx.remote()
|
||||
|
||||
if _, _, err := p.scanWithState(context.Background(), remote, store, ""); err != nil {
|
||||
t.Fatalf("first scan: %v", err)
|
||||
}
|
||||
demoRange := fx.rangeHit["demo_1.2-3_amd64.deb"]
|
||||
|
||||
fx.debBytes["other_9_arm64.deb"] = testsupport.MinimalDeb("other", "9", "arm64")
|
||||
fx.etag = `"v2"`
|
||||
|
||||
if _, changed, err := p.scanWithState(context.Background(), remote, store, `"v1"`); err != nil || !changed {
|
||||
t.Fatalf("second scan changed=%v err=%v", changed, err)
|
||||
}
|
||||
|
||||
rows, _ := store.ListDebMetadataEntries(context.Background(), remote.Name)
|
||||
if len(rows) != 2 {
|
||||
t.Fatalf("want 2 cached rows after incremental derive, got %d", len(rows))
|
||||
}
|
||||
if got := fx.rangeHit["demo_1.2-3_amd64.deb"]; got != demoRange {
|
||||
t.Fatalf("already-cached asset was re-fetched: %d -> %d", demoRange, got)
|
||||
}
|
||||
if fx.rangeHit["other_9_arm64.deb"] == 0 {
|
||||
t.Fatal("newly added asset was not derived")
|
||||
}
|
||||
}
|
||||
|
||||
// (c) The shared limiter caps the request rate.
|
||||
func TestRateLimiterCapsRequestRate(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
p := newTestProvider()
|
||||
p.limiter = rate.NewLimiter(rate.Every(120*time.Millisecond), 1)
|
||||
remote := fx.remote()
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, _, _, err := p.fetchReleases(context.Background(), remote, ""); err != nil {
|
||||
t.Fatalf("fetchReleases %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed < 200*time.Millisecond {
|
||||
t.Fatalf("rate limiter did not throttle: 3 calls took %v, want >= 200ms", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
// (d) Concurrent enqueues for the same remote coalesce to a single queued job.
|
||||
func TestSyncerEnqueueDedup(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-deb", PackageType: models.PackageGitHubDeb, MutableTTL: 3600}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); s.enqueue(remote, false) }()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if got := len(s.jobs); got != 1 {
|
||||
t.Fatalf("want exactly 1 coalesced job, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// (e) Prime-on-create enqueues a prime job.
|
||||
func TestSyncerEnqueuePrime(t *testing.T) {
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := models.Remote{Name: "acme-deb", PackageType: models.PackageGitHubDeb, MutableTTL: 3600}
|
||||
|
||||
s.EnqueuePrime(remote)
|
||||
select {
|
||||
case job := <-s.jobs:
|
||||
if !job.prime || job.remote.Name != "acme-deb" {
|
||||
t.Fatalf("bad prime job: %+v", job)
|
||||
}
|
||||
default:
|
||||
t.Fatal("EnqueuePrime did not enqueue a job")
|
||||
}
|
||||
}
|
||||
|
||||
// (f) A held lease prevents a second replica from scanning.
|
||||
func TestSyncerLeasePreventsSecondReplica(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
claimed, _, err := store.ClaimGitHubDebSyncLease(context.Background(), remote.Name, "replica-1", time.Duration(remote.MutableTTL)*time.Second, syncLeaseDuration)
|
||||
if err != nil || !claimed {
|
||||
t.Fatalf("replica-1 claim: claimed=%v err=%v", claimed, err)
|
||||
}
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote})
|
||||
|
||||
if fx.releasesHit != 0 {
|
||||
t.Fatalf("second replica scanned while lease held: %d releases calls", fx.releasesHit)
|
||||
}
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), remote.Name); len(rows) != 0 {
|
||||
t.Fatalf("second replica derived metadata while lease held: %d rows", len(rows))
|
||||
}
|
||||
}
|
||||
|
||||
// With the syncer wired and the cache empty, an index request enqueues a prime
|
||||
// and returns a retryable 503 when it has not landed within the cold wait.
|
||||
func TestServeRemoteColdStartReturns503(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
p.coldWait = 300 * time.Millisecond
|
||||
_ = newSyncer(store, p, testSyncConfig()) // binds p.syncer, but no workers running
|
||||
remote := fx.remote()
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("cold empty cache must return 503, got %d", rec.Code)
|
||||
}
|
||||
if rec.Header().Get("Retry-After") == "" {
|
||||
t.Fatal("503 should carry Retry-After")
|
||||
}
|
||||
if got := len(p.syncer.jobs); got != 1 {
|
||||
t.Fatalf("cold start did not enqueue a prime, jobs=%d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With the cache warm, the same request serves the index immediately (no 503).
|
||||
func TestServeRemoteWarmCacheServesImmediately(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
_ = newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
if err := p.scan(context.Background(), remote, store); err != nil {
|
||||
t.Fatalf("warm scan: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-deb/Packages", nil)
|
||||
if !p.ServeRemote(rec, req, remote, "Packages", "https://x", store) {
|
||||
t.Fatal("ServeRemote did not handle Packages")
|
||||
}
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("warm cache must serve 200, got %d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// A prime job (freshness 0) runs even right after a sync; a periodic job at the
|
||||
// same moment is gated by the recency window.
|
||||
func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
|
||||
fx := newGitHubFixture(t, true)
|
||||
fx.etag = `"v1"`
|
||||
store := newFakeSyncStore()
|
||||
p := newTestProvider()
|
||||
s := newSyncer(store, p, testSyncConfig())
|
||||
remote := fx.remote()
|
||||
|
||||
var _ provider.RemoteMetadataStore = store
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: true})
|
||||
if rows, _ := store.ListDebMetadataEntries(context.Background(), remote.Name); len(rows) != 1 {
|
||||
t.Fatalf("prime did not derive: %d rows", len(rows))
|
||||
}
|
||||
releasesAfterPrime := fx.releasesHit
|
||||
|
||||
s.process(context.Background(), syncJob{remote: remote, prime: false})
|
||||
if fx.releasesHit != releasesAfterPrime {
|
||||
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
@@ -62,7 +61,6 @@ type PostDeleteHook interface {
|
||||
|
||||
type MetadataStore interface {
|
||||
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
|
||||
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
|
||||
}
|
||||
|
||||
// RemoteServer lets a remote provider fully answer a request itself instead of
|
||||
@@ -85,85 +83,12 @@ type RemoteMetadataStore interface {
|
||||
|
||||
type MetadataDeleter interface {
|
||||
DeleteRPMMetadata(ctx context.Context, repoName, filePath string) error
|
||||
DeleteDebMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type RPMMetadataReader interface {
|
||||
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
|
||||
}
|
||||
|
||||
// DebMetadataReader is the read surface the deb LocalIndexer needs to
|
||||
// regenerate a flat apt repository (Packages/Release) from stored rows.
|
||||
// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it,
|
||||
// mirroring how the rpm provider reaches its RPMMetadataReader.
|
||||
type DebMetadataReader interface {
|
||||
ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error)
|
||||
}
|
||||
|
||||
// DebMetadata is the derived per-package metadata for a Debian .deb, carrying
|
||||
// the full raw control stanza so the Packages index can be regenerated
|
||||
// faithfully alongside the computed size/md5/sha256 apt requires.
|
||||
type DebMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Name string
|
||||
Version string
|
||||
Architecture string
|
||||
Control string
|
||||
Size int64
|
||||
MD5 string
|
||||
SHA256 string
|
||||
// CreatedAt is the persisted insert time; the Release Date: is derived from
|
||||
// the newest value so the index is byte-identical across replicas and
|
||||
// regenerations (issue #117) rather than stamped from wall clock.
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the
|
||||
// Alpine-specific persistence surfaces. They are kept separate from the shared
|
||||
// RPM/Deb metadata interfaces so the apk provider can type-assert the generic
|
||||
// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and
|
||||
// thus perturbing the test doubles of) the rpm and deb providers. *database.DB
|
||||
// satisfies all three.
|
||||
type AlpineMetadataStore interface {
|
||||
InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error
|
||||
}
|
||||
|
||||
type AlpineMetadataDeleter interface {
|
||||
DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error
|
||||
}
|
||||
|
||||
type AlpineMetadataReader interface {
|
||||
ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error)
|
||||
}
|
||||
|
||||
// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding
|
||||
// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the
|
||||
// sha1 of the control gzip stream) and the download/installed sizes.
|
||||
type AlpineMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
ContentHash string
|
||||
Checksum string // C: "Q1" + base64(sha1(control gzip stream))
|
||||
Name string // P:
|
||||
Version string // V:
|
||||
Arch string // A:
|
||||
DownloadSize int64 // S: on-disk .apk size
|
||||
InstalledSize int64 // I: unpacked size from .PKGINFO
|
||||
Description string // T:
|
||||
URL string // U:
|
||||
License string // L:
|
||||
Origin string // o:
|
||||
Maintainer string // m:
|
||||
BuildTime int64 // t:
|
||||
Commit string // c:
|
||||
ProviderPriority string // k:
|
||||
Depends []string // D:
|
||||
Provides []string // p:
|
||||
InstallIf []string // i:
|
||||
}
|
||||
|
||||
type RPMMetadata struct {
|
||||
RepoName string
|
||||
FilePath string
|
||||
@@ -190,9 +115,6 @@ type RPMMetadata struct {
|
||||
Obsoletes []RPMDep
|
||||
Files []RPMFile
|
||||
Changelogs []RPMChangelog
|
||||
// CreatedAt is the persisted upload timestamp; used as a stable, replica-independent
|
||||
// value for the repodata <time>/<revision> fields so generated indexes are deterministic.
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
type RPMDep struct {
|
||||
|
||||
@@ -47,9 +47,6 @@ func (f *fakeStore) DeleteRPMMetadata(_ context.Context, _, filePath string) err
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
func (f *fakeStore) DeleteDebMetadata(context.Context, string, string) error { return nil }
|
||||
|
||||
func (f *fakeStore) ListRPMMetadataEntries(ctx context.Context, _ string) ([]provider.RPMMetadata, error) {
|
||||
// Mirror pgx: a canceled/expired context fails the read. This is what
|
||||
// poisons the repodata response if the read runs on the inbound request.
|
||||
|
||||
@@ -275,7 +275,7 @@ func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader pr
|
||||
filelistsHash := sha256Hex(filelists)
|
||||
otherHash := sha256Hex(other)
|
||||
|
||||
repomd := generateRepomd(repomdRevision(metas), primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
|
||||
repomd := generateRepomd(primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
|
||||
|
||||
w.Header().Set("Content-Type", "application/xml")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
@@ -315,32 +315,8 @@ func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader pro
|
||||
w.Write(generateOtherXMLGZ(metas))
|
||||
}
|
||||
|
||||
// stableUnix maps a persisted timestamp to a fixed integer for repodata's
|
||||
// informational <time>/<timestamp> fields. Zero times (unset) collapse to 0 so
|
||||
// output stays byte-identical across replicas and requests. dnf does not
|
||||
// validate these values.
|
||||
func stableUnix(t time.Time) int64 {
|
||||
if t.IsZero() {
|
||||
return 0
|
||||
}
|
||||
return t.Unix()
|
||||
}
|
||||
|
||||
// repomdRevision derives repomd.xml's <revision>/<timestamp> from persisted
|
||||
// state: the newest package upload time in the repo. It changes only when the
|
||||
// repo's package set does, and is identical on every replica reading the same
|
||||
// rows, so repomd.xml is byte-stable.
|
||||
func repomdRevision(metas []provider.RPMMetadata) string {
|
||||
var max int64
|
||||
for _, m := range metas {
|
||||
if u := stableUnix(m.CreatedAt); u > max {
|
||||
max = u
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%d", max)
|
||||
}
|
||||
|
||||
func generateRepomd(ts string, primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
|
||||
func generateRepomd(primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
|
||||
ts := fmt.Sprintf("%d", time.Now().Unix())
|
||||
var b bytes.Buffer
|
||||
b.WriteString(xml.Header)
|
||||
b.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
|
||||
@@ -383,7 +359,7 @@ func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
|
||||
if m.URL != "" {
|
||||
fmt.Fprintf(&xmlBuf, " <url>%s</url>\n", xmlEscape(m.URL))
|
||||
}
|
||||
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", stableUnix(m.CreatedAt))
|
||||
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", time.Now().Unix())
|
||||
fmt.Fprintf(&xmlBuf, " <size package=\"%d\" installed=\"%d\" archive=\"0\"/>\n", m.RPMSize, m.InstalledSize)
|
||||
fmt.Fprintf(&xmlBuf, " <location href=\"%s\"/>\n", xmlEscape(m.FilePath))
|
||||
fmt.Fprintf(&xmlBuf, " <format>\n")
|
||||
@@ -508,9 +484,6 @@ func xmlEscape(s string) string {
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
// Pin every header field so the compressed bytes (and their sha256) depend
|
||||
// only on the payload, never on wall-clock time or the Go version's gzip defaults.
|
||||
gz.Header = gzip.Header{OS: 255}
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
|
||||
@@ -1,148 +0,0 @@
|
||||
package rpm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
)
|
||||
|
||||
func gunzip(t *testing.T, data []byte) string {
|
||||
t.Helper()
|
||||
zr, err := gzip.NewReader(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatalf("gzip reader: %v", err)
|
||||
}
|
||||
out, err := io.ReadAll(zr)
|
||||
if err != nil {
|
||||
t.Fatalf("gunzip: %v", err)
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// sampleMetas returns a fixed two-package repo state whose upload timestamps are
|
||||
// pinned, so any nondeterminism must come from the generators themselves.
|
||||
func sampleMetas() []provider.RPMMetadata {
|
||||
base := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
return []provider.RPMMetadata{
|
||||
{
|
||||
Name: "alpha", Version: "1.0", Release: "1", Arch: "x86_64",
|
||||
Summary: "a", Description: "d", ContentHash: "sha256:aaa",
|
||||
FilePath: "Packages/alpha-1.0-1.x86_64.rpm", RPMSize: 10, InstalledSize: 20,
|
||||
Provides: []provider.RPMDep{{Name: "alpha"}},
|
||||
Requires: []provider.RPMDep{{Name: "libc", Flags: "GE", Version: "2.0"}},
|
||||
CreatedAt: base,
|
||||
},
|
||||
{
|
||||
Name: "beta", Version: "2.0", Release: "3", Arch: "noarch",
|
||||
Summary: "b", Description: "d2", ContentHash: "sha256:bbb",
|
||||
FilePath: "Packages/beta-2.0-3.noarch.rpm", RPMSize: 30, InstalledSize: 40,
|
||||
CreatedAt: base.Add(time.Hour),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TestRepodataGeneratorsDeterministic is the direct regression guard for #117:
|
||||
// generating each metadata document twice from identical state must yield
|
||||
// byte-identical output (hence an identical sha256). The old code embedded
|
||||
// time.Now() inside primary.xml.gz, so its bytes/hash drifted every second.
|
||||
func TestRepodataGeneratorsDeterministic(t *testing.T) {
|
||||
metas := sampleMetas()
|
||||
gens := map[string]func([]provider.RPMMetadata) []byte{
|
||||
"primary": generatePrimaryXMLGZ,
|
||||
"filelists": generateFilelistsXMLGZ,
|
||||
"other": generateOtherXMLGZ,
|
||||
}
|
||||
for name, gen := range gens {
|
||||
a := gen(metas)
|
||||
b := gen(metas)
|
||||
if sha256Hex(a) != sha256Hex(b) {
|
||||
t.Errorf("%s: sha256 differs between two generations (nondeterministic): %s != %s",
|
||||
name, sha256Hex(a), sha256Hex(b))
|
||||
}
|
||||
}
|
||||
|
||||
// repomd.xml itself must also be byte-stable across regenerations.
|
||||
r1 := generateRepomd(repomdRevision(metas), sha256Hex(generatePrimaryXMLGZ(metas)), 1, "f", 2, "o", 3)
|
||||
r2 := generateRepomd(repomdRevision(metas), sha256Hex(generatePrimaryXMLGZ(metas)), 1, "f", 2, "o", 3)
|
||||
if string(r1) != string(r2) {
|
||||
t.Error("repomd.xml differs between two generations")
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrimaryTimeUsesPersistedCreatedAt proves the <time> element is a pure
|
||||
// function of the persisted upload timestamp, not the wall clock.
|
||||
func TestPrimaryTimeUsesPersistedCreatedAt(t *testing.T) {
|
||||
metas := sampleMetas()
|
||||
out := gunzip(t, generatePrimaryXMLGZ(metas))
|
||||
if want := `<time file="1767323045" build="0"/>`; !strings.Contains(out, want) {
|
||||
t.Errorf("primary.xml missing persisted <time> %q; got:\n%s", want, out)
|
||||
}
|
||||
// A zero (unset) CreatedAt collapses to a fixed 0, never a live clock value.
|
||||
metas[0].CreatedAt = time.Time{}
|
||||
out = gunzip(t, generatePrimaryXMLGZ(metas))
|
||||
if !strings.Contains(out, `<time file="0" build="0"/>`) {
|
||||
t.Errorf("zero CreatedAt should emit file=\"0\"; got:\n%s", out)
|
||||
}
|
||||
}
|
||||
|
||||
type repomdDoc struct {
|
||||
Revision string `xml:"revision"`
|
||||
Data []struct {
|
||||
Type string `xml:"type,attr"`
|
||||
Checksum struct {
|
||||
Value string `xml:",chardata"`
|
||||
} `xml:"checksum"`
|
||||
Location struct {
|
||||
Href string `xml:"href,attr"`
|
||||
} `xml:"location"`
|
||||
} `xml:"data"`
|
||||
}
|
||||
|
||||
// TestRepomdHashMatchesServedBytes asserts the exact invariant #117 violated:
|
||||
// the sha256 advertised in repomd.xml equals the sha256 of the bytes the
|
||||
// content-addressed serve* handler returns for the same repo state.
|
||||
func TestRepomdHashMatchesServedBytes(t *testing.T) {
|
||||
p := &Provider{}
|
||||
reader := fakeRPMReader{metas: sampleMetas()}
|
||||
|
||||
serve := func(path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
r := httptest.NewRequest(http.MethodGet, "/"+path, nil)
|
||||
if !p.ServeLocalIndex(w, r, reader, "repo", path) {
|
||||
t.Fatalf("ServeLocalIndex false for %q", path)
|
||||
}
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("%s: code %d", path, w.Code)
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
var doc repomdDoc
|
||||
if err := xml.Unmarshal(serve("repodata/repomd.xml").Body.Bytes(), &doc); err != nil {
|
||||
t.Fatalf("parse repomd: %v", err)
|
||||
}
|
||||
if len(doc.Data) != 3 {
|
||||
t.Fatalf("expected 3 <data> entries, got %d", len(doc.Data))
|
||||
}
|
||||
|
||||
for _, d := range doc.Data {
|
||||
// The advertised location is content-addressed: repodata/<sha256>-<type>.xml.gz.
|
||||
body := serve("repodata/" + d.Location.Href[len("repodata/"):]).Body.Bytes()
|
||||
got := sha256Hex(body)
|
||||
if got != d.Checksum.Value {
|
||||
t.Errorf("%s: repomd advertises %s but served bytes hash to %s (dnf would reject)",
|
||||
d.Type, d.Checksum.Value, got)
|
||||
}
|
||||
if d.Location.Href != "repodata/"+d.Checksum.Value+"-"+d.Type+".xml.gz" {
|
||||
t.Errorf("%s: location %q not addressed by its checksum %s", d.Type, d.Location.Href, d.Checksum.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,8 +28,6 @@ func (f *fakeMetaStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMeta
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeMetaStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
|
||||
|
||||
type fakeRPMReader struct{ metas []provider.RPMMetadata }
|
||||
|
||||
func (f fakeRPMReader) ListRPMMetadataEntries(_ context.Context, _ string) ([]provider.RPMMetadata, error) {
|
||||
|
||||
@@ -20,8 +20,7 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
|
||||
@@ -35,7 +34,6 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/storage"
|
||||
"git.unkin.net/unkin/artifactapi/internal/tfsign"
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
@@ -51,8 +49,6 @@ type Server struct {
|
||||
tfRegistry *tfregistry.Handler
|
||||
gc *gc.Collector
|
||||
syncer *rpm.Syncer
|
||||
debSyncer *deb.Syncer
|
||||
alpineSyncer *alpine.Syncer
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, version string) (*Server, error) {
|
||||
@@ -100,18 +96,6 @@ func New(cfg *config.Config, version string) (*Server, error) {
|
||||
Workers: cfg.GitHubSyncWorkers,
|
||||
PollInterval: time.Duration(cfg.GitHubSyncPollInterval) * time.Second,
|
||||
})
|
||||
debSyncer := deb.NewSyncer(db, deb.SyncConfig{
|
||||
RatePerSec: cfg.GitHubSyncRatePerSec,
|
||||
Burst: cfg.GitHubSyncBurst,
|
||||
Workers: cfg.GitHubSyncWorkers,
|
||||
PollInterval: time.Duration(cfg.GitHubSyncPollInterval) * time.Second,
|
||||
})
|
||||
alpineSyncer := alpine.NewSyncer(db, alpine.SyncConfig{
|
||||
RatePerSec: cfg.GitHubSyncRatePerSec,
|
||||
Burst: cfg.GitHubSyncBurst,
|
||||
Workers: cfg.GitHubSyncWorkers,
|
||||
PollInterval: time.Duration(cfg.GitHubSyncPollInterval) * time.Second,
|
||||
})
|
||||
|
||||
// The terraform registry signs with a GPG key. A configured file wins (BYO
|
||||
// key); otherwise artifactapi generates one on first start and persists it in
|
||||
@@ -144,8 +128,6 @@ func New(cfg *config.Config, version string) (*Server, error) {
|
||||
tfRegistry: tfRegistry,
|
||||
gc: collector,
|
||||
syncer: syncer,
|
||||
debSyncer: debSyncer,
|
||||
alpineSyncer: alpineSyncer,
|
||||
}
|
||||
|
||||
s.router = s.routes()
|
||||
@@ -175,11 +157,7 @@ func (s *Server) routes() chi.Router {
|
||||
r.Mount("/api/v1", proxyHandler.Routes())
|
||||
r.Mount("/v2", proxyHandler.DockerV2Routes())
|
||||
|
||||
remotesHandler := v2.NewRemotesHandler(s.db, map[models.PackageType]v2.Primer{
|
||||
models.PackageGitHubRPM: s.syncer,
|
||||
models.PackageGitHubDeb: s.debSyncer,
|
||||
models.PackageGitHubAlpine: s.alpineSyncer,
|
||||
})
|
||||
remotesHandler := v2.NewRemotesHandler(s.db, s.syncer)
|
||||
virtualsHandler := v2.NewVirtualsHandler(s.db)
|
||||
healthHandler := v2.NewHealthHandler(s.db, s.cache, s.store)
|
||||
statsHandler := v2.NewStatsHandler(s.db)
|
||||
@@ -247,8 +225,6 @@ func (s *Server) newHTTPServer() *http.Server {
|
||||
func (s *Server) Run(ctx context.Context) error {
|
||||
go s.gc.Run(ctx)
|
||||
go s.syncer.Run(ctx)
|
||||
go s.debSyncer.Run(ctx)
|
||||
go s.alpineSyncer.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
@@ -270,8 +246,6 @@ func (s *Server) Run(ctx context.Context) error {
|
||||
func (s *Server) RunOnListener(ctx context.Context, ln net.Listener) error {
|
||||
go s.gc.Run(ctx)
|
||||
go s.syncer.Run(ctx)
|
||||
go s.debSyncer.Run(ctx)
|
||||
go s.alpineSyncer.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
package testsupport
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// MinimalApk builds a valid-enough Alpine package in pure Go (no committed
|
||||
// binary fixture, no abuild): two concatenated, independently gzipped tar
|
||||
// streams -- a control stream carrying .PKGINFO and a data stream carrying a
|
||||
// single payload file. It mirrors MinimalDeb/MinimalRPM and is parseable by the
|
||||
// alpine provider (which derives arch/name/version and the Q1 pull checksum from
|
||||
// the control stream).
|
||||
func MinimalApk(name, version, arch string) []byte {
|
||||
pkginfo := fmt.Sprintf(
|
||||
"# generated by testsupport\n"+
|
||||
"pkgname = %s\n"+
|
||||
"pkgver = %s\n"+
|
||||
"arch = %s\n"+
|
||||
"pkgdesc = minimal test package\n"+
|
||||
"url = https://example.com/%s\n"+
|
||||
"license = MIT\n"+
|
||||
"origin = %s\n"+
|
||||
"maintainer = e2e <e2e@example.com>\n"+
|
||||
"builddate = 1700000000\n"+
|
||||
"size = 4\n"+
|
||||
"depend = so:libc.musl-x86_64.so.1\n"+
|
||||
"provides = cmd:%s=%s\n",
|
||||
name, version, arch, name, name, name, version)
|
||||
|
||||
control := gzipBytes(tarSingle(".PKGINFO", []byte(pkginfo)))
|
||||
data := gzipBytes(tarSingle("usr/bin/"+name, []byte("body")))
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.Write(control)
|
||||
buf.Write(data)
|
||||
return buf.Bytes()
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package testsupport
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// MinimalDeb builds a valid-enough Debian package in pure Go (no committed
|
||||
// binary fixture, no dpkg-deb): an ar archive of debian-binary, a gzip
|
||||
// control.tar.gz carrying ./control, and an (empty) gzip data.tar.gz. It is the
|
||||
// deb analog of MinimalRPM and is parseable by the deb provider.
|
||||
func MinimalDeb(name, version, arch string) []byte {
|
||||
control := fmt.Sprintf(
|
||||
"Package: %s\nVersion: %s\nArchitecture: %s\nMaintainer: e2e <e2e@example.com>\n"+
|
||||
"Section: utils\nPriority: optional\nDescription: minimal test package\n",
|
||||
name, version, arch)
|
||||
|
||||
controlTarGz := gzipBytes(tarSingle("./control", []byte(control)))
|
||||
dataTarGz := gzipBytes(tarEmpty())
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.WriteString("!<arch>\n")
|
||||
arWrite(&buf, "debian-binary", []byte("2.0\n"))
|
||||
arWrite(&buf, "control.tar.gz", controlTarGz)
|
||||
arWrite(&buf, "data.tar.gz", dataTarGz)
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func arWrite(buf *bytes.Buffer, name string, data []byte) {
|
||||
fmt.Fprintf(buf, "%-16s%-12s%-6s%-6s%-8s%-10d`\n", name, "0", "0", "0", "100644", len(data))
|
||||
buf.Write(data)
|
||||
if len(data)%2 == 1 {
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
|
||||
func tarSingle(name string, data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(data)), Typeflag: tar.TypeReg})
|
||||
tw.Write(data)
|
||||
tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func tarEmpty() []byte {
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
tw.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func gzipBytes(data []byte) []byte {
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
gz.Write(data)
|
||||
gz.Close()
|
||||
return buf.Bytes()
|
||||
}
|
||||
+22
-28
@@ -5,37 +5,31 @@ import "fmt"
|
||||
type PackageType string
|
||||
|
||||
const (
|
||||
PackageGeneric PackageType = "generic"
|
||||
PackageDocker PackageType = "docker"
|
||||
PackageHelm PackageType = "helm"
|
||||
PackagePyPI PackageType = "pypi"
|
||||
PackageNPM PackageType = "npm"
|
||||
PackageRPM PackageType = "rpm"
|
||||
PackageDeb PackageType = "deb"
|
||||
PackageAlpine PackageType = "alpine"
|
||||
PackagePuppet PackageType = "puppet"
|
||||
PackageTerraform PackageType = "terraform"
|
||||
PackageGoProxy PackageType = "goproxy"
|
||||
PackageGitHubRPM PackageType = "github_rpm"
|
||||
PackageGitHubDeb PackageType = "github_deb"
|
||||
PackageGitHubAlpine PackageType = "github_alpine"
|
||||
PackageGeneric PackageType = "generic"
|
||||
PackageDocker PackageType = "docker"
|
||||
PackageHelm PackageType = "helm"
|
||||
PackagePyPI PackageType = "pypi"
|
||||
PackageNPM PackageType = "npm"
|
||||
PackageRPM PackageType = "rpm"
|
||||
PackageAlpine PackageType = "alpine"
|
||||
PackagePuppet PackageType = "puppet"
|
||||
PackageTerraform PackageType = "terraform"
|
||||
PackageGoProxy PackageType = "goproxy"
|
||||
PackageGitHubRPM PackageType = "github_rpm"
|
||||
)
|
||||
|
||||
var validPackageTypes = map[PackageType]bool{
|
||||
PackageGeneric: true,
|
||||
PackageDocker: true,
|
||||
PackageHelm: true,
|
||||
PackagePyPI: true,
|
||||
PackageNPM: true,
|
||||
PackageRPM: true,
|
||||
PackageDeb: true,
|
||||
PackageAlpine: true,
|
||||
PackagePuppet: true,
|
||||
PackageTerraform: true,
|
||||
PackageGoProxy: true,
|
||||
PackageGitHubRPM: true,
|
||||
PackageGitHubDeb: true,
|
||||
PackageGitHubAlpine: true,
|
||||
PackageGeneric: true,
|
||||
PackageDocker: true,
|
||||
PackageHelm: true,
|
||||
PackagePyPI: true,
|
||||
PackageNPM: true,
|
||||
PackageRPM: true,
|
||||
PackageAlpine: true,
|
||||
PackagePuppet: true,
|
||||
PackageTerraform: true,
|
||||
PackageGoProxy: true,
|
||||
PackageGitHubRPM: true,
|
||||
}
|
||||
|
||||
func (p PackageType) Valid() bool {
|
||||
|
||||
@@ -19,8 +19,6 @@ func TestPackageTypeValid(t *testing.T) {
|
||||
models.PackageTerraform,
|
||||
models.PackageGoProxy,
|
||||
models.PackageGitHubRPM,
|
||||
models.PackageGitHubDeb,
|
||||
models.PackageGitHubAlpine,
|
||||
}
|
||||
for _, pt := range valid {
|
||||
if !pt.Valid() {
|
||||
|
||||
@@ -176,44 +176,14 @@ helm install <release> ${name}/<chart>`,
|
||||
];
|
||||
|
||||
case 'alpine':
|
||||
return isLocal
|
||||
? [
|
||||
{
|
||||
title: 'Add the apk repo (real apk repo, APKINDEX auto-generated)',
|
||||
language: 'bash',
|
||||
code: `echo '${url}/api/v1/local/${name}' | sudo tee -a /etc/apk/repositories
|
||||
sudo apk update --allow-untrusted
|
||||
sudo apk add --allow-untrusted <package>`,
|
||||
note: `Served unsigned (parity with the rpm repo's gpgcheck=0) — use --allow-untrusted, or install a signing key. apk fetches <arch>/APKINDEX.tar.gz under this base.`,
|
||||
},
|
||||
{
|
||||
title: 'Publish a .apk (index regenerates automatically)',
|
||||
language: 'bash',
|
||||
code: `curl -fsSL --upload-file ./mypkg-1.0-r0.apk \\
|
||||
${url}/api/v2/remotes/${name}/files/x86_64/mypkg-1.0-r0.apk`,
|
||||
note: 'Upload each package at <arch>/<name>-<version>.apk — apk reconstructs that exact path from the index (APKINDEX carries no filename), so a mismatched path will 404 on install.',
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
title: 'Add the APK repository',
|
||||
language: 'bash',
|
||||
code: `echo '${proxy}/' | sudo tee -a /etc/apk/repositories
|
||||
sudo apk update
|
||||
sudo apk add <package>`,
|
||||
note: 'If the index is unsigned over the proxy, add --allow-untrusted or install the signing key into /etc/apk/keys.',
|
||||
},
|
||||
];
|
||||
|
||||
case 'github_alpine':
|
||||
return [
|
||||
{
|
||||
title: 'Add the apk repo (metadata-only, from GitHub releases)',
|
||||
title: 'Add the APK repository',
|
||||
language: 'bash',
|
||||
code: `echo '${proxy}' | sudo tee -a /etc/apk/repositories
|
||||
sudo apk update --allow-untrusted
|
||||
sudo apk add --allow-untrusted <package>`,
|
||||
note: "The per-arch APKINDEX is synthesized from the configured GitHub repo's release .apk assets; package downloads are redirected to the backing releases remote. Served unsigned, so --allow-untrusted.",
|
||||
code: `echo '${proxy}/' | sudo tee -a /etc/apk/repositories
|
||||
sudo apk update
|
||||
sudo apk add <package>`,
|
||||
note: 'If the index is unsigned over the proxy, add --allow-untrusted or install the signing key into /etc/apk/keys.',
|
||||
},
|
||||
];
|
||||
|
||||
@@ -238,47 +208,6 @@ go mod download`,
|
||||
},
|
||||
];
|
||||
|
||||
case 'deb':
|
||||
return isLocal
|
||||
? [
|
||||
{
|
||||
title: 'Add the apt repo (real apt repo, flat — Packages/Release auto-generated)',
|
||||
language: 'bash',
|
||||
code: `echo 'deb [trusted=yes] ${url}/api/v1/local/${name}/ ./' | sudo tee /etc/apt/sources.list.d/${name}.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install <package>`,
|
||||
note: '[trusted=yes]: artifactapi serves the flat repo unsigned (matches the rpm repo\'s gpgcheck=0). The `./` is the flat-repo suite — apt fetches Packages/Release from the repo root.',
|
||||
},
|
||||
{
|
||||
title: 'Publish a .deb (index regenerates automatically)',
|
||||
language: 'bash',
|
||||
code: `curl -fsSL --upload-file ./my-package_1.0_amd64.deb \\
|
||||
${url}/api/v2/remotes/${name}/files/my-package_1.0_amd64.deb`,
|
||||
},
|
||||
]
|
||||
: [
|
||||
{
|
||||
title: 'Add the apt repo (caching proxy)',
|
||||
language: 'bash',
|
||||
code: `echo 'deb ${proxy} <suite> <component>' | sudo tee /etc/apt/sources.list.d/${name}.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install <package>`,
|
||||
note: "Signatures are verified against the upstream mirror's real signed Release through the proxy (no [trusted=yes] needed). Example suite/component: bookworm main.",
|
||||
},
|
||||
];
|
||||
|
||||
case 'github_deb':
|
||||
return [
|
||||
{
|
||||
title: 'Add the apt repo (metadata-only, from GitHub releases)',
|
||||
language: 'bash',
|
||||
code: `echo 'deb [trusted=yes] ${proxy}/ ./' | sudo tee /etc/apt/sources.list.d/${name}.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install <package>`,
|
||||
note: "The apt index is synthesized from the configured GitHub repo's release .deb assets; package downloads are redirected to the backing releases remote. Served unsigned, so [trusted=yes].",
|
||||
},
|
||||
];
|
||||
|
||||
case 'generic':
|
||||
default:
|
||||
return [
|
||||
|
||||
Reference in New Issue
Block a user