Add github_deb metadata-only package type (#112)
ci/woodpecker/tag/docker Pipeline was successful

## Why

This stacks the Debian/apt analog of `github_rpm` on top of the deb local+remote work (#111). It lets a GitHub repo's `.deb` release assets be consumed as a real apt repository without artifactapi ever precaching whole packages: it derives per-asset control metadata from a ranged prefix fetch, synthesizes a flat apt repo from the cache, and redirects the actual `.deb` downloads to a backend `releases_remote` (the generic github.com remote).

Base is `benvin/deb-local-remote` (stacked) to keep the diff atomic.

## How

- Adds `github_deb` to the package-type enum and validity map.
- Adds the `github_deb` provider mirroring `github_rpm`: `ServeRemote` serves `Packages`/`Packages.gz`/`Release`, returns 404 for `InRelease`/`Release.gpg` (unsigned, consumed via `[trusted=yes]`), and 302-redirects `*.deb` to `{proxyBaseURL}/api/v1/remote/{releases_remote}/{path}`; cold-start prime with a retryable 503.
- `deriveAsset` ranged-GETs the front of the `.deb` (an `ar` archive), locates and fully reads `control.tar.*`, and parses the control paragraph — doubling the range if the control member is truncated. The Packages `SHA256` comes from the GitHub asset `digest` when present, else a one-time full stream; `MD5sum` is left unset (apt verifies against SHA256 under `[trusted=yes]`).
- Adds a `github_deb` background Syncer (own worker pool, shared rate limiter, deduped queue) with per-remote DB-lease-gated scans so only one replica scans per window.
- Adds the `github_deb_sync_state` table plus `ListGitHubDebRemotes` / `ClaimGitHubDebSyncLease` / `ReleaseGitHubDebSyncLease` DB helpers, kept separate from the rpm ones.
- Primes `github_deb` remotes on create and runs the deb syncer alongside the rpm one; prime-on-create is routed by package type.
- Reuses the deb apt-index generators and control parser; the Packages generator now skips empty hash lines so a SHA256-only entry is valid.

## Notes / deviations

- **Filename convention:** the `Filename` stored in the Packages index is the **github-relative** asset path (same as rpm's `assetPath`), not `pool/<asset>`. This is required for the `.deb` 302 to `{releases_remote=github}/{path}` to resolve against github.com; it still matches the `*.deb` redirect rule.
- **GitHub client helpers** (releases pagination, ranged GET, auth headers) are duplicated into the deb package rather than shared, because the rpm equivalents are unexported in `package rpm` and the task requires not modifying the rpm provider.
- `go build`, `go vet`, `go mod tidy`, and `make test` (`-race`, incl. the Postgres lease integration tests) all pass; pre-commit clean.

Do not merge — for review.

---------

Co-authored-by: unkin-agent <unkin-agent@git.unkin.net>
Reviewed-on: #112
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #112.
This commit is contained in:
2026-08-11 23:28:11 +10:00
committed by BenVincent
parent 60f008debc
commit 5fde0ee58e
11 changed files with 1935 additions and 14 deletions
+70
View File
@@ -0,0 +1,70 @@
package database
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// ListGitHubDebRemotes returns every github_deb remote so the syncer can sweep
// them on each poll tick.
func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error) {
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubDeb)
if err != nil {
return nil, err
}
defer rows.Close()
var remotes []models.Remote
for rows.Next() {
var r models.Remote
if err := scanRemote(rows, &r); err != nil {
return nil, err
}
remotes = append(remotes, r)
}
return remotes, rows.Err()
}
// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
// succeeds only when the remote is due (never synced, or synced longer than
// freshness ago) and no live lease is held by another replica. A zero freshness
// (prime scans) ignores the recency gate. The returned etag is the stored
// releases-list ETag, shared across replicas.
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
row := db.Pool.QueryRow(ctx, `
INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
VALUES ($1, $2, now() + make_interval(secs => $4))
ON CONFLICT (remote_name) DO UPDATE
SET sync_lease_owner = $2,
sync_lease_expires = now() + make_interval(secs => $4)
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
RETURNING s.etag
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
var etag string
if err := row.Scan(&etag); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return false, "", nil
}
return false, "", err
}
return true, etag, nil
}
// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
// the owning replica may release; last_synced_at advances so the next poll waits
// a full freshness window, and etag is persisted for the next conditional request.
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
_, err := db.Pool.Exec(ctx, `
UPDATE github_deb_sync_state
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
WHERE remote_name = $1 AND sync_lease_owner = $2
`, remoteName, owner, syncedAt, etag)
return err
}
+90
View File
@@ -0,0 +1,90 @@
package database
import (
"testing"
"time"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func seedGitHubDebRemote(t *testing.T, name string) {
t.Helper()
if err := testDB.CreateRemote(ctx(), &models.Remote{
Name: name, PackageType: models.PackageGitHubDeb, RepoType: models.RepoTypeRemote,
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
}); err != nil {
t.Fatalf("seed github_deb remote: %v", err)
}
}
// TestGitHubDebSyncLease exercises the real SQL: exactly one replica may hold the
// lease, the recency window blocks a too-soon periodic re-claim, and a prime
// (freshness 0) bypasses recency but still respects a live lease.
func TestGitHubDebSyncLease(t *testing.T) {
requireDB(t)
name := "ghdeb-lease-" + time.Now().Format("150405.000000")
seedGitHubDebRemote(t, name)
const lease = 15 * time.Minute
freshness := time.Hour
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
if err != nil || !claimed {
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
}
if etag != "" {
t.Fatalf("initial etag should be empty, got %q", etag)
}
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
if err != nil {
t.Fatalf("replica-2 claim err: %v", err)
}
if claimed2 {
t.Fatal("replica-2 claimed while replica-1 holds the lease")
}
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
t.Fatalf("release: %v", err)
}
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
if err != nil {
t.Fatalf("replica-2 recency claim err: %v", err)
}
if claimed3 {
t.Fatal("periodic claim succeeded inside the freshness window")
}
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
if err != nil || !claimed4 {
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
}
if etag4 != `"etag-1"` {
t.Fatalf("prime claim etag = %q, want persisted \"etag-1\"", etag4)
}
}
func TestListGitHubDebRemotes(t *testing.T) {
requireDB(t)
name := "ghdeb-list-" + time.Now().Format("150405.000000")
seedGitHubDebRemote(t, name)
seedRemote(t, "generic-"+time.Now().Format("150405.000000"))
remotes, err := testDB.ListGitHubDebRemotes(ctx())
if err != nil {
t.Fatalf("list: %v", err)
}
found := false
for _, r := range remotes {
if r.PackageType != models.PackageGitHubDeb {
t.Fatalf("non-github_deb remote returned: %s (%s)", r.Name, r.PackageType)
}
if r.Name == name {
found = true
}
}
if !found {
t.Fatalf("seeded remote %q not returned", name)
}
}
+8
View File
@@ -190,6 +190,14 @@ func (db *DB) migrate() error {
sync_lease_expires TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS github_deb_sync_state (
remote_name TEXT PRIMARY KEY,
etag TEXT DEFAULT '',
last_synced_at TIMESTAMPTZ,
sync_lease_owner TEXT DEFAULT '',
sync_lease_expires TIMESTAMPTZ
);
CREATE TABLE IF NOT EXISTS signing_keys (
purpose TEXT PRIMARY KEY,
private_key_armor TEXT NOT NULL,