unkin-agent
5fde0ee58e
ci/woodpecker/tag/docker Pipeline was successful
## Why This stacks the Debian/apt analog of `github_rpm` on top of the deb local+remote work (#111). It lets a GitHub repo's `.deb` release assets be consumed as a real apt repository without artifactapi ever precaching whole packages: it derives per-asset control metadata from a ranged prefix fetch, synthesizes a flat apt repo from the cache, and redirects the actual `.deb` downloads to a backend `releases_remote` (the generic github.com remote). Base is `benvin/deb-local-remote` (stacked) to keep the diff atomic. ## How - Adds `github_deb` to the package-type enum and validity map. - Adds the `github_deb` provider mirroring `github_rpm`: `ServeRemote` serves `Packages`/`Packages.gz`/`Release`, returns 404 for `InRelease`/`Release.gpg` (unsigned, consumed via `[trusted=yes]`), and 302-redirects `*.deb` to `{proxyBaseURL}/api/v1/remote/{releases_remote}/{path}`; cold-start prime with a retryable 503. - `deriveAsset` ranged-GETs the front of the `.deb` (an `ar` archive), locates and fully reads `control.tar.*`, and parses the control paragraph — doubling the range if the control member is truncated. The Packages `SHA256` comes from the GitHub asset `digest` when present, else a one-time full stream; `MD5sum` is left unset (apt verifies against SHA256 under `[trusted=yes]`). - Adds a `github_deb` background Syncer (own worker pool, shared rate limiter, deduped queue) with per-remote DB-lease-gated scans so only one replica scans per window. - Adds the `github_deb_sync_state` table plus `ListGitHubDebRemotes` / `ClaimGitHubDebSyncLease` / `ReleaseGitHubDebSyncLease` DB helpers, kept separate from the rpm ones. - Primes `github_deb` remotes on create and runs the deb syncer alongside the rpm one; prime-on-create is routed by package type. - Reuses the deb apt-index generators and control parser; the Packages generator now skips empty hash lines so a SHA256-only entry is valid. ## Notes / deviations - **Filename convention:** the `Filename` stored in the Packages index is the **github-relative** asset path (same as rpm's `assetPath`), not `pool/<asset>`. This is required for the `.deb` 302 to `{releases_remote=github}/{path}` to resolve against github.com; it still matches the `*.deb` redirect rule. - **GitHub client helpers** (releases pagination, ranged GET, auth headers) are duplicated into the deb package rather than shared, because the rpm equivalents are unexported in `package rpm` and the task requires not modifying the rpm provider. - `go build`, `go vet`, `go mod tidy`, and `make test` (`-race`, incl. the Postgres lease integration tests) all pass; pre-commit clean. Do not merge — for review. --------- Co-authored-by: unkin-agent <unkin-agent@git.unkin.net> Reviewed-on: #112 Co-authored-by: unkin-agent <unkin-agent@unkin.net> Co-committed-by: unkin-agent <unkin-agent@unkin.net>
91 lines
2.6 KiB
Go
91 lines
2.6 KiB
Go
package database
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
func seedGitHubDebRemote(t *testing.T, name string) {
|
|
t.Helper()
|
|
if err := testDB.CreateRemote(ctx(), &models.Remote{
|
|
Name: name, PackageType: models.PackageGitHubDeb, RepoType: models.RepoTypeRemote,
|
|
BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
|
|
}); err != nil {
|
|
t.Fatalf("seed github_deb remote: %v", err)
|
|
}
|
|
}
|
|
|
|
// TestGitHubDebSyncLease exercises the real SQL: exactly one replica may hold the
|
|
// lease, the recency window blocks a too-soon periodic re-claim, and a prime
|
|
// (freshness 0) bypasses recency but still respects a live lease.
|
|
func TestGitHubDebSyncLease(t *testing.T) {
|
|
requireDB(t)
|
|
name := "ghdeb-lease-" + time.Now().Format("150405.000000")
|
|
seedGitHubDebRemote(t, name)
|
|
|
|
const lease = 15 * time.Minute
|
|
freshness := time.Hour
|
|
|
|
claimed, etag, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-1", freshness, lease)
|
|
if err != nil || !claimed {
|
|
t.Fatalf("replica-1 first claim: claimed=%v err=%v", claimed, err)
|
|
}
|
|
if etag != "" {
|
|
t.Fatalf("initial etag should be empty, got %q", etag)
|
|
}
|
|
|
|
claimed2, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
|
if err != nil {
|
|
t.Fatalf("replica-2 claim err: %v", err)
|
|
}
|
|
if claimed2 {
|
|
t.Fatal("replica-2 claimed while replica-1 holds the lease")
|
|
}
|
|
|
|
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
|
|
t.Fatalf("release: %v", err)
|
|
}
|
|
|
|
claimed3, _, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", freshness, lease)
|
|
if err != nil {
|
|
t.Fatalf("replica-2 recency claim err: %v", err)
|
|
}
|
|
if claimed3 {
|
|
t.Fatal("periodic claim succeeded inside the freshness window")
|
|
}
|
|
|
|
claimed4, etag4, err := testDB.ClaimGitHubDebSyncLease(ctx(), name, "replica-2", 0, lease)
|
|
if err != nil || !claimed4 {
|
|
t.Fatalf("prime claim: claimed=%v err=%v", claimed4, err)
|
|
}
|
|
if etag4 != `"etag-1"` {
|
|
t.Fatalf("prime claim etag = %q, want persisted \"etag-1\"", etag4)
|
|
}
|
|
}
|
|
|
|
func TestListGitHubDebRemotes(t *testing.T) {
|
|
requireDB(t)
|
|
name := "ghdeb-list-" + time.Now().Format("150405.000000")
|
|
seedGitHubDebRemote(t, name)
|
|
seedRemote(t, "generic-"+time.Now().Format("150405.000000"))
|
|
|
|
remotes, err := testDB.ListGitHubDebRemotes(ctx())
|
|
if err != nil {
|
|
t.Fatalf("list: %v", err)
|
|
}
|
|
found := false
|
|
for _, r := range remotes {
|
|
if r.PackageType != models.PackageGitHubDeb {
|
|
t.Fatalf("non-github_deb remote returned: %s (%s)", r.Name, r.PackageType)
|
|
}
|
|
if r.Name == name {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatalf("seeded remote %q not returned", name)
|
|
}
|
|
}
|