unkin-agent
5a06c16797
## Why
`github_alpine` is the Alpine/apk analog of the existing `github_deb`/`github_rpm` metadata-only remotes. It lets a plain GitHub-releases repo of `.apk` files be consumed as a real apk repository without artifactapi ever precaching the packages: it scans the repo's releases, derives each package's `.PKGINFO` from a ranged prefix fetch, synthesizes a per-arch `APKINDEX.tar.gz` from the cached metadata, and redirects the actual `.apk` downloads to a backend `releases_remote`. It stacks on the apk-local branch, reusing that work's alpine APKINDEX generator, `.apk`/`.PKGINFO` parser, Q1 pull-checksum, and `AlpineMetadata` store.
## How
- **`pkg/models`**: add `PackageGitHubAlpine` to the enum + validators (and test).
- **`internal/provider/alpine/github.go`**: the `github_alpine` provider. `ServeRemote` serves per-arch `<arch>/APKINDEX.tar.gz` (reusing `generateAPKIndex` over arch-filtered `AlpineMetadata` rows, `normalizeIndexPath` for apk's `./` dot-segment), 302-redirects `*.apk` to `{proxyBaseURL}/api/v1/remote/{releases_remote}/{path}`, and cold-starts with a 503 + `Retry-After`. `scanWithState` lists releases with ETag/If-None-Match and incrementally derives/prunes. `deriveAsset` does a **ranged GET of just the front of the `.apk`** — the control gzip stream carrying `.PKGINFO` sits near the front — doubling the range on truncation; it parses `.PKGINFO` and computes the `C:` Q1 checksum (`Q1`+base64(sha1(control stream))). `FilePath` = the github-relative asset path so the redirect resolves.
- **`internal/provider/alpine/syncer.go`**: a parallel background `Syncer` (own worker pool, shared rate limiter, deduped queue, DB lease), separate from the deb/rpm syncers.
- **`internal/database/alpine_github_sync.go`** + `github_alpine_sync_state` table: `ListGitHubAlpineRemotes` + Claim/Release per-remote sync lease, kept separate from the deb/rpm tables.
- **`internal/server/server.go`**: construct + `Run` the alpine syncer alongside deb/rpm and register it in the `PackageType→Primer` map (priming on create then flows through the existing generic `remotes.go` path).
The rpm/deb providers, syncers, and tables are untouched — this adds parallel alpine equivalents and reuses shared helpers already present in the alpine package.
## Tests
Mirror the deb github tests: scan derives `.PKGINFO`/Q1 from a ranged prefix of a `testsupport.MinimalApk` served over an httptest range server (no full download); diff/prune; pattern filter; per-arch `ServeRemote` routing (index served per-arch and grouped, dot-segment collapse, `.apk` → 302, cold-start 503, warm 200, canceled-request-serves-cache); DB lease prevents a second replica; prime bypasses the recency window. `go build`/`go vet`/`go mod tidy` clean, `make test` (-race) green, pre-commit green.
Reviewed-on: #115
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
72 lines
2.5 KiB
Go
72 lines
2.5 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
// ListGitHubAlpineRemotes returns every github_alpine remote so the syncer can
|
|
// sweep them on each poll tick.
|
|
func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error) {
|
|
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubAlpine)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var remotes []models.Remote
|
|
for rows.Next() {
|
|
var r models.Remote
|
|
if err := scanRemote(rows, &r); err != nil {
|
|
return nil, err
|
|
}
|
|
remotes = append(remotes, r)
|
|
}
|
|
return remotes, rows.Err()
|
|
}
|
|
|
|
// ClaimGitHubAlpineSyncLease atomically claims the per-remote sync lease. It
|
|
// succeeds only when the remote is due (never synced, or synced longer than
|
|
// freshness ago) and no live lease is held by another replica. A zero freshness
|
|
// (prime scans) ignores the recency gate. The returned etag is the stored
|
|
// releases-list ETag, shared across replicas.
|
|
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
|
row := db.Pool.QueryRow(ctx, `
|
|
INSERT INTO github_alpine_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
|
VALUES ($1, $2, now() + make_interval(secs => $4))
|
|
ON CONFLICT (remote_name) DO UPDATE
|
|
SET sync_lease_owner = $2,
|
|
sync_lease_expires = now() + make_interval(secs => $4)
|
|
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
|
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
|
RETURNING s.etag
|
|
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
|
|
|
var etag string
|
|
if err := row.Scan(&etag); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return false, "", nil
|
|
}
|
|
return false, "", err
|
|
}
|
|
return true, etag, nil
|
|
}
|
|
|
|
// ReleaseGitHubAlpineSyncLease records the completed scan and frees the lease.
|
|
// Only the owning replica may release; last_synced_at advances so the next poll
|
|
// waits a full freshness window, and etag is persisted for the next conditional
|
|
// request.
|
|
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
|
_, err := db.Pool.Exec(ctx, `
|
|
UPDATE github_alpine_sync_state
|
|
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
|
WHERE remote_name = $1 AND sync_lease_owner = $2
|
|
`, remoteName, owner, syncedAt, etag)
|
|
return err
|
|
}
|