unkin-agent
c88a84e7db
Add the Debian/apt analog of github_rpm: a metadata-only remote that scans a GitHub repo's releases for .deb assets, derives per-asset control metadata via a ranged prefix fetch (never downloading whole packages), synthesizes a flat apt repository, and redirects .deb downloads to a backend releases_remote. - Add PackageGitHubDeb to the package-type enum + validity map. - Add the github_deb provider (internal/provider/deb/github.go): ServeRemote serves Packages/Packages.gz/Release, 404s the signed index variants (consumed via [trusted=yes]), and 302-redirects *.deb to the releases_remote; deriveAsset ranged-GETs the ar prefix, locates control.tar.*, and parses the control paragraph, doubling the range on truncation; sha256 comes from the asset digest when present, else a one-time full stream. - Add the github_deb background Syncer (internal/provider/deb/syncer.go): its own worker pool, shared rate limiter, deduped queue, and DB-lease-gated scans. - Add github_deb_sync_state table plus ListGitHubDebRemotes/Claim/Release DB helpers (separate from the rpm ones). - Prime github_deb remotes on create and run the deb syncer alongside the rpm one; route prime-on-create by package type. - Reuse the deb apt-index generators and control parser; skip empty hash lines in the Packages index so a SHA256-only metadata entry is valid.
71 lines
2.5 KiB
Go
71 lines
2.5 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
|
|
"git.unkin.net/unkin/artifactapi/pkg/models"
|
|
)
|
|
|
|
// ListGitHubDebRemotes returns every github_deb remote so the syncer can sweep
|
|
// them on each poll tick.
|
|
func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error) {
|
|
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes WHERE package_type = $1 ORDER BY name`, models.PackageGitHubDeb)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var remotes []models.Remote
|
|
for rows.Next() {
|
|
var r models.Remote
|
|
if err := scanRemote(rows, &r); err != nil {
|
|
return nil, err
|
|
}
|
|
remotes = append(remotes, r)
|
|
}
|
|
return remotes, rows.Err()
|
|
}
|
|
|
|
// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
|
|
// succeeds only when the remote is due (never synced, or synced longer than
|
|
// freshness ago) and no live lease is held by another replica. A zero freshness
|
|
// (prime scans) ignores the recency gate. The returned etag is the stored
|
|
// releases-list ETag, shared across replicas.
|
|
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
|
|
row := db.Pool.QueryRow(ctx, `
|
|
INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
|
|
VALUES ($1, $2, now() + make_interval(secs => $4))
|
|
ON CONFLICT (remote_name) DO UPDATE
|
|
SET sync_lease_owner = $2,
|
|
sync_lease_expires = now() + make_interval(secs => $4)
|
|
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
|
|
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
|
|
RETURNING s.etag
|
|
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
|
|
|
|
var etag string
|
|
if err := row.Scan(&etag); err != nil {
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return false, "", nil
|
|
}
|
|
return false, "", err
|
|
}
|
|
return true, etag, nil
|
|
}
|
|
|
|
// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
|
|
// the owning replica may release; last_synced_at advances so the next poll waits
|
|
// a full freshness window, and etag is persisted for the next conditional request.
|
|
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
|
|
_, err := db.Pool.Exec(ctx, `
|
|
UPDATE github_deb_sync_state
|
|
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
|
|
WHERE remote_name = $1 AND sync_lease_owner = $2
|
|
`, remoteName, owner, syncedAt, etag)
|
|
return err
|
|
}
|