Add github_alpine metadata-only package type (#115)
## Why
`github_alpine` is the Alpine/apk analog of the existing `github_deb`/`github_rpm` metadata-only remotes. It lets a plain GitHub-releases repo of `.apk` files be consumed as a real apk repository without artifactapi ever precaching the packages: it scans the repo's releases, derives each package's `.PKGINFO` from a ranged prefix fetch, synthesizes a per-arch `APKINDEX.tar.gz` from the cached metadata, and redirects the actual `.apk` downloads to a backend `releases_remote`. It stacks on the apk-local branch, reusing that work's alpine APKINDEX generator, `.apk`/`.PKGINFO` parser, Q1 pull-checksum, and `AlpineMetadata` store.
## How
- **`pkg/models`**: add `PackageGitHubAlpine` to the enum + validators (and test).
- **`internal/provider/alpine/github.go`**: the `github_alpine` provider. `ServeRemote` serves per-arch `<arch>/APKINDEX.tar.gz` (reusing `generateAPKIndex` over arch-filtered `AlpineMetadata` rows, `normalizeIndexPath` for apk's `./` dot-segment), 302-redirects `*.apk` to `{proxyBaseURL}/api/v1/remote/{releases_remote}/{path}`, and cold-starts with a 503 + `Retry-After`. `scanWithState` lists releases with ETag/If-None-Match and incrementally derives/prunes. `deriveAsset` does a **ranged GET of just the front of the `.apk`** — the control gzip stream carrying `.PKGINFO` sits near the front — doubling the range on truncation; it parses `.PKGINFO` and computes the `C:` Q1 checksum (`Q1`+base64(sha1(control stream))). `FilePath` = the github-relative asset path so the redirect resolves.
- **`internal/provider/alpine/syncer.go`**: a parallel background `Syncer` (own worker pool, shared rate limiter, deduped queue, DB lease), separate from the deb/rpm syncers.
- **`internal/database/alpine_github_sync.go`** + `github_alpine_sync_state` table: `ListGitHubAlpineRemotes` + Claim/Release per-remote sync lease, kept separate from the deb/rpm tables.
- **`internal/server/server.go`**: construct + `Run` the alpine syncer alongside deb/rpm and register it in the `PackageType→Primer` map (priming on create then flows through the existing generic `remotes.go` path).
The rpm/deb providers, syncers, and tables are untouched — this adds parallel alpine equivalents and reuses shared helpers already present in the alpine package.
## Tests
Mirror the deb github tests: scan derives `.PKGINFO`/Q1 from a ranged prefix of a `testsupport.MinimalApk` served over an httptest range server (no full download); diff/prune; pattern filter; per-arch `ServeRemote` routing (index served per-arch and grouped, dot-segment collapse, `.apk` → 302, cold-start 503, warm 200, canceled-request-serves-cache); DB lease prevents a second replica; prime bypasses the recency window. `go build`/`go vet`/`go mod tidy` clean, `make test` (-race) green, pre-commit green.
Reviewed-on: #115
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
This commit was merged in pull request #115.
This commit is contained in:
@@ -20,7 +20,7 @@ import (
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/gc"
|
||||
"git.unkin.net/unkin/artifactapi/internal/githubauth"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
|
||||
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
|
||||
@@ -52,6 +52,7 @@ type Server struct {
|
||||
gc *gc.Collector
|
||||
syncer *rpm.Syncer
|
||||
debSyncer *deb.Syncer
|
||||
alpineSyncer *alpine.Syncer
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, version string) (*Server, error) {
|
||||
@@ -105,6 +106,12 @@ func New(cfg *config.Config, version string) (*Server, error) {
|
||||
Workers: cfg.GitHubSyncWorkers,
|
||||
PollInterval: time.Duration(cfg.GitHubSyncPollInterval) * time.Second,
|
||||
})
|
||||
alpineSyncer := alpine.NewSyncer(db, alpine.SyncConfig{
|
||||
RatePerSec: cfg.GitHubSyncRatePerSec,
|
||||
Burst: cfg.GitHubSyncBurst,
|
||||
Workers: cfg.GitHubSyncWorkers,
|
||||
PollInterval: time.Duration(cfg.GitHubSyncPollInterval) * time.Second,
|
||||
})
|
||||
|
||||
// The terraform registry signs with a GPG key. A configured file wins (BYO
|
||||
// key); otherwise artifactapi generates one on first start and persists it in
|
||||
@@ -138,6 +145,7 @@ func New(cfg *config.Config, version string) (*Server, error) {
|
||||
gc: collector,
|
||||
syncer: syncer,
|
||||
debSyncer: debSyncer,
|
||||
alpineSyncer: alpineSyncer,
|
||||
}
|
||||
|
||||
s.router = s.routes()
|
||||
@@ -168,8 +176,9 @@ func (s *Server) routes() chi.Router {
|
||||
r.Mount("/v2", proxyHandler.DockerV2Routes())
|
||||
|
||||
remotesHandler := v2.NewRemotesHandler(s.db, map[models.PackageType]v2.Primer{
|
||||
models.PackageGitHubRPM: s.syncer,
|
||||
models.PackageGitHubDeb: s.debSyncer,
|
||||
models.PackageGitHubRPM: s.syncer,
|
||||
models.PackageGitHubDeb: s.debSyncer,
|
||||
models.PackageGitHubAlpine: s.alpineSyncer,
|
||||
})
|
||||
virtualsHandler := v2.NewVirtualsHandler(s.db)
|
||||
healthHandler := v2.NewHealthHandler(s.db, s.cache, s.store)
|
||||
@@ -239,6 +248,7 @@ func (s *Server) Run(ctx context.Context) error {
|
||||
go s.gc.Run(ctx)
|
||||
go s.syncer.Run(ctx)
|
||||
go s.debSyncer.Run(ctx)
|
||||
go s.alpineSyncer.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
@@ -261,6 +271,7 @@ func (s *Server) RunOnListener(ctx context.Context, ln net.Listener) error {
|
||||
go s.gc.Run(ctx)
|
||||
go s.syncer.Run(ctx)
|
||||
go s.debSyncer.Run(ctx)
|
||||
go s.alpineSyncer.Run(ctx)
|
||||
|
||||
httpServer := s.newHTTPServer()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user