unkin-agent
aa96c8af70
github_alpine is the Alpine/apk analog of github_deb/github_rpm: a metadata-only remote that scans a GitHub repo's releases for .apk assets, derives each package's .PKGINFO via a ranged prefix fetch (never downloading whole packages), synthesizes a per-arch APKINDEX.tar.gz from that cached metadata, and 302-redirects .apk downloads to a backend releases_remote. It stacks on the apk-local work, reusing the alpine provider's APKINDEX generator, .PKGINFO parser, Q1 checksum, and AlpineMetadata store. - pkg/models: add PackageGitHubAlpine to the enum + validators - internal/provider/alpine/github.go: the github_alpine provider (ServeRemote per-arch index + .apk redirect, cold-start 503, scanWithState incremental derive, ranged .PKGINFO prefix fetch with range-doubling on truncation) - internal/provider/alpine/syncer.go: parallel background Syncer (worker pool, shared limiter, deduped queue, DB lease) - internal/database/alpine_github_sync.go + github_alpine_sync_state table: remote enumeration + per-remote sync lease - internal/api/v2/remotes.go: primed on create via the shared Primer map - internal/server/server.go: construct + Run the alpine syncer, register it in the Primer map - tests mirror the deb github_test/syncer_test (scan/diff/prune, ranged .PKGINFO parse, per-arch ServeRemote routing, .apk 302, DB lease)
247 lines
7.9 KiB
Go
247 lines
7.9 KiB
Go
package database
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
type DB struct {
|
|
Pool *pgxpool.Pool
|
|
}
|
|
|
|
func New(dsn string) (*DB, error) {
|
|
pool, err := pgxpool.New(context.Background(), dsn)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("connect to postgres: %w", err)
|
|
}
|
|
|
|
if err := pool.Ping(context.Background()); err != nil {
|
|
pool.Close()
|
|
return nil, fmt.Errorf("ping postgres: %w", err)
|
|
}
|
|
|
|
db := &DB{Pool: pool}
|
|
if err := db.migrate(); err != nil {
|
|
pool.Close()
|
|
return nil, fmt.Errorf("run migrations: %w", err)
|
|
}
|
|
|
|
return db, nil
|
|
}
|
|
|
|
func (db *DB) Close() {
|
|
db.Pool.Close()
|
|
}
|
|
|
|
func (db *DB) migrate() error {
|
|
ctx := context.Background()
|
|
|
|
_, err := db.Pool.Exec(ctx, `
|
|
CREATE TABLE IF NOT EXISTS remotes (
|
|
name TEXT PRIMARY KEY,
|
|
package_type TEXT NOT NULL,
|
|
repo_type TEXT DEFAULT 'remote',
|
|
base_url TEXT NOT NULL DEFAULT '',
|
|
description TEXT DEFAULT '',
|
|
username TEXT DEFAULT '',
|
|
password TEXT DEFAULT '',
|
|
immutable_ttl INTEGER DEFAULT 0,
|
|
mutable_ttl INTEGER DEFAULT 3600,
|
|
check_mutable BOOLEAN DEFAULT TRUE,
|
|
patterns TEXT[] DEFAULT '{}',
|
|
blocklist TEXT[] DEFAULT '{}',
|
|
mutable_patterns TEXT[] DEFAULT '{}',
|
|
immutable_patterns TEXT[] DEFAULT '{}',
|
|
ban_tags_enabled BOOLEAN DEFAULT FALSE,
|
|
ban_tags TEXT[] DEFAULT '{}',
|
|
quarantine_enabled BOOLEAN DEFAULT FALSE,
|
|
quarantine_days INTEGER DEFAULT 3,
|
|
stale_on_error BOOLEAN DEFAULT TRUE,
|
|
releases_remote TEXT DEFAULT '',
|
|
managed_by TEXT DEFAULT '',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS virtuals (
|
|
name TEXT PRIMARY KEY,
|
|
package_type TEXT NOT NULL,
|
|
description TEXT DEFAULT '',
|
|
members TEXT[] NOT NULL,
|
|
managed_by TEXT DEFAULT '',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS blobs (
|
|
content_hash TEXT PRIMARY KEY,
|
|
s3_key TEXT NOT NULL,
|
|
size_bytes BIGINT NOT NULL,
|
|
content_type TEXT DEFAULT 'application/octet-stream',
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS artifacts (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
|
|
path TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
|
upstream_etag TEXT DEFAULT '',
|
|
upstream_last_modified TIMESTAMPTZ,
|
|
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
|
|
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
|
|
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
|
|
fetch_count BIGINT DEFAULT 1,
|
|
access_count BIGINT DEFAULT 1,
|
|
UNIQUE(remote_name, path)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
|
|
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
|
|
|
|
CREATE TABLE IF NOT EXISTS local_files (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
repo_name TEXT NOT NULL,
|
|
file_path TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
UNIQUE(repo_name, file_path)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS access_log (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
remote_name TEXT NOT NULL,
|
|
path TEXT NOT NULL,
|
|
cache_hit BOOLEAN NOT NULL,
|
|
size_bytes BIGINT DEFAULT 0,
|
|
upstream_ms INTEGER DEFAULT 0,
|
|
client_ip TEXT DEFAULT '',
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
|
|
|
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
|
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_dial_timeout INTEGER DEFAULT 0;
|
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_tls_timeout INTEGER DEFAULT 0;
|
|
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS upstream_response_header_timeout INTEGER DEFAULT 0;
|
|
|
|
CREATE TABLE IF NOT EXISTS rpm_metadata (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
repo_name TEXT NOT NULL,
|
|
file_path TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
epoch INTEGER DEFAULT 0,
|
|
version TEXT NOT NULL,
|
|
release TEXT NOT NULL,
|
|
arch TEXT NOT NULL,
|
|
summary TEXT DEFAULT '',
|
|
description TEXT DEFAULT '',
|
|
rpm_size BIGINT DEFAULT 0,
|
|
installed_size BIGINT DEFAULT 0,
|
|
license TEXT DEFAULT '',
|
|
vendor TEXT DEFAULT '',
|
|
build_group TEXT DEFAULT '',
|
|
build_host TEXT DEFAULT '',
|
|
source_rpm TEXT DEFAULT '',
|
|
url TEXT DEFAULT '',
|
|
packager TEXT DEFAULT '',
|
|
requires JSONB DEFAULT '[]',
|
|
provides JSONB DEFAULT '[]',
|
|
conflicts JSONB DEFAULT '[]',
|
|
obsoletes JSONB DEFAULT '[]',
|
|
files JSONB DEFAULT '[]',
|
|
changelogs JSONB DEFAULT '[]',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
UNIQUE(repo_name, file_path)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
|
|
|
|
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS conflicts JSONB DEFAULT '[]';
|
|
ALTER TABLE rpm_metadata ADD COLUMN IF NOT EXISTS obsoletes JSONB DEFAULT '[]';
|
|
|
|
CREATE TABLE IF NOT EXISTS deb_metadata (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
repo_name TEXT NOT NULL,
|
|
file_path TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
version TEXT NOT NULL,
|
|
architecture TEXT NOT NULL,
|
|
control TEXT NOT NULL,
|
|
size BIGINT DEFAULT 0,
|
|
md5 TEXT DEFAULT '',
|
|
sha256 TEXT DEFAULT '',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
UNIQUE(repo_name, file_path)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_deb_metadata_repo ON deb_metadata(repo_name);
|
|
|
|
CREATE TABLE IF NOT EXISTS alpine_metadata (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
repo_name TEXT NOT NULL,
|
|
file_path TEXT NOT NULL,
|
|
content_hash TEXT NOT NULL,
|
|
checksum TEXT NOT NULL,
|
|
name TEXT NOT NULL,
|
|
version TEXT NOT NULL,
|
|
arch TEXT NOT NULL,
|
|
download_size BIGINT DEFAULT 0,
|
|
installed_size BIGINT DEFAULT 0,
|
|
description TEXT DEFAULT '',
|
|
url TEXT DEFAULT '',
|
|
license TEXT DEFAULT '',
|
|
origin TEXT DEFAULT '',
|
|
maintainer TEXT DEFAULT '',
|
|
build_time BIGINT DEFAULT 0,
|
|
commit_hash TEXT DEFAULT '',
|
|
provider_priority TEXT DEFAULT '',
|
|
depends TEXT DEFAULT '',
|
|
provides TEXT DEFAULT '',
|
|
install_if TEXT DEFAULT '',
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
UNIQUE(repo_name, file_path)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo ON alpine_metadata(repo_name);
|
|
CREATE INDEX IF NOT EXISTS idx_alpine_metadata_repo_arch ON alpine_metadata(repo_name, arch);
|
|
|
|
CREATE TABLE IF NOT EXISTS github_rpm_sync_state (
|
|
remote_name TEXT PRIMARY KEY,
|
|
etag TEXT DEFAULT '',
|
|
last_synced_at TIMESTAMPTZ,
|
|
sync_lease_owner TEXT DEFAULT '',
|
|
sync_lease_expires TIMESTAMPTZ
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS github_deb_sync_state (
|
|
remote_name TEXT PRIMARY KEY,
|
|
etag TEXT DEFAULT '',
|
|
last_synced_at TIMESTAMPTZ,
|
|
sync_lease_owner TEXT DEFAULT '',
|
|
sync_lease_expires TIMESTAMPTZ
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS github_alpine_sync_state (
|
|
remote_name TEXT PRIMARY KEY,
|
|
etag TEXT DEFAULT '',
|
|
last_synced_at TIMESTAMPTZ,
|
|
sync_lease_owner TEXT DEFAULT '',
|
|
sync_lease_expires TIMESTAMPTZ
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS signing_keys (
|
|
purpose TEXT PRIMARY KEY,
|
|
private_key_armor TEXT NOT NULL,
|
|
key_id TEXT NOT NULL,
|
|
created_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
`)
|
|
return err
|
|
}
|