Complete rewrite of ArtifactAPI from Python/FastAPI to Go as a single binary. Core engine: - 10 package providers: generic, docker, helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy — each with built-in mutable patterns - Content-addressable storage (SHA256 dedup across all remotes) - Three-tier caching: Redis (TTL/locks) → S3/MinIO (blobs) → upstream - Classifier with allowlist/blocklist per-remote (empty = allow all) - Circuit breaker, conditional revalidation, stale-on-error - Background garbage collection for orphaned blobs - Access logging to PostgreSQL API: - v1 proxy endpoints (backwards compatible) - v2 management API: CRUD remotes/virtuals, object browser, stats, health, SSE events, probe/test endpoint - Virtual repos with index merging (Helm YAML + PyPI HTML) Frontend (React + Vite, separate Dockerfile): - Dashboard with stats, health indicators, top remotes - Remotes list with type filter, remote detail with config/patterns - Object browser with pagination and evict - Test Remote page: probe any remote path, see headers/size/timing - Virtuals page with expandable member lists TUI (Bubble Tea): - Dashboard, remotes list/detail, object browser, virtuals - Vim-style navigation, artifactapi tui --endpoint <url> Infrastructure: - S3 client supports MinIO, Ceph RGW, AWS S3 (minio-go) - PostgreSQL schema with migrations - Docker Compose: API + UI + Postgres 17 + Redis 7 + MinIO - Makefile with Go version check, build/test/lint/fmt/e2e targets - Distroless Docker image (~15MB) Testing: - Unit tests for models, classifier, providers, mergers - E2E tests with testcontainers-go (real Postgres/Redis/MinIO) Terraform config: - All 40 production remotes + helm virtual as HCL - Provider repo: terraform-provider-artifactapi v0.0.1 (separate) --------- Co-authored-by: Ben Vincent <ben@unkin.net> Reviewed-on: #47
This commit was merged in pull request #47.
This commit is contained in:
@@ -0,0 +1,111 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"sync"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/database"
|
||||
"git.unkin.net/unkin/artifactapi/internal/provider"
|
||||
"git.unkin.net/unkin/artifactapi/internal/proxy"
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type Engine struct {
|
||||
db *database.DB
|
||||
proxyEngine *proxy.Engine
|
||||
}
|
||||
|
||||
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
|
||||
return &Engine{db: db, proxyEngine: proxyEngine}
|
||||
}
|
||||
|
||||
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
|
||||
merger, err := GetMerger(virt.PackageType)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("unsupported virtual type %q: %w", virt.PackageType, err)
|
||||
}
|
||||
|
||||
members, err := e.fetchMemberIndexes(ctx, virt, path)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
if len(members) == 0 {
|
||||
return nil, "", fmt.Errorf("no members reachable for virtual %q", virt.Name)
|
||||
}
|
||||
|
||||
merged, err := merger.MergeIndexes(members, proxyBaseURL)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("merge indexes: %w", err)
|
||||
}
|
||||
|
||||
contentType := "application/octet-stream"
|
||||
switch virt.PackageType {
|
||||
case models.PackageHelm:
|
||||
contentType = "text/yaml"
|
||||
case models.PackagePyPI:
|
||||
contentType = "text/html"
|
||||
}
|
||||
|
||||
return merged, contentType, nil
|
||||
}
|
||||
|
||||
func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, path string) ([]MemberIndex, error) {
|
||||
type result struct {
|
||||
index MemberIndex
|
||||
err error
|
||||
}
|
||||
|
||||
results := make([]result, len(virt.Members))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
for i, memberName := range virt.Members {
|
||||
wg.Add(1)
|
||||
go func(idx int, name string) {
|
||||
defer wg.Done()
|
||||
|
||||
remote, err := e.db.GetRemote(ctx, name)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("remote %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
|
||||
prov, err := provider.Get(remote.PackageType)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)}
|
||||
return
|
||||
}
|
||||
|
||||
fetchResult, err := e.proxyEngine.Fetch(ctx, *remote, path, prov)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("fetch %q/%s: %w", name, path, err)}
|
||||
return
|
||||
}
|
||||
defer fetchResult.Reader.Close()
|
||||
|
||||
body, err := io.ReadAll(fetchResult.Reader)
|
||||
if err != nil {
|
||||
results[idx] = result{err: fmt.Errorf("read %q: %w", name, err)}
|
||||
return
|
||||
}
|
||||
|
||||
results[idx] = result{index: MemberIndex{RemoteName: name, Body: body}}
|
||||
}(i, memberName)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
var members []MemberIndex
|
||||
for _, r := range results {
|
||||
if r.err != nil {
|
||||
slog.Warn("virtual member fetch failed", "error", r.err)
|
||||
continue
|
||||
}
|
||||
members = append(members, r.index)
|
||||
}
|
||||
|
||||
return members, nil
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterMerger(models.PackageHelm, &HelmMerger{})
|
||||
}
|
||||
|
||||
type HelmMerger struct{}
|
||||
|
||||
type helmIndex struct {
|
||||
APIVersion string `yaml:"apiVersion"`
|
||||
Entries map[string][]helmChartVersion `yaml:"entries"`
|
||||
Generated string `yaml:"generated,omitempty"`
|
||||
}
|
||||
|
||||
type helmChartVersion struct {
|
||||
Name string `yaml:"name"`
|
||||
Version string `yaml:"version"`
|
||||
URLs []string `yaml:"urls"`
|
||||
rest map[string]any
|
||||
}
|
||||
|
||||
func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
|
||||
merged := &helmIndex{
|
||||
APIVersion: "v1",
|
||||
Entries: make(map[string][]helmChartVersion),
|
||||
}
|
||||
|
||||
seen := map[string]map[string]bool{}
|
||||
|
||||
for _, member := range members {
|
||||
var idx helmIndex
|
||||
if err := yaml.Unmarshal(member.Body, &idx); err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for chart, versions := range idx.Entries {
|
||||
if seen[chart] == nil {
|
||||
seen[chart] = map[string]bool{}
|
||||
}
|
||||
for _, ver := range versions {
|
||||
key := chart + ":" + ver.Version
|
||||
if seen[chart][ver.Version] {
|
||||
continue
|
||||
}
|
||||
seen[chart][ver.Version] = true
|
||||
|
||||
if proxyBaseURL != "" {
|
||||
for i, u := range ver.URLs {
|
||||
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
|
||||
ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
member.RemoteName,
|
||||
extractPath(u))
|
||||
} else {
|
||||
ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
member.RemoteName,
|
||||
u)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
merged.Entries[chart] = append(merged.Entries[chart], ver)
|
||||
_ = key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yaml.Marshal(merged)
|
||||
}
|
||||
|
||||
func extractPath(rawURL string) string {
|
||||
idx := strings.Index(rawURL, "://")
|
||||
if idx == -1 {
|
||||
return rawURL
|
||||
}
|
||||
rest := rawURL[idx+3:]
|
||||
slashIdx := strings.Index(rest, "/")
|
||||
if slashIdx == -1 {
|
||||
return ""
|
||||
}
|
||||
return rest[slashIdx+1:]
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
package virtual_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
func TestHelmMerger_BasicMerge(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "repo-a",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
nginx:
|
||||
- name: nginx
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- https://charts-a.example.com/nginx-1.0.0.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "repo-b",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
redis:
|
||||
- name: redis
|
||||
version: "2.0.0"
|
||||
urls:
|
||||
- https://charts-b.example.com/redis-2.0.0.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "nginx") {
|
||||
t.Error("expected nginx in merged index")
|
||||
}
|
||||
if !strings.Contains(body, "redis") {
|
||||
t.Error("expected redis in merged index")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-a") {
|
||||
t.Error("expected proxy URL for repo-a")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-b") {
|
||||
t.Error("expected proxy URL for repo-b")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmMerger_Dedup(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
idx := []byte(`apiVersion: v1
|
||||
entries:
|
||||
nginx:
|
||||
- name: nginx
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- nginx-1.0.0.tgz
|
||||
`)
|
||||
|
||||
members := []virtual.MemberIndex{
|
||||
{RemoteName: "repo-a", Body: idx},
|
||||
{RemoteName: "repo-b", Body: idx},
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes(members, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := strings.Count(string(result), "name: nginx")
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 entry for nginx, got %d\n%s", count, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHelmMerger_PriorityOrder(t *testing.T) {
|
||||
m := &virtual.HelmMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "priority-repo",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
chart:
|
||||
- name: chart
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- chart-from-priority.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "fallback-repo",
|
||||
Body: []byte(`apiVersion: v1
|
||||
entries:
|
||||
chart:
|
||||
- name: chart
|
||||
version: "1.0.0"
|
||||
urls:
|
||||
- chart-from-fallback.tgz
|
||||
`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "priority-repo") {
|
||||
t.Error("expected priority repo URL to win")
|
||||
}
|
||||
if strings.Contains(body, "fallback-repo") {
|
||||
t.Error("expected fallback repo to be excluded for duplicate")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
type MemberIndex struct {
|
||||
RemoteName string
|
||||
Body []byte
|
||||
}
|
||||
|
||||
type IndexMerger interface {
|
||||
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
|
||||
}
|
||||
|
||||
var mergers = map[models.PackageType]IndexMerger{}
|
||||
|
||||
func RegisterMerger(pt models.PackageType, m IndexMerger) {
|
||||
mergers[pt] = m
|
||||
}
|
||||
|
||||
func GetMerger(pt models.PackageType) (IndexMerger, error) {
|
||||
m, ok := mergers[pt]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("no merger registered for package type %q", pt)
|
||||
}
|
||||
return m, nil
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package virtual
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
func init() {
|
||||
RegisterMerger(models.PackagePyPI, &PyPIMerger{})
|
||||
}
|
||||
|
||||
type PyPIMerger struct{}
|
||||
|
||||
func (m *PyPIMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
|
||||
links := map[string]string{}
|
||||
|
||||
for _, member := range members {
|
||||
body := string(member.Body)
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "<a ") {
|
||||
continue
|
||||
}
|
||||
|
||||
href := extractHref(line)
|
||||
text := extractLinkText(line)
|
||||
if text == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, exists := links[text]; exists {
|
||||
continue
|
||||
}
|
||||
|
||||
if proxyBaseURL != "" && href != "" {
|
||||
href = fmt.Sprintf("%s/api/v1/remote/%s/%s",
|
||||
strings.TrimRight(proxyBaseURL, "/"),
|
||||
member.RemoteName,
|
||||
strings.TrimLeft(href, "/"))
|
||||
}
|
||||
|
||||
links[text] = href
|
||||
}
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(links))
|
||||
for k := range links {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("<!DOCTYPE html>\n<html><body>\n")
|
||||
for _, name := range keys {
|
||||
sb.WriteString(fmt.Sprintf(" <a href=\"%s\">%s</a>\n", links[name], name))
|
||||
}
|
||||
sb.WriteString("</body></html>\n")
|
||||
|
||||
return []byte(sb.String()), nil
|
||||
}
|
||||
|
||||
func extractHref(tag string) string {
|
||||
idx := strings.Index(tag, `href="`)
|
||||
if idx == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[idx+6:]
|
||||
end := strings.Index(rest, `"`)
|
||||
if end == -1 {
|
||||
return rest
|
||||
}
|
||||
return rest[:end]
|
||||
}
|
||||
|
||||
func extractLinkText(tag string) string {
|
||||
start := strings.Index(tag, ">")
|
||||
if start == -1 {
|
||||
return ""
|
||||
}
|
||||
rest := tag[start+1:]
|
||||
end := strings.Index(rest, "</a>")
|
||||
if end == -1 {
|
||||
return strings.TrimSpace(rest)
|
||||
}
|
||||
return strings.TrimSpace(rest[:end])
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package virtual_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/internal/virtual"
|
||||
)
|
||||
|
||||
func TestPyPIMerger_BasicMerge(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
member1 := virtual.MemberIndex{
|
||||
RemoteName: "pypi-a",
|
||||
Body: []byte(`<!DOCTYPE html>
|
||||
<html><body>
|
||||
<a href="/simple/requests/">requests</a>
|
||||
<a href="/simple/flask/">flask</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
member2 := virtual.MemberIndex{
|
||||
RemoteName: "pypi-b",
|
||||
Body: []byte(`<!DOCTYPE html>
|
||||
<html><body>
|
||||
<a href="/simple/django/">django</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
if !strings.Contains(body, "requests") {
|
||||
t.Error("expected requests")
|
||||
}
|
||||
if !strings.Contains(body, "flask") {
|
||||
t.Error("expected flask")
|
||||
}
|
||||
if !strings.Contains(body, "django") {
|
||||
t.Error("expected django")
|
||||
}
|
||||
if !strings.Contains(body, "proxy.example.com/api/v1/remote/pypi-a") {
|
||||
t.Error("expected proxy URL for pypi-a")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPIMerger_Dedup(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
idx := []byte(`<html><body>
|
||||
<a href="/simple/requests/">requests</a>
|
||||
</body></html>`)
|
||||
|
||||
members := []virtual.MemberIndex{
|
||||
{RemoteName: "a", Body: idx},
|
||||
{RemoteName: "b", Body: idx},
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes(members, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
count := strings.Count(string(result), "<a ")
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 <a> tag for deduplicated requests, got %d\n%s", count, result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPyPIMerger_Sorted(t *testing.T) {
|
||||
m := &virtual.PyPIMerger{}
|
||||
|
||||
member := virtual.MemberIndex{
|
||||
RemoteName: "pypi",
|
||||
Body: []byte(`<html><body>
|
||||
<a href="/z/">zebra</a>
|
||||
<a href="/a/">alpha</a>
|
||||
<a href="/m/">middle</a>
|
||||
</body></html>`),
|
||||
}
|
||||
|
||||
result, err := m.MergeIndexes([]virtual.MemberIndex{member}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
body := string(result)
|
||||
alphaIdx := strings.Index(body, "alpha")
|
||||
middleIdx := strings.Index(body, "middle")
|
||||
zebraIdx := strings.Index(body, "zebra")
|
||||
|
||||
if alphaIdx > middleIdx || middleIdx > zebraIdx {
|
||||
t.Error("expected sorted output")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user