Feat/v3 go rewrite (#47)
ci/woodpecker/tag/docker Pipeline was successful

Complete rewrite of ArtifactAPI from Python/FastAPI to Go as a single binary.

Core engine:
- 10 package providers: generic, docker, helm, pypi, npm, rpm, alpine,
  puppet, terraform, goproxy — each with built-in mutable patterns
- Content-addressable storage (SHA256 dedup across all remotes)
- Three-tier caching: Redis (TTL/locks) → S3/MinIO (blobs) → upstream
- Classifier with allowlist/blocklist per-remote (empty = allow all)
- Circuit breaker, conditional revalidation, stale-on-error
- Background garbage collection for orphaned blobs
- Access logging to PostgreSQL

API:
- v1 proxy endpoints (backwards compatible)
- v2 management API: CRUD remotes/virtuals, object browser, stats,
  health, SSE events, probe/test endpoint
- Virtual repos with index merging (Helm YAML + PyPI HTML)

Frontend (React + Vite, separate Dockerfile):
- Dashboard with stats, health indicators, top remotes
- Remotes list with type filter, remote detail with config/patterns
- Object browser with pagination and evict
- Test Remote page: probe any remote path, see headers/size/timing
- Virtuals page with expandable member lists

TUI (Bubble Tea):
- Dashboard, remotes list/detail, object browser, virtuals
- Vim-style navigation, artifactapi tui --endpoint <url>

Infrastructure:
- S3 client supports MinIO, Ceph RGW, AWS S3 (minio-go)
- PostgreSQL schema with migrations
- Docker Compose: API + UI + Postgres 17 + Redis 7 + MinIO
- Makefile with Go version check, build/test/lint/fmt/e2e targets
- Distroless Docker image (~15MB)

Testing:
- Unit tests for models, classifier, providers, mergers
- E2E tests with testcontainers-go (real Postgres/Redis/MinIO)

Terraform config:
- All 40 production remotes + helm virtual as HCL
- Provider repo: terraform-provider-artifactapi v0.0.1 (separate)

---------

Co-authored-by: Ben Vincent <ben@unkin.net>
Reviewed-on: #47
This commit was merged in pull request #47.
This commit is contained in:
2026-06-07 19:30:35 +10:00
parent f25bf6cb29
commit b46c116f6b
160 changed files with 11448 additions and 7907 deletions
+106
View File
@@ -0,0 +1,106 @@
package v1
import (
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
type ProxyHandler struct {
engine *proxy.Engine
virtualEngine *virtual.Engine
db *database.DB
}
func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB) *ProxyHandler {
return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db}
}
func (h *ProxyHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/remote/{remoteName}/*", h.handleProxy)
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
return r
}
func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "remoteName")
path := chi.URLParam(r, "*")
remote, err := h.db.GetRemote(r.Context(), remoteName)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", remoteName), http.StatusNotFound)
return
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError)
return
}
result, err := h.engine.Fetch(r.Context(), *remote, path, prov)
if err != nil {
var proxyErr *proxy.ProxyError
if errors.As(err, &proxyErr) {
http.Error(w, proxyErr.Message, proxyErr.Status)
return
}
slog.Error("proxy fetch failed", "remote", remoteName, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
defer result.Reader.Close()
w.Header().Set("Content-Type", result.ContentType)
w.Header().Set("X-Artifact-Source", result.Source)
if result.Size > 0 {
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
}
w.WriteHeader(http.StatusOK)
io.Copy(w, result.Reader)
}
func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) {
virtualName := chi.URLParam(r, "virtualName")
path := chi.URLParam(r, "*")
virt, err := h.db.GetVirtual(r.Context(), virtualName)
if err != nil {
http.Error(w, fmt.Sprintf("virtual %q not found", virtualName), http.StatusNotFound)
return
}
proxyBaseURL := fmt.Sprintf("%s://%s", scheme(r), r.Host)
body, contentType, err := h.virtualEngine.Fetch(r.Context(), *virt, path, proxyBaseURL)
if err != nil {
slog.Error("virtual fetch failed", "virtual", virtualName, "path", path, "error", err)
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("X-Artifact-Source", "virtual")
w.WriteHeader(http.StatusOK)
w.Write(body)
}
func scheme(r *http.Request) string {
if r.TLS != nil {
return "https"
}
if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
return fwd
}
return "http"
}
+49
View File
@@ -0,0 +1,49 @@
package v2
import (
"fmt"
"net/http"
"time"
"github.com/go-chi/chi/v5"
)
type EventsHandler struct{}
func NewEventsHandler() *EventsHandler {
return &EventsHandler{}
}
func (h *EventsHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.stream)
return r
}
func (h *EventsHandler) stream(w http.ResponseWriter, r *http.Request) {
flusher, ok := w.(http.Flusher)
if !ok {
http.Error(w, "streaming not supported", http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.WriteHeader(http.StatusOK)
flusher.Flush()
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-r.Context().Done():
return
case <-ticker.C:
fmt.Fprintf(w, ": keepalive\n\n")
flusher.Flush()
}
}
}
+43
View File
@@ -0,0 +1,43 @@
package v2
import (
"net/http"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
)
type HealthHandler struct {
db *database.DB
cache *cache.Redis
store *storage.S3
}
func NewHealthHandler(db *database.DB, c *cache.Redis, s *storage.S3) *HealthHandler {
return &HealthHandler{db: db, cache: c, store: s}
}
func (h *HealthHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.health)
return r
}
func (h *HealthHandler) health(w http.ResponseWriter, r *http.Request) {
status := map[string]string{
"status": "ok",
"postgres": "ok",
"redis": "ok",
"s3": "ok",
}
if err := h.db.Pool.Ping(r.Context()); err != nil {
status["postgres"] = "error"
status["status"] = "degraded"
}
writeJSON(w, http.StatusOK, status)
}
+57
View File
@@ -0,0 +1,57 @@
package v2
import (
"fmt"
"net/http"
"strconv"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
)
type ObjectsHandler struct {
db *database.DB
}
func NewObjectsHandler(db *database.DB) *ObjectsHandler {
return &ObjectsHandler{db: db}
}
func (h *ObjectsHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.list)
r.Delete("/*", h.evict)
return r
}
func (h *ObjectsHandler) list(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "name")
limit, _ := strconv.Atoi(r.URL.Query().Get("per_page"))
if limit <= 0 || limit > 100 {
limit = 50
}
page, _ := strconv.Atoi(r.URL.Query().Get("page"))
if page <= 0 {
page = 1
}
offset := (page - 1) * limit
artifacts, err := h.db.ListArtifacts(r.Context(), remoteName, limit, offset)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, artifacts)
}
func (h *ObjectsHandler) evict(w http.ResponseWriter, r *http.Request) {
remoteName := chi.URLParam(r, "name")
path := chi.URLParam(r, "*")
if err := h.db.DeleteArtifact(r.Context(), remoteName, path); err != nil {
http.Error(w, fmt.Sprintf("evict failed: %v", err), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
+109
View File
@@ -0,0 +1,109 @@
package v2
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/proxy"
)
type ProbeHandler struct {
engine *proxy.Engine
db *database.DB
}
func NewProbeHandler(engine *proxy.Engine, db *database.DB) *ProbeHandler {
return &ProbeHandler{engine: engine, db: db}
}
func (h *ProbeHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Post("/", h.probe)
return r
}
type probeRequest struct {
Remote string `json:"remote"`
Path string `json:"path"`
}
type probeResponse struct {
Status int `json:"status"`
Source string `json:"source,omitempty"`
ContentType string `json:"content_type,omitempty"`
SizeBytes int64 `json:"size_bytes"`
Headers map[string]string `json:"headers,omitempty"`
DurationMS int64 `json:"duration_ms"`
Error string `json:"error,omitempty"`
}
func (h *ProbeHandler) probe(w http.ResponseWriter, r *http.Request) {
var req probeRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
}
if req.Remote == "" || req.Path == "" {
http.Error(w, "remote and path are required", http.StatusBadRequest)
return
}
remote, err := h.db.GetRemote(r.Context(), req.Remote)
if err != nil {
writeJSON(w, http.StatusOK, probeResponse{
Status: 404,
Error: fmt.Sprintf("remote %q not found", req.Remote),
})
return
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
writeJSON(w, http.StatusOK, probeResponse{
Status: 500,
Error: fmt.Sprintf("no provider for %q", remote.PackageType),
})
return
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
start := time.Now()
result, err := h.engine.Fetch(ctx, *remote, req.Path, prov)
duration := time.Since(start).Milliseconds()
if err != nil {
writeJSON(w, http.StatusOK, probeResponse{
Status: 502,
DurationMS: duration,
Error: err.Error(),
})
return
}
io.Copy(io.Discard, result.Reader)
result.Reader.Close()
writeJSON(w, http.StatusOK, probeResponse{
Status: 200,
Source: result.Source,
ContentType: result.ContentType,
SizeBytes: result.Size,
DurationMS: duration,
Headers: map[string]string{
"X-Artifact-Source": result.Source,
"X-Artifact-Size": fmt.Sprintf("%d", result.Size),
"Content-Type": result.ContentType,
},
})
}
+96
View File
@@ -0,0 +1,96 @@
package v2
import (
"encoding/json"
"fmt"
"net/http"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type RemotesHandler struct {
db *database.DB
}
func NewRemotesHandler(db *database.DB) *RemotesHandler {
return &RemotesHandler{db: db}
}
func (h *RemotesHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.list)
r.Post("/", h.create)
r.Get("/{name}", h.get)
r.Put("/{name}", h.update)
r.Delete("/{name}", h.del)
return r
}
func (h *RemotesHandler) list(w http.ResponseWriter, r *http.Request) {
remotes, err := h.db.ListRemotes(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, remotes)
}
func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
remote, err := h.db.GetRemote(r.Context(), name)
if err != nil {
http.Error(w, fmt.Sprintf("remote %q not found", name), http.StatusNotFound)
return
}
writeJSON(w, http.StatusOK, remote)
}
func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) {
var remote models.Remote
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
}
if !remote.PackageType.Valid() {
http.Error(w, fmt.Sprintf("invalid package type: %q", remote.PackageType), http.StatusBadRequest)
return
}
if err := h.db.CreateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusCreated, remote)
}
func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
var remote models.Remote
if err := json.NewDecoder(r.Body).Decode(&remote); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
}
remote.Name = name
if err := h.db.UpdateRemote(r.Context(), &remote); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, remote)
}
func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
if err := h.db.DeleteRemote(r.Context(), name); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
func writeJSON(w http.ResponseWriter, status int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
json.NewEncoder(w).Encode(v)
}
+42
View File
@@ -0,0 +1,42 @@
package v2
import (
"net/http"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
)
type StatsHandler struct {
db *database.DB
}
func NewStatsHandler(db *database.DB) *StatsHandler {
return &StatsHandler{db: db}
}
func (h *StatsHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.overview)
r.Get("/top-remotes", h.topRemotes)
return r
}
func (h *StatsHandler) overview(w http.ResponseWriter, r *http.Request) {
stats, err := h.db.GetOverviewStats(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, stats)
}
func (h *StatsHandler) topRemotes(w http.ResponseWriter, r *http.Request) {
remotes, err := h.db.GetTopRemotes(r.Context(), 10)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, remotes)
}
+86
View File
@@ -0,0 +1,86 @@
package v2
import (
"encoding/json"
"fmt"
"net/http"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type VirtualsHandler struct {
db *database.DB
}
func NewVirtualsHandler(db *database.DB) *VirtualsHandler {
return &VirtualsHandler{db: db}
}
func (h *VirtualsHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/", h.list)
r.Post("/", h.create)
r.Get("/{name}", h.get)
r.Put("/{name}", h.update)
r.Delete("/{name}", h.del)
return r
}
func (h *VirtualsHandler) list(w http.ResponseWriter, r *http.Request) {
virtuals, err := h.db.ListVirtuals(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, virtuals)
}
func (h *VirtualsHandler) get(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
virt, err := h.db.GetVirtual(r.Context(), name)
if err != nil {
http.Error(w, fmt.Sprintf("virtual %q not found", name), http.StatusNotFound)
return
}
writeJSON(w, http.StatusOK, virt)
}
func (h *VirtualsHandler) create(w http.ResponseWriter, r *http.Request) {
var virt models.Virtual
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
}
if err := h.db.CreateVirtual(r.Context(), &virt); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusCreated, virt)
}
func (h *VirtualsHandler) update(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
var virt models.Virtual
if err := json.NewDecoder(r.Body).Decode(&virt); err != nil {
http.Error(w, "invalid json", http.StatusBadRequest)
return
}
virt.Name = name
if err := h.db.UpdateVirtual(r.Context(), &virt); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusOK, virt)
}
func (h *VirtualsHandler) del(w http.ResponseWriter, r *http.Request) {
name := chi.URLParam(r, "name")
if err := h.db.DeleteVirtual(r.Context(), name); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusNoContent)
}
+18
View File
@@ -0,0 +1,18 @@
package auth
import (
"encoding/base64"
"net/http"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func BasicHeaders(remote models.Remote) http.Header {
h := http.Header{}
if remote.Username != "" {
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString(
[]byte(remote.Username+":"+remote.Password),
))
}
return h
}
+105
View File
@@ -0,0 +1,105 @@
package cache
import (
"context"
"fmt"
"time"
"github.com/redis/go-redis/v9"
)
type Redis struct {
client *redis.Client
}
func NewRedis(url string) (*Redis, error) {
opts, err := redis.ParseURL(url)
if err != nil {
return nil, fmt.Errorf("parse redis url: %w", err)
}
client := redis.NewClient(opts)
if err := client.Ping(context.Background()).Err(); err != nil {
return nil, fmt.Errorf("ping redis: %w", err)
}
return &Redis{client: client}, nil
}
func (r *Redis) Close() error {
return r.client.Close()
}
func (r *Redis) SetTTL(ctx context.Context, remote, path string, ttl time.Duration) error {
key := fmt.Sprintf("ttl:%s:%s", remote, path)
return r.client.Set(ctx, key, "1", ttl).Err()
}
func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error) {
key := fmt.Sprintf("ttl:%s:%s", remote, path)
exists, err := r.client.Exists(ctx, key).Result()
if err != nil {
return false, err
}
return exists > 0, nil
}
func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
key := fmt.Sprintf("lock:%s:%s", remote, path)
ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
return ok, err
}
func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
key := fmt.Sprintf("lock:%s:%s", remote, path)
return r.client.Del(ctx, key).Err()
}
func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error {
key := fmt.Sprintf("etag:%s:%s", remote, path)
return r.client.Set(ctx, key, etag, ttl).Err()
}
func (r *Redis) GetETag(ctx context.Context, remote, path string) (string, error) {
key := fmt.Sprintf("etag:%s:%s", remote, path)
val, err := r.client.Get(ctx, key).Result()
if err == redis.Nil {
return "", nil
}
return val, err
}
func (r *Redis) IncrCircuitFailure(ctx context.Context, remote string, cooldown time.Duration) (int64, error) {
key := fmt.Sprintf("circuit:%s", remote)
pipe := r.client.Pipeline()
incr := pipe.Incr(ctx, key)
pipe.Expire(ctx, key, cooldown)
_, err := pipe.Exec(ctx)
if err != nil {
return 0, err
}
return incr.Val(), nil
}
func (r *Redis) ResetCircuit(ctx context.Context, remote string) error {
key := fmt.Sprintf("circuit:%s", remote)
return r.client.Del(ctx, key).Err()
}
func (r *Redis) GetCircuitFailures(ctx context.Context, remote string) (int64, error) {
key := fmt.Sprintf("circuit:%s", remote)
val, err := r.client.Get(ctx, key).Int64()
if err == redis.Nil {
return 0, nil
}
return val, err
}
func (r *Redis) FlushRemote(ctx context.Context, remote string) error {
iter := r.client.Scan(ctx, 0, fmt.Sprintf("*:%s:*", remote), 100).Iterator()
for iter.Next(ctx) {
r.client.Del(ctx, iter.Val())
}
return iter.Err()
}
+72
View File
@@ -0,0 +1,72 @@
package config
import (
"fmt"
"os"
"strconv"
)
type Config struct {
ListenAddr string
DBHost string
DBPort int
DBUser string
DBPass string
DBName string
DBSSL string
RedisURL string
S3Endpoint string
S3AccessKey string
S3SecretKey string
S3Bucket string
S3Secure bool
S3Region string
}
func (c *Config) DatabaseDSN() string {
return fmt.Sprintf(
"postgres://%s:%s@%s:%d/%s?sslmode=%s",
c.DBUser, c.DBPass, c.DBHost, c.DBPort, c.DBName, c.DBSSL,
)
}
func Load() (*Config, error) {
dbPort, err := strconv.Atoi(getenv("DBPORT", "5432"))
if err != nil {
return nil, fmt.Errorf("invalid DBPORT: %w", err)
}
s3Secure, _ := strconv.ParseBool(getenv("MINIO_SECURE", "false"))
cfg := &Config{
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
DBHost: getenv("DBHOST", "localhost"),
DBPort: dbPort,
DBUser: getenv("DBUSER", "artifacts"),
DBPass: getenv("DBPASS", ""),
DBName: getenv("DBNAME", "artifacts"),
DBSSL: getenv("DBSSL", "disable"),
RedisURL: getenv("REDIS_URL", "redis://localhost:6379"),
S3Endpoint: getenv("MINIO_ENDPOINT", "localhost:9000"),
S3AccessKey: getenv("MINIO_ACCESS_KEY", ""),
S3SecretKey: getenv("MINIO_SECRET_KEY", ""),
S3Bucket: getenv("MINIO_BUCKET", "artifacts"),
S3Secure: s3Secure,
S3Region: getenv("MINIO_REGION", ""),
}
return cfg, nil
}
func getenv(key, fallback string) string {
if v := os.Getenv(key); v != "" {
return v
}
return fallback
}
+153
View File
@@ -0,0 +1,153 @@
package database
import (
"context"
"time"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func (db *DB) UpsertBlob(ctx context.Context, contentHash, s3Key string, sizeBytes int64, contentType string) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO blobs (content_hash, s3_key, size_bytes, content_type)
VALUES ($1, $2, $3, $4)
ON CONFLICT (content_hash) DO NOTHING
`, contentHash, s3Key, sizeBytes, contentType)
return err
}
func (db *DB) UpsertArtifact(ctx context.Context, remoteName, path, contentHash, upstreamETag string) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO artifacts (remote_name, path, content_hash, upstream_etag)
VALUES ($1, $2, $3, $4)
ON CONFLICT (remote_name, path) DO UPDATE SET
content_hash = EXCLUDED.content_hash,
upstream_etag = EXCLUDED.upstream_etag,
last_fetched_at = NOW(),
fetch_count = artifacts.fetch_count + 1
`, remoteName, path, contentHash, upstreamETag)
return err
}
func (db *DB) GetArtifact(ctx context.Context, remoteName, path string) (*models.Artifact, error) {
row := db.Pool.QueryRow(ctx, `
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
a.last_accessed_at, a.fetch_count, a.access_count,
b.size_bytes, b.content_type
FROM artifacts a
JOIN blobs b ON a.content_hash = b.content_hash
WHERE a.remote_name = $1 AND a.path = $2
`, remoteName, path)
var a models.Artifact
err := row.Scan(
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
&a.SizeBytes, &a.ContentType,
)
if err != nil {
return nil, err
}
return &a, nil
}
func (db *DB) TouchArtifactAccess(ctx context.Context, remoteName, path string) error {
_, err := db.Pool.Exec(ctx, `
UPDATE artifacts SET
last_accessed_at = NOW(),
access_count = access_count + 1
WHERE remote_name = $1 AND path = $2
`, remoteName, path)
return err
}
func (db *DB) ListArtifacts(ctx context.Context, remoteName string, limit, offset int) ([]models.Artifact, error) {
rows, err := db.Pool.Query(ctx, `
SELECT a.id, a.remote_name, a.path, a.content_hash, a.upstream_etag,
a.upstream_last_modified, a.first_seen_at, a.last_fetched_at,
a.last_accessed_at, a.fetch_count, a.access_count,
b.size_bytes, b.content_type
FROM artifacts a
JOIN blobs b ON a.content_hash = b.content_hash
WHERE a.remote_name = $1
ORDER BY a.path
LIMIT $2 OFFSET $3
`, remoteName, limit, offset)
if err != nil {
return nil, err
}
defer rows.Close()
var artifacts []models.Artifact
for rows.Next() {
var a models.Artifact
if err := rows.Scan(
&a.ID, &a.RemoteName, &a.Path, &a.ContentHash, &a.UpstreamETag,
&a.UpstreamLastModified, &a.FirstSeenAt, &a.LastFetchedAt,
&a.LastAccessedAt, &a.FetchCount, &a.AccessCount,
&a.SizeBytes, &a.ContentType,
); err != nil {
return nil, err
}
artifacts = append(artifacts, a)
}
return artifacts, rows.Err()
}
func (db *DB) DeleteArtifact(ctx context.Context, remoteName, path string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM artifacts WHERE remote_name = $1 AND path = $2`, remoteName, path)
return err
}
func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cacheHit bool, sizeBytes int64, upstreamMS int, clientIP string) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO access_log (remote_name, path, cache_hit, size_bytes, upstream_ms, client_ip)
VALUES ($1, $2, $3, $4, $5, $6)
`, remoteName, path, cacheHit, sizeBytes, upstreamMS, clientIP)
return err
}
func (db *DB) FindOrphanedBlobs(ctx context.Context) ([]models.Blob, error) {
rows, err := db.Pool.Query(ctx, `
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
FROM blobs b
WHERE b.content_hash NOT IN (
SELECT content_hash FROM artifacts
UNION
SELECT content_hash FROM local_files
)
`)
if err != nil {
return nil, err
}
defer rows.Close()
var blobs []models.Blob
for rows.Next() {
var b models.Blob
if err := rows.Scan(&b.ContentHash, &b.S3Key, &b.SizeBytes, &b.ContentType, &b.CreatedAt); err != nil {
return nil, err
}
blobs = append(blobs, b)
}
return blobs, rows.Err()
}
func (db *DB) DeleteBlob(ctx context.Context, contentHash string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM blobs WHERE content_hash = $1`, contentHash)
return err
}
func (db *DB) DeleteColdArtifacts(ctx context.Context, remoteName string, olderThan time.Duration) (int64, error) {
cutoff := time.Now().Add(-olderThan)
tag, err := db.Pool.Exec(ctx, `
DELETE FROM artifacts
WHERE remote_name = $1 AND last_accessed_at < $2
`, remoteName, cutoff)
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
+126
View File
@@ -0,0 +1,126 @@
package database
import (
"context"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
)
type DB struct {
Pool *pgxpool.Pool
}
func New(dsn string) (*DB, error) {
pool, err := pgxpool.New(context.Background(), dsn)
if err != nil {
return nil, fmt.Errorf("connect to postgres: %w", err)
}
if err := pool.Ping(context.Background()); err != nil {
pool.Close()
return nil, fmt.Errorf("ping postgres: %w", err)
}
db := &DB{Pool: pool}
if err := db.migrate(); err != nil {
pool.Close()
return nil, fmt.Errorf("run migrations: %w", err)
}
return db, nil
}
func (db *DB) Close() {
db.Pool.Close()
}
func (db *DB) migrate() error {
ctx := context.Background()
_, err := db.Pool.Exec(ctx, `
CREATE TABLE IF NOT EXISTS remotes (
name TEXT PRIMARY KEY,
package_type TEXT NOT NULL,
base_url TEXT NOT NULL,
description TEXT DEFAULT '',
username TEXT DEFAULT '',
password TEXT DEFAULT '',
immutable_ttl INTEGER DEFAULT 0,
mutable_ttl INTEGER DEFAULT 3600,
check_mutable BOOLEAN DEFAULT TRUE,
patterns TEXT[] DEFAULT '{}',
blocklist TEXT[] DEFAULT '{}',
mutable_patterns TEXT[] DEFAULT '{}',
immutable_patterns TEXT[] DEFAULT '{}',
ban_tags_enabled BOOLEAN DEFAULT FALSE,
ban_tags TEXT[] DEFAULT '{}',
quarantine_enabled BOOLEAN DEFAULT FALSE,
quarantine_days INTEGER DEFAULT 3,
stale_on_error BOOLEAN DEFAULT TRUE,
releases_remote TEXT DEFAULT '',
managed_by TEXT DEFAULT '',
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS virtuals (
name TEXT PRIMARY KEY,
package_type TEXT NOT NULL,
description TEXT DEFAULT '',
members TEXT[] NOT NULL,
managed_by TEXT DEFAULT '',
created_at TIMESTAMPTZ DEFAULT NOW(),
updated_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS blobs (
content_hash TEXT PRIMARY KEY,
s3_key TEXT NOT NULL,
size_bytes BIGINT NOT NULL,
content_type TEXT DEFAULT 'application/octet-stream',
created_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS artifacts (
id BIGSERIAL PRIMARY KEY,
remote_name TEXT NOT NULL REFERENCES remotes(name) ON DELETE CASCADE,
path TEXT NOT NULL,
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
upstream_etag TEXT DEFAULT '',
upstream_last_modified TIMESTAMPTZ,
first_seen_at TIMESTAMPTZ DEFAULT NOW(),
last_fetched_at TIMESTAMPTZ DEFAULT NOW(),
last_accessed_at TIMESTAMPTZ DEFAULT NOW(),
fetch_count BIGINT DEFAULT 1,
access_count BIGINT DEFAULT 1,
UNIQUE(remote_name, path)
);
CREATE INDEX IF NOT EXISTS idx_artifacts_remote ON artifacts(remote_name);
CREATE INDEX IF NOT EXISTS idx_artifacts_last_accessed ON artifacts(last_accessed_at);
CREATE TABLE IF NOT EXISTS local_files (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL REFERENCES blobs(content_hash),
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE TABLE IF NOT EXISTS access_log (
id BIGSERIAL PRIMARY KEY,
remote_name TEXT NOT NULL,
path TEXT NOT NULL,
cache_hit BOOLEAN NOT NULL,
size_bytes BIGINT DEFAULT 0,
upstream_ms INTEGER DEFAULT 0,
client_ip TEXT DEFAULT '',
created_at TIMESTAMPTZ DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
`)
return err
}
+99
View File
@@ -0,0 +1,99 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const remoteCols = `name, package_type, base_url, description, username, password,
immutable_ttl, mutable_ttl, check_mutable,
patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by, created_at, updated_at`
func scanRemote(scanner interface{ Scan(...any) error }, r *models.Remote) error {
return scanner.Scan(
&r.Name, &r.PackageType, &r.BaseURL, &r.Description, &r.Username, &r.Password,
&r.ImmutableTTL, &r.MutableTTL, &r.CheckMutable,
&r.Patterns, &r.Blocklist, &r.MutablePatterns, &r.ImmutablePatterns,
&r.BanTagsEnabled, &r.BanTags,
&r.QuarantineEnabled, &r.QuarantineDays, &r.StaleOnError,
&r.ReleasesRemote, &r.ManagedBy, &r.CreatedAt, &r.UpdatedAt,
)
}
func (db *DB) GetRemote(ctx context.Context, name string) (*models.Remote, error) {
row := db.Pool.QueryRow(ctx, `SELECT `+remoteCols+` FROM remotes WHERE name = $1`, name)
var r models.Remote
if err := scanRemote(row, &r); err != nil {
return nil, err
}
return &r, nil
}
func (db *DB) ListRemotes(ctx context.Context) ([]models.Remote, error) {
rows, err := db.Pool.Query(ctx, `SELECT `+remoteCols+` FROM remotes ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
var remotes []models.Remote
for rows.Next() {
var r models.Remote
if err := scanRemote(rows, &r); err != nil {
return nil, err
}
remotes = append(remotes, r)
}
return remotes, rows.Err()
}
func (db *DB) CreateRemote(ctx context.Context, r *models.Remote) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO remotes (
name, package_type, base_url, description, username, password,
immutable_ttl, mutable_ttl, check_mutable,
patterns, blocklist, mutable_patterns, immutable_patterns,
ban_tags_enabled, ban_tags,
quarantine_enabled, quarantine_days, stale_on_error,
releases_remote, managed_by
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)
`,
r.Name, r.PackageType, r.BaseURL, r.Description, r.Username, r.Password,
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
r.BanTagsEnabled, r.BanTags,
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy,
)
return err
}
func (db *DB) UpdateRemote(ctx context.Context, r *models.Remote) error {
_, err := db.Pool.Exec(ctx, `
UPDATE remotes SET
package_type=$2, base_url=$3, description=$4, username=$5, password=$6,
immutable_ttl=$7, mutable_ttl=$8, check_mutable=$9,
patterns=$10, blocklist=$11, mutable_patterns=$12, immutable_patterns=$13,
ban_tags_enabled=$14, ban_tags=$15,
quarantine_enabled=$16, quarantine_days=$17, stale_on_error=$18,
releases_remote=$19, managed_by=$20, updated_at=NOW()
WHERE name=$1
`,
r.Name, r.PackageType, r.BaseURL, r.Description, r.Username, r.Password,
r.ImmutableTTL, r.MutableTTL, r.CheckMutable,
r.Patterns, r.Blocklist, r.MutablePatterns, r.ImmutablePatterns,
r.BanTagsEnabled, r.BanTags,
r.QuarantineEnabled, r.QuarantineDays, r.StaleOnError,
r.ReleasesRemote, r.ManagedBy,
)
return err
}
func (db *DB) DeleteRemote(ctx context.Context, name string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM remotes WHERE name = $1`, name)
return err
}
+78
View File
@@ -0,0 +1,78 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func (db *DB) GetOverviewStats(ctx context.Context) (*models.OverviewStats, error) {
var stats models.OverviewStats
err := db.Pool.QueryRow(ctx, `SELECT COUNT(*) FROM remotes`).Scan(&stats.TotalRemotes)
if err != nil {
return nil, err
}
err = db.Pool.QueryRow(ctx, `SELECT COALESCE(COUNT(*), 0), COALESCE(SUM(b.size_bytes), 0)
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash`).
Scan(&stats.TotalObjects, &stats.TotalBytes)
if err != nil {
return nil, err
}
err = db.Pool.QueryRow(ctx, `
SELECT COALESCE(
(SELECT COUNT(*) FROM artifacts) - (SELECT COUNT(DISTINCT content_hash) FROM artifacts),
0
)`).Scan(&stats.TotalBlobsDeduped)
if err != nil {
return nil, err
}
return &stats, nil
}
type RemoteStatRow struct {
Name string `json:"name"`
ObjectCount int64 `json:"object_count"`
TotalBytes int64 `json:"total_bytes"`
Requests30d int64 `json:"requests_30d"`
}
func (db *DB) GetTopRemotes(ctx context.Context, limit int) ([]RemoteStatRow, error) {
rows, err := db.Pool.Query(ctx, `
SELECT r.name,
COALESCE(a.cnt, 0) AS object_count,
COALESCE(a.total_bytes, 0) AS total_bytes,
COALESCE(l.req_count, 0) AS requests_30d
FROM remotes r
LEFT JOIN (
SELECT remote_name, COUNT(*) AS cnt, SUM(b.size_bytes) AS total_bytes
FROM artifacts a JOIN blobs b ON a.content_hash = b.content_hash
GROUP BY remote_name
) a ON r.name = a.remote_name
LEFT JOIN (
SELECT remote_name, COUNT(*) AS req_count
FROM access_log
WHERE created_at > NOW() - INTERVAL '30 days'
GROUP BY remote_name
) l ON r.name = l.remote_name
ORDER BY COALESCE(a.total_bytes, 0) DESC
LIMIT $1
`, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var result []RemoteStatRow
for rows.Next() {
var r RemoteStatRow
if err := rows.Scan(&r.Name, &r.ObjectCount, &r.TotalBytes, &r.Requests30d); err != nil {
return nil, err
}
result = append(result, r)
}
return result, rows.Err()
}
+64
View File
@@ -0,0 +1,64 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func (db *DB) GetVirtual(ctx context.Context, name string) (*models.Virtual, error) {
row := db.Pool.QueryRow(ctx, `
SELECT name, package_type, description, members, managed_by, created_at, updated_at
FROM virtuals WHERE name = $1
`, name)
var v models.Virtual
err := row.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt)
if err != nil {
return nil, err
}
return &v, nil
}
func (db *DB) ListVirtuals(ctx context.Context) ([]models.Virtual, error) {
rows, err := db.Pool.Query(ctx, `
SELECT name, package_type, description, members, managed_by, created_at, updated_at
FROM virtuals ORDER BY name
`)
if err != nil {
return nil, err
}
defer rows.Close()
var virtuals []models.Virtual
for rows.Next() {
var v models.Virtual
if err := rows.Scan(&v.Name, &v.PackageType, &v.Description, &v.Members, &v.ManagedBy, &v.CreatedAt, &v.UpdatedAt); err != nil {
return nil, err
}
virtuals = append(virtuals, v)
}
return virtuals, rows.Err()
}
func (db *DB) CreateVirtual(ctx context.Context, v *models.Virtual) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO virtuals (name, package_type, description, members, managed_by)
VALUES ($1, $2, $3, $4, $5)
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
return err
}
func (db *DB) UpdateVirtual(ctx context.Context, v *models.Virtual) error {
_, err := db.Pool.Exec(ctx, `
UPDATE virtuals SET
package_type=$2, description=$3, members=$4, managed_by=$5, updated_at=NOW()
WHERE name=$1
`, v.Name, v.PackageType, v.Description, v.Members, v.ManagedBy)
return err
}
func (db *DB) DeleteVirtual(ctx context.Context, name string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM virtuals WHERE name = $1`, name)
return err
}
+67
View File
@@ -0,0 +1,67 @@
package gc
import (
"context"
"log/slog"
"time"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/storage"
)
type Collector struct {
db *database.DB
store *storage.S3
interval time.Duration
}
func New(db *database.DB, store *storage.S3, interval time.Duration) *Collector {
return &Collector{db: db, store: store, interval: interval}
}
func (c *Collector) Run(ctx context.Context) {
slog.Info("gc started", "interval", c.interval)
ticker := time.NewTicker(c.interval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
slog.Info("gc stopped")
return
case <-ticker.C:
c.sweep(ctx)
}
}
}
func (c *Collector) sweep(ctx context.Context) {
start := time.Now()
orphaned, err := c.db.FindOrphanedBlobs(ctx)
if err != nil {
slog.Error("gc: find orphaned blobs", "error", err)
return
}
deleted := 0
for _, blob := range orphaned {
if err := c.store.Delete(ctx, blob.S3Key); err != nil {
slog.Warn("gc: delete s3 object", "key", blob.S3Key, "error", err)
continue
}
if err := c.db.DeleteBlob(ctx, blob.ContentHash); err != nil {
slog.Warn("gc: delete blob row", "hash", blob.ContentHash, "error", err)
continue
}
deleted++
}
if deleted > 0 || len(orphaned) > 0 {
slog.Info("gc sweep complete",
"orphaned_found", len(orphaned),
"deleted", deleted,
"duration_ms", time.Since(start).Milliseconds(),
)
}
}
+15
View File
@@ -0,0 +1,15 @@
package gc_test
import (
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/gc"
)
func TestNew(t *testing.T) {
c := gc.New(nil, nil, 1*time.Hour)
if c == nil {
t.Fatal("expected non-nil collector")
}
}
+48
View File
@@ -0,0 +1,48 @@
package alpine
import (
"context"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageAlpine }
func (p *Provider) Classify(path string) provider.Mutability {
if strings.HasSuffix(path, "APKINDEX.tar.gz") {
return provider.Mutable
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
if strings.HasSuffix(path, ".apk") {
return "application/vnd.android.package-archive"
}
if strings.HasSuffix(path, ".tar.gz") {
return "application/gzip"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
+62
View File
@@ -0,0 +1,62 @@
package docker
import (
"context"
"encoding/base64"
"net/http"
"regexp"
"strings"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
var (
tagManifestRe = regexp.MustCompile(`/manifests/[^/]+$`)
digestManifestRe = regexp.MustCompile(`/manifests/sha256:[0-9a-fA-F]+$`)
tagsListRe = regexp.MustCompile(`/tags/list$`)
)
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageDocker }
func (p *Provider) Classify(path string) provider.Mutability {
if tagsListRe.MatchString(path) {
return provider.Mutable
}
if tagManifestRe.MatchString(path) && !digestManifestRe.MatchString(path) {
return provider.Mutable
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
if strings.Contains(path, "/blobs/") {
return "application/octet-stream"
}
if strings.Contains(path, "/manifests/") {
return "application/vnd.docker.distribution.manifest.v2+json"
}
return "application/json"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/v2/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
h := http.Header{}
if remote.Username != "" && remote.Password != "" {
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
}
return h, nil
}
+54
View File
@@ -0,0 +1,54 @@
package docker_test
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestProvider_Type(t *testing.T) {
p := &docker.Provider{}
if p.Type() != models.PackageDocker {
t.Errorf("expected docker, got %q", p.Type())
}
}
func TestProvider_Classify(t *testing.T) {
p := &docker.Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"library/nginx/manifests/latest", provider.Mutable},
{"library/nginx/manifests/v1.25", provider.Mutable},
{"library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890", provider.Immutable},
{"library/nginx/tags/list", provider.Mutable},
{"library/nginx/blobs/sha256:abc123", provider.Immutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestProvider_UpstreamURL(t *testing.T) {
p := &docker.Provider{}
got := p.UpstreamURL(models.Remote{BaseURL: "https://registry-1.docker.io"}, "library/nginx/manifests/latest")
want := "https://registry-1.docker.io/v2/library/nginx/manifests/latest"
if got != want {
t.Errorf("got %q, want %q", got, want)
}
}
func TestProvider_ContentType(t *testing.T) {
p := &docker.Provider{}
if p.ContentType("x/blobs/sha256:abc") != "application/octet-stream" {
t.Error("blobs should be octet-stream")
}
if p.ContentType("x/manifests/latest") != "application/vnd.docker.distribution.manifest.v2+json" {
t.Error("manifests should be manifest type")
}
}
+68
View File
@@ -0,0 +1,68 @@
package generic
import (
"context"
"encoding/base64"
"net/http"
"path"
"strings"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageGeneric }
func (p *Provider) Classify(_ string) provider.Mutability {
return provider.Immutable
}
var contentTypeMap = map[string]string{
".tar.gz": "application/gzip",
".tgz": "application/gzip",
".gz": "application/gzip",
".zip": "application/zip",
".whl": "application/zip",
".exe": "application/x-msdownload",
".rpm": "application/x-rpm",
".xml": "application/xml",
".yaml": "text/yaml",
".yml": "text/yaml",
".json": "application/json",
".sig": "application/octet-stream",
}
func (p *Provider) ContentType(filePath string) string {
lower := strings.ToLower(filePath)
if strings.HasSuffix(lower, ".tar.gz") {
return "application/gzip"
}
ext := path.Ext(lower)
if ct, ok := contentTypeMap[ext]; ok {
return ct
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, reqPath string) string {
base := strings.TrimRight(remote.BaseURL, "/")
return base + "/" + strings.TrimLeft(reqPath, "/")
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
h := http.Header{}
if remote.Username != "" {
h.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(remote.Username+":"+remote.Password)))
}
return h, nil
}
+69
View File
@@ -0,0 +1,69 @@
package generic_test
import (
"context"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestProvider_Type(t *testing.T) {
p := &generic.Provider{}
if p.Type() != models.PackageGeneric {
t.Errorf("expected generic, got %q", p.Type())
}
}
func TestProvider_Classify_AllImmutable(t *testing.T) {
p := &generic.Provider{}
paths := []string{"file.tar.gz", "path/to/binary", "index.html", "data.json"}
for _, path := range paths {
if p.Classify(path) != provider.Immutable {
t.Errorf("generic should classify %q as immutable", path)
}
}
}
func TestProvider_ContentType(t *testing.T) {
p := &generic.Provider{}
tests := []struct{ path, want string }{
{"file.tar.gz", "application/gzip"},
{"file.tgz", "application/gzip"},
{"file.zip", "application/zip"},
{"file.rpm", "application/x-rpm"},
{"file.json", "application/json"},
{"file.unknown", "application/octet-stream"},
}
for _, tt := range tests {
if got := p.ContentType(tt.path); got != tt.want {
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
}
}
}
func TestProvider_UpstreamURL(t *testing.T) {
p := &generic.Provider{}
got := p.UpstreamURL(models.Remote{BaseURL: "https://example.com/repo"}, "path/to/file.tar.gz")
want := "https://example.com/repo/path/to/file.tar.gz"
if got != want {
t.Errorf("got %q, want %q", got, want)
}
}
func TestProvider_AuthHeaders_BasicAuth(t *testing.T) {
p := &generic.Provider{}
h, _ := p.AuthHeaders(context.Background(), models.Remote{Username: "user", Password: "pass"})
if h.Get("Authorization") != "Basic dXNlcjpwYXNz" {
t.Errorf("unexpected auth header: %q", h.Get("Authorization"))
}
}
func TestProvider_AuthHeaders_NoAuth(t *testing.T) {
p := &generic.Provider{}
h, _ := p.AuthHeaders(context.Background(), models.Remote{})
if h.Get("Authorization") != "" {
t.Error("expected no auth header")
}
}
+54
View File
@@ -0,0 +1,54 @@
package goproxy
import (
"context"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageGoProxy }
func (p *Provider) Classify(path string) provider.Mutability {
if strings.HasSuffix(path, "/@v/list") || strings.HasSuffix(path, "/@latest") {
return provider.Mutable
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
if strings.HasSuffix(path, ".zip") {
return "application/zip"
}
if strings.HasSuffix(path, ".mod") {
return "text/plain"
}
if strings.HasSuffix(path, ".info") {
return "application/json"
}
if strings.HasSuffix(path, "/list") {
return "text/plain"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
+50
View File
@@ -0,0 +1,50 @@
package goproxy_test
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestProvider_Type(t *testing.T) {
p := &goproxy.Provider{}
if p.Type() != models.PackageGoProxy {
t.Errorf("expected goproxy, got %q", p.Type())
}
}
func TestProvider_Classify(t *testing.T) {
p := &goproxy.Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"golang.org/x/net/@v/list", provider.Mutable},
{"golang.org/x/net/@latest", provider.Mutable},
{"golang.org/x/net/@v/v0.1.0.info", provider.Immutable},
{"golang.org/x/net/@v/v0.1.0.mod", provider.Immutable},
{"golang.org/x/net/@v/v0.1.0.zip", provider.Immutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestProvider_ContentType(t *testing.T) {
p := &goproxy.Provider{}
tests := []struct{ path, want string }{
{"m/@v/v1.0.0.zip", "application/zip"},
{"m/@v/v1.0.0.mod", "text/plain"},
{"m/@v/v1.0.0.info", "application/json"},
{"m/@v/list", "text/plain"},
}
for _, tt := range tests {
if got := p.ContentType(tt.path); got != tt.want {
t.Errorf("ContentType(%q) = %q, want %q", tt.path, got, tt.want)
}
}
}
+58
View File
@@ -0,0 +1,58 @@
package helm
import (
"context"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageHelm }
func (p *Provider) Classify(path string) provider.Mutability {
if strings.HasSuffix(path, "index.yaml") || strings.HasSuffix(path, "index.yml") {
return provider.Mutable
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
if strings.HasSuffix(path, ".tgz") || strings.HasSuffix(path, ".tar.gz") {
return "application/gzip"
}
if strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") {
return "text/yaml"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" {
return nil, nil
}
content := string(body)
baseURL := strings.TrimRight(remote.BaseURL, "/")
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
if rewritten == content {
return nil, nil
}
return []byte(rewritten), nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
+51
View File
@@ -0,0 +1,51 @@
package helm_test
import (
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestProvider_Type(t *testing.T) {
p := &helm.Provider{}
if p.Type() != models.PackageHelm {
t.Errorf("expected helm, got %q", p.Type())
}
}
func TestProvider_Classify(t *testing.T) {
p := &helm.Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"index.yaml", provider.Mutable},
{"index.yml", provider.Mutable},
{"chart-1.0.tgz", provider.Immutable},
{"charts/nginx-1.0.tgz", provider.Immutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestProvider_RewriteResponse(t *testing.T) {
p := &helm.Provider{}
body := []byte("urls:\n- https://charts.example.com/chart-1.0.tgz")
remote := models.Remote{Name: "helm-test", BaseURL: "https://charts.example.com"}
rewritten, err := p.RewriteResponse(body, remote, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
if rewritten == nil {
t.Fatal("expected rewrite")
}
if !strings.Contains(string(rewritten), "proxy.example.com/api/v1/remote/helm-test") {
t.Errorf("expected proxy URL in body: %s", rewritten)
}
}
+56
View File
@@ -0,0 +1,56 @@
package npm
import (
"context"
"encoding/json"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageNPM }
func (p *Provider) Classify(path string) provider.Mutability {
if strings.HasSuffix(path, ".tgz") {
return provider.Immutable
}
return provider.Mutable
}
func (p *Provider) ContentType(path string) string {
if strings.HasSuffix(path, ".tgz") {
return "application/gzip"
}
return "application/json"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" || !json.Valid(body) {
return nil, nil
}
content := string(body)
baseURL := strings.TrimRight(remote.BaseURL, "/")
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
rewritten := strings.ReplaceAll(content, baseURL, proxyURL)
if rewritten == content {
return nil, nil
}
return []byte(rewritten), nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
+52
View File
@@ -0,0 +1,52 @@
package provider
import (
"context"
"fmt"
"net/http"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type Mutability int
const (
Immutable Mutability = iota
Mutable
)
type Provider interface {
Type() models.PackageType
Classify(path string) Mutability
ContentType(path string) string
UpstreamURL(remote models.Remote, path string) string
RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error)
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
}
type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
}
type MemberIndex struct {
RemoteName string
Body []byte
}
var registry = map[models.PackageType]Provider{}
func Register(p Provider) {
registry[p.Type()] = p
}
func Get(t models.PackageType) (Provider, error) {
p, ok := registry[t]
if !ok {
return nil, fmt.Errorf("no provider registered for package type %q", t)
}
return p, nil
}
func All() map[models.PackageType]Provider {
return registry
}
+56
View File
@@ -0,0 +1,56 @@
package puppet
import (
"context"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackagePuppet }
func (p *Provider) Classify(path string) provider.Mutability {
if strings.HasPrefix(path, "v3/modules/") || strings.HasPrefix(path, "v3/releases") {
return provider.Mutable
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
if strings.HasSuffix(path, ".tar.gz") {
return "application/gzip"
}
if strings.HasPrefix(path, "v3/") {
return "application/json"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" {
return nil, nil
}
content := string(body)
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
content = strings.ReplaceAll(content, `"/v3/files/`, `"`+proxyURL+`/v3/files/`)
baseURL := strings.TrimRight(remote.BaseURL, "/")
content = strings.ReplaceAll(content, baseURL, proxyURL)
return []byte(content), nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
+62
View File
@@ -0,0 +1,62 @@
package pypi
import (
"context"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackagePyPI }
func (p *Provider) Classify(path string) provider.Mutability {
if strings.Contains(path, "simple/") {
return provider.Mutable
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
lower := strings.ToLower(path)
if strings.HasSuffix(lower, ".whl") || strings.HasSuffix(lower, ".zip") {
return "application/zip"
}
if strings.HasSuffix(lower, ".tar.gz") {
return "application/gzip"
}
if strings.Contains(path, "simple/") {
return "text/html"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
if strings.HasPrefix(path, "simple/") {
return "https://pypi.org/" + path
}
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if proxyBaseURL == "" {
return nil, nil
}
content := string(body)
proxyURL := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name + "/"
content = strings.ReplaceAll(content, "https://files.pythonhosted.org/", proxyURL)
content = strings.ReplaceAll(content, "../../", proxyURL)
return []byte(content), nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
+57
View File
@@ -0,0 +1,57 @@
package rpm
import (
"context"
"net/http"
"regexp"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
var mutableRe = []*regexp.Regexp{
regexp.MustCompile(`repomd\.xml$`),
regexp.MustCompile(`repodata/`),
regexp.MustCompile(`Packages\.gz$`),
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageRPM }
func (p *Provider) Classify(path string) provider.Mutability {
for _, re := range mutableRe {
if re.MatchString(path) {
return provider.Mutable
}
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
if strings.HasSuffix(path, ".rpm") {
return "application/x-rpm"
}
if strings.HasSuffix(path, ".xml") || strings.HasSuffix(path, ".xml.gz") || strings.HasSuffix(path, ".xml.xz") {
return "application/xml"
}
return "application/octet-stream"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
+35
View File
@@ -0,0 +1,35 @@
package rpm_test
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestProvider_Type(t *testing.T) {
p := &rpm.Provider{}
if p.Type() != models.PackageRPM {
t.Errorf("expected rpm, got %q", p.Type())
}
}
func TestProvider_Classify(t *testing.T) {
p := &rpm.Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"repomd.xml", provider.Mutable},
{"repodata/primary.xml.gz", provider.Mutable},
{"Packages.gz", provider.Mutable},
{"package-1.0.rpm", provider.Immutable},
{"RPM-GPG-KEY-almalinux", provider.Immutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
+88
View File
@@ -0,0 +1,88 @@
package terraform
import (
"context"
"encoding/json"
"net/http"
"net/url"
"regexp"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`)
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageTerraform }
func (p *Provider) Classify(path string) provider.Mutability {
if versionsRe.MatchString(path) {
return provider.Mutable
}
return provider.Immutable
}
func (p *Provider) ContentType(path string) string {
lower := strings.ToLower(path)
if strings.HasSuffix(lower, ".zip") {
return "application/zip"
}
if strings.HasSuffix(lower, ".sig") {
return "application/octet-stream"
}
return "application/json"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/v1/providers/" + strings.TrimLeft(path, "/")
}
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseURL string) ([]byte, error) {
if remote.ReleasesRemote == "" {
return nil, nil
}
if !json.Valid(body) {
return nil, nil
}
var data map[string]any
if err := json.Unmarshal(body, &data); err != nil {
return nil, nil
}
changed := false
for _, field := range []string{"download_url", "shasums_url", "shasums_signature_url"} {
if val, ok := data[field].(string); ok && val != "" {
rewritten := rewriteDownloadURL(val, remote.ReleasesRemote, proxyBaseURL)
if rewritten != val {
data[field] = rewritten
changed = true
}
}
}
if !changed {
return nil, nil
}
return json.Marshal(data)
}
func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string {
parsed, err := url.Parse(originalURL)
if err != nil || proxyBaseURL == "" {
return originalURL
}
return strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + releasesRemote + parsed.Path
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
@@ -0,0 +1,55 @@
package terraform_test
import (
"encoding/json"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestProvider_Type(t *testing.T) {
p := &terraform.Provider{}
if p.Type() != models.PackageTerraform {
t.Errorf("expected terraform, got %q", p.Type())
}
}
func TestProvider_Classify(t *testing.T) {
p := &terraform.Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"hashicorp/vault/versions", provider.Mutable},
{"hashicorp/vault/0.28.0/download/linux/amd64", provider.Immutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestProvider_RewriteResponse_DownloadInfo(t *testing.T) {
p := &terraform.Provider{}
remote := models.Remote{Name: "tf", ReleasesRemote: "hashicorp-releases"}
body, _ := json.Marshal(map[string]any{
"download_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/file.zip",
"shasums_url": "https://releases.hashicorp.com/terraform-provider-vault/0.28.0/SHA256SUMS",
})
rewritten, err := p.RewriteResponse(body, remote, "https://proxy")
if err != nil {
t.Fatal(err)
}
if rewritten == nil {
t.Fatal("expected rewrite")
}
var result map[string]any
json.Unmarshal(rewritten, &result)
if !strings.Contains(result["download_url"].(string), "proxy/api/v1/remote/hashicorp-releases") {
t.Errorf("download_url not rewritten: %s", result["download_url"])
}
}
+60
View File
@@ -0,0 +1,60 @@
package proxy
import (
"context"
"time"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const (
defaultCircuitThreshold = 5
defaultCircuitCooldown = 60 * time.Second
)
type CircuitBreaker struct {
cache *cache.Redis
threshold int64
cooldown time.Duration
}
func NewCircuitBreaker(c *cache.Redis) *CircuitBreaker {
return &CircuitBreaker{
cache: c,
threshold: defaultCircuitThreshold,
cooldown: defaultCircuitCooldown,
}
}
func (cb *CircuitBreaker) IsOpen(ctx context.Context, remote string) bool {
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
if err != nil {
return false
}
return failures >= cb.threshold
}
func (cb *CircuitBreaker) RecordFailure(ctx context.Context, remote string) {
cb.cache.IncrCircuitFailure(ctx, remote, cb.cooldown)
}
func (cb *CircuitBreaker) RecordSuccess(ctx context.Context, remote string) {
cb.cache.ResetCircuit(ctx, remote)
}
func (cb *CircuitBreaker) Health(ctx context.Context, remote string) models.RemoteHealth {
failures, err := cb.cache.GetCircuitFailures(ctx, remote)
if err != nil {
return models.RemoteHealth{Status: "unknown"}
}
switch {
case failures == 0:
return models.RemoteHealth{Status: "healthy", ConsecutiveFailures: int(failures)}
case failures < cb.threshold:
return models.RemoteHealth{Status: "degraded", ConsecutiveFailures: int(failures)}
default:
return models.RemoteHealth{Status: "down", ConsecutiveFailures: int(failures)}
}
}
+14
View File
@@ -0,0 +1,14 @@
package proxy_test
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/proxy"
)
func TestCircuitBreaker_New(t *testing.T) {
cb := proxy.NewCircuitBreaker(nil)
if cb == nil {
t.Fatal("expected non-nil circuit breaker")
}
}
+80
View File
@@ -0,0 +1,80 @@
package proxy
import (
"regexp"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type Classification int
const (
ClassImmutable Classification = iota
ClassMutable
ClassDenied
)
func (c Classification) String() string {
switch c {
case ClassImmutable:
return "immutable"
case ClassMutable:
return "mutable"
case ClassDenied:
return "denied"
default:
return "unknown"
}
}
type Classifier struct {
provider provider.Provider
}
func NewClassifier(p provider.Provider) *Classifier {
return &Classifier{provider: p}
}
func (c *Classifier) Classify(remote models.Remote, path string) Classification {
if matchesAny(path, compilePatterns(remote.Blocklist)) {
return ClassDenied
}
if len(remote.Patterns) > 0 && !matchesAny(path, compilePatterns(remote.Patterns)) {
return ClassDenied
}
if matchesAny(path, compilePatterns(remote.ImmutablePatterns)) {
return ClassImmutable
}
if matchesAny(path, compilePatterns(remote.MutablePatterns)) {
return ClassMutable
}
if c.provider.Classify(path) == provider.Mutable {
return ClassMutable
}
return ClassImmutable
}
func compilePatterns(patterns []string) []*regexp.Regexp {
compiled := make([]*regexp.Regexp, 0, len(patterns))
for _, p := range patterns {
if re, err := regexp.Compile(p); err == nil {
compiled = append(compiled, re)
}
}
return compiled
}
func matchesAny(path string, patterns []*regexp.Regexp) bool {
for _, re := range patterns {
if re.MatchString(path) {
return true
}
}
return false
}
+129
View File
@@ -0,0 +1,129 @@
package proxy_test
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider/docker"
"git.unkin.net/unkin/artifactapi/internal/provider/generic"
"git.unkin.net/unkin/artifactapi/internal/provider/helm"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestClassifier_EmptyPatternsAllowsAll(t *testing.T) {
c := proxy.NewClassifier(&generic.Provider{})
remote := models.Remote{Name: "test"}
if c.Classify(remote, "any/path") == proxy.ClassDenied {
t.Error("empty patterns should allow all paths")
}
}
func TestClassifier_PatternsActAsAllowlist(t *testing.T) {
c := proxy.NewClassifier(&generic.Provider{})
remote := models.Remote{
Name: "test",
Patterns: []string{`^releases/`},
}
if c.Classify(remote, "releases/v1.0/app.tar.gz") == proxy.ClassDenied {
t.Error("path matching patterns should be allowed")
}
if c.Classify(remote, "uploads/other.tar.gz") != proxy.ClassDenied {
t.Error("path not matching patterns should be denied")
}
}
func TestClassifier_BlocklistDenies(t *testing.T) {
c := proxy.NewClassifier(&generic.Provider{})
remote := models.Remote{
Name: "test",
Blocklist: []string{`\.exe$`},
}
if c.Classify(remote, "malware.exe") != proxy.ClassDenied {
t.Error("blocklist match should deny")
}
if c.Classify(remote, "legit.tar.gz") == proxy.ClassDenied {
t.Error("non-blocked path should be allowed")
}
}
func TestClassifier_BlocklistBeforePatterns(t *testing.T) {
c := proxy.NewClassifier(&generic.Provider{})
remote := models.Remote{
Name: "test",
Patterns: []string{`^releases/`},
Blocklist: []string{`releases/v0\.1/`},
}
if c.Classify(remote, "releases/v0.1/app.tar.gz") != proxy.ClassDenied {
t.Error("blocklist should take priority")
}
}
func TestClassifier_GenericAllImmutable(t *testing.T) {
c := proxy.NewClassifier(&generic.Provider{})
remote := models.Remote{Name: "test"}
if c.Classify(remote, "any/file.tar.gz") != proxy.ClassImmutable {
t.Error("generic provider should classify everything as immutable")
}
}
func TestClassifier_GenericMutableOverride(t *testing.T) {
c := proxy.NewClassifier(&generic.Provider{})
remote := models.Remote{
Name: "test",
MutablePatterns: []string{`/archive/refs/heads/`},
}
if c.Classify(remote, "repo/archive/refs/heads/main.tar.gz") != proxy.ClassMutable {
t.Error("mutable_patterns should override provider default")
}
if c.Classify(remote, "repo/releases/v1.0.tar.gz") != proxy.ClassImmutable {
t.Error("non-mutable path should stay immutable")
}
}
func TestClassifier_ImmutableOverride(t *testing.T) {
c := proxy.NewClassifier(&helm.Provider{})
remote := models.Remote{
Name: "test",
ImmutablePatterns: []string{`special-index\.yaml$`},
}
if c.Classify(remote, "special-index.yaml") != proxy.ClassImmutable {
t.Error("immutable_patterns should force immutable even for normally mutable paths")
}
}
func TestClassifier_HelmAutoClassifies(t *testing.T) {
c := proxy.NewClassifier(&helm.Provider{})
remote := models.Remote{Name: "test"}
if c.Classify(remote, "index.yaml") != proxy.ClassMutable {
t.Error("helm should auto-classify index.yaml as mutable")
}
if c.Classify(remote, "chart-1.0.tgz") != proxy.ClassImmutable {
t.Error("helm should auto-classify .tgz as immutable")
}
}
func TestClassifier_DockerAutoClassifies(t *testing.T) {
c := proxy.NewClassifier(&docker.Provider{})
remote := models.Remote{Name: "test"}
if c.Classify(remote, "library/nginx/manifests/latest") != proxy.ClassMutable {
t.Error("docker should classify tag manifest as mutable")
}
if c.Classify(remote, "library/nginx/manifests/sha256:abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890") != proxy.ClassImmutable {
t.Error("docker should classify digest manifest as immutable")
}
if c.Classify(remote, "library/nginx/blobs/sha256:abc") != proxy.ClassImmutable {
t.Error("docker should classify blobs as immutable")
}
}
func TestClassifier_RPMAutoClassifies(t *testing.T) {
c := proxy.NewClassifier(&rpm.Provider{})
remote := models.Remote{Name: "test"}
if c.Classify(remote, "repodata/primary.xml.gz") != proxy.ClassMutable {
t.Error("rpm should classify repodata as mutable")
}
if c.Classify(remote, "packages/foo-1.0.rpm") != proxy.ClassImmutable {
t.Error("rpm should classify .rpm as immutable")
}
}
+341
View File
@@ -0,0 +1,341 @@
package proxy
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"log/slog"
"net/http"
"time"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
const fetchLockTTL = 30 * time.Second
type Engine struct {
db *database.DB
cache *cache.Redis
store *storage.S3
cas *storage.CAS
}
func NewEngine(db *database.DB, c *cache.Redis, s *storage.S3) *Engine {
return &Engine{
db: db,
cache: c,
store: s,
cas: storage.NewCAS(s),
}
}
type FetchResult struct {
Reader io.ReadCloser
ContentType string
Size int64
Source string // "cache" or "remote"
}
func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*FetchResult, error) {
classifier := NewClassifier(prov)
class := classifier.Classify(remote, path)
if class == ClassDenied {
return nil, &ProxyError{Status: http.StatusForbidden, Message: "access denied"}
}
ttl := e.ttlFor(remote, class)
fresh, err := e.cache.CheckTTL(ctx, remote.Name, path)
if err != nil {
slog.Warn("redis check failed, treating as miss", "error", err)
}
if fresh {
result, err := e.serveFromStore(ctx, remote, path)
if err == nil {
result.Source = "cache"
go e.logAccess(remote.Name, path, true, result.Size, 0)
return result, nil
}
slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path)
}
locked, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
if err != nil {
slog.Warn("lock acquire failed", "error", err)
}
if !locked {
time.Sleep(500 * time.Millisecond)
result, err := e.serveFromStore(ctx, remote, path)
if err == nil {
result.Source = "cache"
go e.logAccess(remote.Name, path, true, result.Size, 0)
return result, nil
}
}
if locked {
defer e.cache.ReleaseLock(ctx, remote.Name, path)
}
if class == ClassMutable && remote.CheckMutable {
etag, _ := e.cache.GetETag(ctx, remote.Name, path)
if etag != "" {
notModified, err := e.checkUpstream(ctx, remote, path, etag, prov)
if err == nil && notModified {
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
result, err := e.serveFromStore(ctx, remote, path)
if err == nil {
result.Source = "cache"
go e.logAccess(remote.Name, path, true, result.Size, 0)
return result, nil
}
}
}
}
start := time.Now()
result, err := e.fetchFromUpstream(ctx, remote, path, prov, class, ttl)
upstreamMS := int(time.Since(start).Milliseconds())
if err != nil {
if remote.StaleOnError && isNetworkError(err) {
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
stale, serr := e.serveFromStore(ctx, remote, path)
if serr == nil {
slog.Warn("serving stale on upstream error", "remote", remote.Name, "path", path, "error", err)
stale.Source = "cache"
go e.logAccess(remote.Name, path, true, stale.Size, 0)
return stale, nil
}
}
return nil, err
}
go e.logAccess(remote.Name, path, false, result.Size, upstreamMS)
return result, nil
}
func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration) (*FetchResult, error) {
url := prov.UpstreamURL(remote, path)
authHeaders, err := prov.AuthHeaders(ctx, remote)
if err != nil {
return nil, fmt.Errorf("auth headers: %w", err)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("create request: %w", err)
}
for k, vv := range authHeaders {
for _, v := range vv {
req.Header.Add(k, v)
}
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, &UpstreamError{Err: err}
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
}
body, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
return nil, fmt.Errorf("read upstream body: %w", err)
}
rewritten, err := prov.RewriteResponse(body, remote, "")
if err != nil {
return nil, fmt.Errorf("rewrite response: %w", err)
}
if rewritten != nil {
body = rewritten
}
contentType := prov.ContentType(path)
if ct := resp.Header.Get("Content-Type"); ct != "" && contentType == "application/octet-stream" {
contentType = ct
}
if class == ClassMutable {
s3Key := storage.IndexKey(remote.Name, path)
if err := e.store.Upload(ctx, s3Key, bytesReader(body), int64(len(body)), contentType); err != nil {
return nil, fmt.Errorf("upload index: %w", err)
}
etag := resp.Header.Get("ETag")
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
if etag != "" {
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
}
} else {
hash := sha256Hash(body)
s3Key := storage.BlobKey(hash)
exists, _ := e.store.Exists(ctx, s3Key)
if !exists {
if err := e.store.Upload(ctx, s3Key, bytesReader(body), int64(len(body)), contentType); err != nil {
return nil, fmt.Errorf("upload blob: %w", err)
}
}
contentHash := fmt.Sprintf("sha256:%s", hash)
if err := e.db.UpsertBlob(ctx, contentHash, s3Key, int64(len(body)), contentType); err != nil {
slog.Warn("upsert blob failed", "error", err)
}
if err := e.db.UpsertArtifact(ctx, remote.Name, path, contentHash, resp.Header.Get("ETag")); err != nil {
slog.Warn("upsert artifact failed", "error", err)
}
_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
if etag := resp.Header.Get("ETag"); etag != "" {
_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
}
}
return &FetchResult{
Reader: io.NopCloser(bytesReader(body)),
ContentType: contentType,
Size: int64(len(body)),
Source: "remote",
}, nil
}
func (e *Engine) serveFromStore(ctx context.Context, remote models.Remote, path string) (*FetchResult, error) {
artifact, err := e.db.GetArtifact(ctx, remote.Name, path)
if err == nil && artifact != nil {
reader, info, err := e.store.Download(ctx, artifact.ContentHash[len("sha256:"):])
if err == nil {
_ = e.db.TouchArtifactAccess(ctx, remote.Name, path)
return &FetchResult{
Reader: reader,
ContentType: info.ContentType,
Size: info.Size,
}, nil
}
s3Key := storage.BlobKey(artifact.ContentHash[len("sha256:"):])
reader, info, err = e.store.Download(ctx, s3Key)
if err == nil {
_ = e.db.TouchArtifactAccess(ctx, remote.Name, path)
return &FetchResult{
Reader: reader,
ContentType: info.ContentType,
Size: info.Size,
}, nil
}
}
s3Key := storage.IndexKey(remote.Name, path)
reader, info, err := e.store.Download(ctx, s3Key)
if err != nil {
return nil, fmt.Errorf("not in store: %w", err)
}
return &FetchResult{
Reader: reader,
ContentType: info.ContentType,
Size: info.Size,
}, nil
}
func (e *Engine) checkUpstream(ctx context.Context, remote models.Remote, path, etag string, prov provider.Provider) (bool, error) {
url := prov.UpstreamURL(remote, path)
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return false, err
}
req.Header.Set("If-None-Match", etag)
authHeaders, err := prov.AuthHeaders(ctx, remote)
if err != nil {
return false, err
}
for k, vv := range authHeaders {
for _, v := range vv {
req.Header.Add(k, v)
}
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return false, &UpstreamError{Err: err}
}
resp.Body.Close()
return resp.StatusCode == http.StatusNotModified, nil
}
func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration {
switch class {
case ClassImmutable:
if remote.ImmutableTTL == 0 {
return 0
}
return time.Duration(remote.ImmutableTTL) * time.Second
default:
return time.Duration(remote.MutableTTL) * time.Second
}
}
func (e *Engine) logAccess(remoteName, path string, cacheHit bool, size int64, upstreamMS int) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = e.db.InsertAccessLog(ctx, remoteName, path, cacheHit, size, upstreamMS, "")
}
func sha256Hash(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
func bytesReader(data []byte) io.Reader {
return io.NewSectionReader(readerAt(data), 0, int64(len(data)))
}
type readerAt []byte
func (r readerAt) ReadAt(p []byte, off int64) (n int, err error) {
if off >= int64(len(r)) {
return 0, io.EOF
}
n = copy(p, r[off:])
if off+int64(n) >= int64(len(r)) {
err = io.EOF
}
return
}
type ProxyError struct {
Status int
Message string
}
func (e *ProxyError) Error() string { return e.Message }
type UpstreamError struct {
Err error
}
func (e *UpstreamError) Error() string { return fmt.Sprintf("upstream error: %v", e.Err) }
func (e *UpstreamError) Unwrap() error { return e.Err }
func isNetworkError(err error) bool {
if _, ok := err.(*UpstreamError); ok {
return true
}
return false
}
+45
View File
@@ -0,0 +1,45 @@
package server
import (
"log/slog"
"net/http"
"time"
"github.com/go-chi/chi/v5/middleware"
)
func cors(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
if r.Method == http.MethodOptions {
w.WriteHeader(http.StatusNoContent)
return
}
next.ServeHTTP(w, r)
})
}
func NewStructuredLogger() func(next http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
defer func() {
slog.Info("request",
"method", r.Method,
"path", r.URL.Path,
"status", ww.Status(),
"bytes", ww.BytesWritten(),
"duration_ms", time.Since(start).Milliseconds(),
"remote", r.RemoteAddr,
"request_id", middleware.GetReqID(r.Context()),
)
}()
next.ServeHTTP(ww, r)
})
}
}
+181
View File
@@ -0,0 +1,181 @@
package server
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/http"
"time"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/internal/config"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/gc"
_ "git.unkin.net/unkin/artifactapi/internal/provider/alpine"
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
_ "git.unkin.net/unkin/artifactapi/internal/provider/generic"
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
_ "git.unkin.net/unkin/artifactapi/internal/provider/rpm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/terraform"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
type Server struct {
cfg *config.Config
router chi.Router
db *database.DB
cache *cache.Redis
store *storage.S3
engine *proxy.Engine
virtEngine *virtual.Engine
gc *gc.Collector
}
func New(cfg *config.Config) (*Server, error) {
db, err := database.New(cfg.DatabaseDSN())
if err != nil {
return nil, fmt.Errorf("database: %w", err)
}
redis, err := cache.NewRedis(cfg.RedisURL)
if err != nil {
return nil, fmt.Errorf("redis: %w", err)
}
s3, err := storage.NewS3(cfg.S3Endpoint, cfg.S3AccessKey, cfg.S3SecretKey, cfg.S3Bucket, cfg.S3Secure, cfg.S3Region)
if err != nil {
return nil, fmt.Errorf("s3: %w", err)
}
engine := proxy.NewEngine(db, redis, s3)
virtEngine := virtual.NewEngine(db, engine)
collector := gc.New(db, s3, 1*time.Hour)
s := &Server{
cfg: cfg,
db: db,
cache: redis,
store: s3,
engine: engine,
virtEngine: virtEngine,
gc: collector,
}
s.router = s.routes()
return s, nil
}
func (s *Server) routes() chi.Router {
r := chi.NewRouter()
r.Use(middleware.RequestID)
r.Use(middleware.RealIP)
r.Use(NewStructuredLogger())
r.Use(middleware.Recoverer)
r.Use(cors)
r.Get("/health", s.handleHealth)
r.Get("/", s.handleRoot)
proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db)
r.Mount("/api/v1", proxyHandler.Routes())
remotesHandler := v2.NewRemotesHandler(s.db)
virtualsHandler := v2.NewVirtualsHandler(s.db)
healthHandler := v2.NewHealthHandler(s.db, s.cache, s.store)
statsHandler := v2.NewStatsHandler(s.db)
eventsHandler := v2.NewEventsHandler()
probeHandler := v2.NewProbeHandler(s.engine, s.db)
r.Route("/api/v2", func(r chi.Router) {
r.Mount("/remotes", remotesHandler.Routes())
r.Mount("/virtuals", virtualsHandler.Routes())
r.Mount("/health", healthHandler.Routes())
r.Mount("/stats", statsHandler.Routes())
r.Mount("/events", eventsHandler.Routes())
r.Mount("/probe", probeHandler.Routes())
r.Route("/remotes/{name}/objects", func(r chi.Router) {
objHandler := v2.NewObjectsHandler(s.db)
r.Get("/", objHandler.Routes().ServeHTTP)
r.Delete("/*", objHandler.Routes().ServeHTTP)
})
})
return r
}
func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, `{"status":"ok"}`)
}
func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, `{"name":"artifactapi","version":"3.0.0-dev"}`)
}
func (s *Server) newHTTPServer() *http.Server {
return &http.Server{
Addr: s.cfg.ListenAddr,
Handler: s.router,
ReadTimeout: 30 * time.Second,
WriteTimeout: 300 * time.Second,
IdleTimeout: 120 * time.Second,
}
}
func (s *Server) Run(ctx context.Context) error {
go s.gc.Run(ctx)
httpServer := s.newHTTPServer()
go func() {
<-ctx.Done()
slog.Info("shutting down server")
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = httpServer.Shutdown(shutdownCtx)
}()
slog.Info("starting server", "addr", s.cfg.ListenAddr)
if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
func (s *Server) RunOnListener(ctx context.Context, ln net.Listener) error {
go s.gc.Run(ctx)
httpServer := s.newHTTPServer()
go func() {
<-ctx.Done()
shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_ = httpServer.Shutdown(shutdownCtx)
}()
slog.Info("starting server", "addr", ln.Addr().String())
if err := httpServer.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
return nil
}
+72
View File
@@ -0,0 +1,72 @@
package storage
import (
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
)
type CAS struct {
s3 *S3
}
func NewCAS(s3 *S3) *CAS {
return &CAS{s3: s3}
}
type CASResult struct {
ContentHash string
S3Key string
SizeBytes int64
AlreadyExists bool
}
func (c *CAS) Store(ctx context.Context, reader io.Reader, contentType string) (*CASResult, error) {
tmp, err := os.CreateTemp("", "artifact-*")
if err != nil {
return nil, fmt.Errorf("create temp file: %w", err)
}
defer os.Remove(tmp.Name())
defer tmp.Close()
hasher := sha256.New()
size, err := io.Copy(io.MultiWriter(tmp, hasher), reader)
if err != nil {
return nil, fmt.Errorf("write temp file: %w", err)
}
hash := hex.EncodeToString(hasher.Sum(nil))
s3Key := BlobKey(hash)
exists, err := c.s3.Exists(ctx, s3Key)
if err != nil {
return nil, fmt.Errorf("check blob exists: %w", err)
}
if !exists {
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
return nil, fmt.Errorf("seek temp file: %w", err)
}
if err := c.s3.Upload(ctx, s3Key, tmp, size, contentType); err != nil {
return nil, fmt.Errorf("upload blob: %w", err)
}
}
return &CASResult{
ContentHash: fmt.Sprintf("sha256:%s", hash),
S3Key: s3Key,
SizeBytes: size,
AlreadyExists: exists,
}, nil
}
func BlobKey(hash string) string {
return fmt.Sprintf("blobs/sha256/%s", hash)
}
func IndexKey(remote, path string) string {
return fmt.Sprintf("indexes/%s/%s", remote, path)
}
+99
View File
@@ -0,0 +1,99 @@
package storage
import (
"context"
"fmt"
"io"
"log/slog"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type S3 struct {
client *minio.Client
bucket string
}
func NewS3(endpoint, accessKey, secretKey, bucket string, secure bool, region string) (*S3, error) {
opts := &minio.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: secure,
}
if region != "" {
opts.Region = region
}
client, err := minio.New(endpoint, opts)
if err != nil {
return nil, fmt.Errorf("create s3 client: %w", err)
}
s := &S3{client: client, bucket: bucket}
if err := s.ensureBucket(context.Background()); err != nil {
return nil, err
}
return s, nil
}
func (s *S3) ensureBucket(ctx context.Context) error {
exists, err := s.client.BucketExists(ctx, s.bucket)
if err != nil {
return fmt.Errorf("check bucket: %w", err)
}
if !exists {
if err := s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{}); err != nil {
return fmt.Errorf("create bucket: %w", err)
}
slog.Info("created bucket", "bucket", s.bucket)
}
return nil
}
func (s *S3) Upload(ctx context.Context, key string, reader io.Reader, size int64, contentType string) error {
_, err := s.client.PutObject(ctx, s.bucket, key, reader, size, minio.PutObjectOptions{
ContentType: contentType,
})
return err
}
func (s *S3) Download(ctx context.Context, key string) (io.ReadCloser, *minio.ObjectInfo, error) {
obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, nil, err
}
info, err := obj.Stat()
if err != nil {
obj.Close()
return nil, nil, err
}
return obj, &info, nil
}
func (s *S3) Exists(ctx context.Context, key string) (bool, error) {
_, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
resp := minio.ToErrorResponse(err)
if resp.Code == "NoSuchKey" {
return false, nil
}
return false, err
}
return true, nil
}
func (s *S3) Delete(ctx context.Context, key string) error {
return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
}
func (s *S3) Stat(ctx context.Context, key string) (*minio.ObjectInfo, error) {
info, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
return nil, err
}
return &info, nil
}
+305
View File
@@ -0,0 +1,305 @@
package tui
import (
"context"
"fmt"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"git.unkin.net/unkin/artifactapi/pkg/client"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type view int
const (
viewDashboard view = iota
viewRemotes
viewRemoteDetail
viewObjects
viewVirtuals
)
type model struct {
client *client.Client
view view
width int
height int
err error
loading bool
stats *models.OverviewStats
remotes []models.Remote
virtuals []models.Virtual
objects []models.Artifact
selectedRemote string
cursor int
page int
}
func New(endpoint string) *model {
return &model{
client: client.New(endpoint),
view: viewDashboard,
loading: true,
page: 1,
}
}
func (m *model) Run() error {
p := tea.NewProgram(m, tea.WithAltScreen())
_, err := p.Run()
return err
}
func (m *model) Init() tea.Cmd {
return m.loadDashboard()
}
type dashboardLoaded struct {
stats *models.OverviewStats
remotes []models.Remote
virtuals []models.Virtual
}
type remotesLoaded struct{ remotes []models.Remote }
type virtualsLoaded struct{ virtuals []models.Virtual }
type objectsLoaded struct{ objects []models.Artifact }
type errMsg struct{ err error }
func (m *model) loadDashboard() tea.Cmd {
return func() tea.Msg {
ctx := context.Background()
stats, err := m.client.Stats(ctx)
if err != nil {
return errMsg{err}
}
remotes, _ := m.client.ListRemotes(ctx)
virtuals, _ := m.client.ListVirtuals(ctx)
return dashboardLoaded{stats: stats, remotes: remotes, virtuals: virtuals}
}
}
func (m *model) loadRemotes() tea.Cmd {
return func() tea.Msg {
remotes, err := m.client.ListRemotes(context.Background())
if err != nil {
return errMsg{err}
}
return remotesLoaded{remotes}
}
}
func (m *model) loadVirtuals() tea.Cmd {
return func() tea.Msg {
virtuals, err := m.client.ListVirtuals(context.Background())
if err != nil {
return errMsg{err}
}
return virtualsLoaded{virtuals}
}
}
func (m *model) loadObjects() tea.Cmd {
return func() tea.Msg {
objects, err := m.client.ListObjects(context.Background(), m.selectedRemote, m.page, 30)
if err != nil {
return errMsg{err}
}
return objectsLoaded{objects}
}
}
func (m *model) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width = msg.Width
m.height = msg.Height
return m, nil
case tea.KeyMsg:
return m.handleKey(msg)
case dashboardLoaded:
m.loading = false
m.stats = msg.stats
m.remotes = msg.remotes
m.virtuals = msg.virtuals
return m, nil
case remotesLoaded:
m.loading = false
m.remotes = msg.remotes
m.cursor = 0
return m, nil
case virtualsLoaded:
m.loading = false
m.virtuals = msg.virtuals
m.cursor = 0
return m, nil
case objectsLoaded:
m.loading = false
m.objects = msg.objects
m.cursor = 0
return m, nil
case errMsg:
m.loading = false
m.err = msg.err
return m, nil
}
return m, nil
}
func (m *model) handleKey(msg tea.KeyMsg) (tea.Model, tea.Cmd) {
switch msg.String() {
case "q", "ctrl+c":
if m.view == viewDashboard {
return m, tea.Quit
}
m.view = viewDashboard
m.cursor = 0
m.loading = true
return m, m.loadDashboard()
case "esc":
switch m.view {
case viewRemoteDetail, viewObjects:
m.view = viewRemotes
m.cursor = 0
m.loading = true
return m, m.loadRemotes()
case viewRemotes, viewVirtuals:
m.view = viewDashboard
m.cursor = 0
m.loading = true
return m, m.loadDashboard()
default:
return m, tea.Quit
}
case "1":
m.view = viewDashboard
m.loading = true
return m, m.loadDashboard()
case "2":
m.view = viewRemotes
m.loading = true
return m, m.loadRemotes()
case "3":
m.view = viewVirtuals
m.loading = true
return m, m.loadVirtuals()
case "j", "down":
m.cursor++
m.clampCursor()
return m, nil
case "k", "up":
if m.cursor > 0 {
m.cursor--
}
return m, nil
case "enter":
return m.handleEnter()
case "r":
m.loading = true
switch m.view {
case viewDashboard:
return m, m.loadDashboard()
case viewRemotes:
return m, m.loadRemotes()
case viewVirtuals:
return m, m.loadVirtuals()
case viewObjects:
return m, m.loadObjects()
}
}
return m, nil
}
func (m *model) handleEnter() (tea.Model, tea.Cmd) {
switch m.view {
case viewRemotes:
if m.cursor < len(m.remotes) {
m.selectedRemote = m.remotes[m.cursor].Name
m.view = viewRemoteDetail
return m, nil
}
case viewRemoteDetail:
m.view = viewObjects
m.page = 1
m.loading = true
return m, m.loadObjects()
}
return m, nil
}
func (m *model) clampCursor() {
max := 0
switch m.view {
case viewRemotes:
max = len(m.remotes) - 1
case viewVirtuals:
max = len(m.virtuals) - 1
case viewObjects:
max = len(m.objects) - 1
}
if m.cursor > max {
m.cursor = max
}
if m.cursor < 0 {
m.cursor = 0
}
}
func (m *model) View() string {
if m.loading {
return m.chrome("Loading...")
}
if m.err != nil {
return m.chrome(errStyle.Render(fmt.Sprintf("Error: %v", m.err)))
}
var body string
switch m.view {
case viewDashboard:
body = m.viewDashboard()
case viewRemotes:
body = m.viewRemotesList()
case viewRemoteDetail:
body = m.viewRemoteDetail()
case viewObjects:
body = m.viewObjectsList()
case viewVirtuals:
body = m.viewVirtualsList()
}
return m.chrome(body)
}
func (m *model) chrome(body string) string {
nav := navStyle.Render(
"[1] Dashboard [2] Remotes [3] Virtuals │ [r] Refresh [q] Quit",
)
return lipgloss.JoinVertical(lipgloss.Left, body, "", nav)
}
var (
titleStyle = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.Color("12"))
navStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
errStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9"))
mutedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8"))
selStyle = lipgloss.NewStyle().Background(lipgloss.Color("4")).Foreground(lipgloss.Color("15"))
)
+140
View File
@@ -0,0 +1,140 @@
package tui
import (
"fmt"
"strings"
"git.unkin.net/unkin/artifactapi/internal/tui/views"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func (m *model) viewDashboard() string {
return titleStyle.Render("ArtifactAPI Dashboard") + "\n\n" +
views.RenderDashboard(m.stats, len(m.remotes), len(m.virtuals)) +
"\n\n" + mutedStyle.Render("Press [2] for remotes, [3] for virtuals")
}
func (m *model) viewRemotesList() string {
var sb strings.Builder
sb.WriteString(titleStyle.Render("Remotes") + "\n\n")
if len(m.remotes) == 0 {
sb.WriteString(mutedStyle.Render("No remotes configured"))
return sb.String()
}
for i, r := range m.remotes {
line := fmt.Sprintf(" %-25s %-12s %s", r.Name, r.PackageType, r.Description)
if i == m.cursor {
sb.WriteString(selStyle.Render(line))
} else {
sb.WriteString(line)
}
sb.WriteString("\n")
}
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · enter detail · esc back"))
return sb.String()
}
func (m *model) viewRemoteDetail() string {
var r *remoteView
for i := range m.remotes {
if m.remotes[i].Name == m.selectedRemote {
r = &remoteView{m.remotes[i]}
break
}
}
if r == nil {
return mutedStyle.Render("Remote not found")
}
var sb strings.Builder
sb.WriteString(titleStyle.Render(r.Name) + "\n\n")
sb.WriteString(fmt.Sprintf(" Type: %s\n", r.PackageType))
sb.WriteString(fmt.Sprintf(" Base URL: %s\n", r.BaseURL))
sb.WriteString(fmt.Sprintf(" Description: %s\n", r.Description))
sb.WriteString(fmt.Sprintf(" Immutable TTL: %s\n", ttlStr(r.ImmutableTTL)))
sb.WriteString(fmt.Sprintf(" Mutable TTL: %ds\n", r.MutableTTL))
sb.WriteString(fmt.Sprintf(" Revalidation: %v\n", r.CheckMutable))
sb.WriteString(fmt.Sprintf(" Stale on Error: %v\n", r.StaleOnError))
if len(r.Patterns) > 0 {
sb.WriteString(fmt.Sprintf(" Patterns: %s\n", strings.Join(r.Patterns, ", ")))
}
if len(r.Blocklist) > 0 {
sb.WriteString(fmt.Sprintf(" Blocklist: %s\n", strings.Join(r.Blocklist, ", ")))
}
if r.ManagedBy != "" {
sb.WriteString(fmt.Sprintf(" Managed by: %s\n", r.ManagedBy))
}
sb.WriteString("\n" + mutedStyle.Render("enter → browse objects · esc back"))
return sb.String()
}
func (m *model) viewObjectsList() string {
var sb strings.Builder
sb.WriteString(titleStyle.Render(fmt.Sprintf("Objects: %s (page %d)", m.selectedRemote, m.page)) + "\n\n")
if len(m.objects) == 0 {
sb.WriteString(mutedStyle.Render("No cached objects"))
return sb.String()
}
for i, a := range m.objects {
size := views.FormatBytes(a.SizeBytes)
line := fmt.Sprintf(" %-50s %10s %5d hits", truncate(a.Path, 50), size, a.AccessCount)
if i == m.cursor {
sb.WriteString(selStyle.Render(line))
} else {
sb.WriteString(line)
}
sb.WriteString("\n")
}
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · esc back"))
return sb.String()
}
func (m *model) viewVirtualsList() string {
var sb strings.Builder
sb.WriteString(titleStyle.Render("Virtual Repositories") + "\n\n")
if len(m.virtuals) == 0 {
sb.WriteString(mutedStyle.Render("No virtual repositories configured"))
return sb.String()
}
for i, v := range m.virtuals {
line := fmt.Sprintf(" %-25s %-12s %d members %s",
v.Name, v.PackageType, len(v.Members), v.Description)
if i == m.cursor {
sb.WriteString(selStyle.Render(line))
} else {
sb.WriteString(line)
}
sb.WriteString("\n")
}
sb.WriteString("\n" + mutedStyle.Render("j/k navigate · esc back"))
return sb.String()
}
type remoteView struct {
models.Remote
}
func ttlStr(ttl int) string {
if ttl == 0 {
return "forever"
}
return fmt.Sprintf("%ds", ttl)
}
func truncate(s string, max int) string {
if len(s) <= max {
return s
}
return s[:max-3] + "..."
}
+45
View File
@@ -0,0 +1,45 @@
package views
import (
"fmt"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func FormatBytes(bytes int64) string {
if bytes == 0 {
return "0 B"
}
units := []string{"B", "KB", "MB", "GB", "TB"}
i := 0
b := float64(bytes)
for b >= 1024 && i < len(units)-1 {
b /= 1024
i++
}
if i == 0 {
return fmt.Sprintf("%.0f %s", b, units[i])
}
return fmt.Sprintf("%.1f %s", b, units[i])
}
func RenderDashboard(stats *models.OverviewStats, remoteCount, virtualCount int) string {
if stats == nil {
return "No stats available"
}
return fmt.Sprintf(
"╭─ Dashboard ──────────────────────────────╮\n"+
"│ Remotes: %-24d│\n"+
"│ Cached Objects: %-24d│\n"+
"│ Storage Used: %-24s│\n"+
"│ Dedup Savings: %-20d blobs │\n"+
"│ Virtuals: %-24d│\n"+
"╰──────────────────────────────────────────╯",
stats.TotalRemotes,
stats.TotalObjects,
FormatBytes(stats.TotalBytes),
stats.TotalBlobsDeduped,
virtualCount,
)
}
+111
View File
@@ -0,0 +1,111 @@
package virtual
import (
"context"
"fmt"
"io"
"log/slog"
"sync"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type Engine struct {
db *database.DB
proxyEngine *proxy.Engine
}
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
return &Engine{db: db, proxyEngine: proxyEngine}
}
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
merger, err := GetMerger(virt.PackageType)
if err != nil {
return nil, "", fmt.Errorf("unsupported virtual type %q: %w", virt.PackageType, err)
}
members, err := e.fetchMemberIndexes(ctx, virt, path)
if err != nil {
return nil, "", err
}
if len(members) == 0 {
return nil, "", fmt.Errorf("no members reachable for virtual %q", virt.Name)
}
merged, err := merger.MergeIndexes(members, proxyBaseURL)
if err != nil {
return nil, "", fmt.Errorf("merge indexes: %w", err)
}
contentType := "application/octet-stream"
switch virt.PackageType {
case models.PackageHelm:
contentType = "text/yaml"
case models.PackagePyPI:
contentType = "text/html"
}
return merged, contentType, nil
}
func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, path string) ([]MemberIndex, error) {
type result struct {
index MemberIndex
err error
}
results := make([]result, len(virt.Members))
var wg sync.WaitGroup
for i, memberName := range virt.Members {
wg.Add(1)
go func(idx int, name string) {
defer wg.Done()
remote, err := e.db.GetRemote(ctx, name)
if err != nil {
results[idx] = result{err: fmt.Errorf("remote %q: %w", name, err)}
return
}
prov, err := provider.Get(remote.PackageType)
if err != nil {
results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)}
return
}
fetchResult, err := e.proxyEngine.Fetch(ctx, *remote, path, prov)
if err != nil {
results[idx] = result{err: fmt.Errorf("fetch %q/%s: %w", name, path, err)}
return
}
defer fetchResult.Reader.Close()
body, err := io.ReadAll(fetchResult.Reader)
if err != nil {
results[idx] = result{err: fmt.Errorf("read %q: %w", name, err)}
return
}
results[idx] = result{index: MemberIndex{RemoteName: name, Body: body}}
}(i, memberName)
}
wg.Wait()
var members []MemberIndex
for _, r := range results {
if r.err != nil {
slog.Warn("virtual member fetch failed", "error", r.err)
continue
}
members = append(members, r.index)
}
return members, nil
}
+92
View File
@@ -0,0 +1,92 @@
package virtual
import (
"fmt"
"strings"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
RegisterMerger(models.PackageHelm, &HelmMerger{})
}
type HelmMerger struct{}
type helmIndex struct {
APIVersion string `yaml:"apiVersion"`
Entries map[string][]helmChartVersion `yaml:"entries"`
Generated string `yaml:"generated,omitempty"`
}
type helmChartVersion struct {
Name string `yaml:"name"`
Version string `yaml:"version"`
URLs []string `yaml:"urls"`
rest map[string]any
}
func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
merged := &helmIndex{
APIVersion: "v1",
Entries: make(map[string][]helmChartVersion),
}
seen := map[string]map[string]bool{}
for _, member := range members {
var idx helmIndex
if err := yaml.Unmarshal(member.Body, &idx); err != nil {
continue
}
for chart, versions := range idx.Entries {
if seen[chart] == nil {
seen[chart] = map[string]bool{}
}
for _, ver := range versions {
key := chart + ":" + ver.Version
if seen[chart][ver.Version] {
continue
}
seen[chart][ver.Version] = true
if proxyBaseURL != "" {
for i, u := range ver.URLs {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
member.RemoteName,
extractPath(u))
} else {
ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
member.RemoteName,
u)
}
}
}
merged.Entries[chart] = append(merged.Entries[chart], ver)
_ = key
}
}
}
return yaml.Marshal(merged)
}
func extractPath(rawURL string) string {
idx := strings.Index(rawURL, "://")
if idx == -1 {
return rawURL
}
rest := rawURL[idx+3:]
slashIdx := strings.Index(rest, "/")
if slashIdx == -1 {
return ""
}
return rest[slashIdx+1:]
}
+124
View File
@@ -0,0 +1,124 @@
package virtual_test
import (
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
func TestHelmMerger_BasicMerge(t *testing.T) {
m := &virtual.HelmMerger{}
member1 := virtual.MemberIndex{
RemoteName: "repo-a",
Body: []byte(`apiVersion: v1
entries:
nginx:
- name: nginx
version: "1.0.0"
urls:
- https://charts-a.example.com/nginx-1.0.0.tgz
`),
}
member2 := virtual.MemberIndex{
RemoteName: "repo-b",
Body: []byte(`apiVersion: v1
entries:
redis:
- name: redis
version: "2.0.0"
urls:
- https://charts-b.example.com/redis-2.0.0.tgz
`),
}
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
body := string(result)
if !strings.Contains(body, "nginx") {
t.Error("expected nginx in merged index")
}
if !strings.Contains(body, "redis") {
t.Error("expected redis in merged index")
}
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-a") {
t.Error("expected proxy URL for repo-a")
}
if !strings.Contains(body, "proxy.example.com/api/v1/remote/repo-b") {
t.Error("expected proxy URL for repo-b")
}
}
func TestHelmMerger_Dedup(t *testing.T) {
m := &virtual.HelmMerger{}
idx := []byte(`apiVersion: v1
entries:
nginx:
- name: nginx
version: "1.0.0"
urls:
- nginx-1.0.0.tgz
`)
members := []virtual.MemberIndex{
{RemoteName: "repo-a", Body: idx},
{RemoteName: "repo-b", Body: idx},
}
result, err := m.MergeIndexes(members, "")
if err != nil {
t.Fatal(err)
}
count := strings.Count(string(result), "name: nginx")
if count != 1 {
t.Errorf("expected 1 entry for nginx, got %d\n%s", count, result)
}
}
func TestHelmMerger_PriorityOrder(t *testing.T) {
m := &virtual.HelmMerger{}
member1 := virtual.MemberIndex{
RemoteName: "priority-repo",
Body: []byte(`apiVersion: v1
entries:
chart:
- name: chart
version: "1.0.0"
urls:
- chart-from-priority.tgz
`),
}
member2 := virtual.MemberIndex{
RemoteName: "fallback-repo",
Body: []byte(`apiVersion: v1
entries:
chart:
- name: chart
version: "1.0.0"
urls:
- chart-from-fallback.tgz
`),
}
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy")
if err != nil {
t.Fatal(err)
}
body := string(result)
if !strings.Contains(body, "priority-repo") {
t.Error("expected priority repo URL to win")
}
if strings.Contains(body, "fallback-repo") {
t.Error("expected fallback repo to be excluded for duplicate")
}
}
+30
View File
@@ -0,0 +1,30 @@
package virtual
import (
"fmt"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type MemberIndex struct {
RemoteName string
Body []byte
}
type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
}
var mergers = map[models.PackageType]IndexMerger{}
func RegisterMerger(pt models.PackageType, m IndexMerger) {
mergers[pt] = m
}
func GetMerger(pt models.PackageType) (IndexMerger, error) {
m, ok := mergers[pt]
if !ok {
return nil, fmt.Errorf("no merger registered for package type %q", pt)
}
return m, nil
}
+89
View File
@@ -0,0 +1,89 @@
package virtual
import (
"fmt"
"sort"
"strings"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
RegisterMerger(models.PackagePyPI, &PyPIMerger{})
}
type PyPIMerger struct{}
func (m *PyPIMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
links := map[string]string{}
for _, member := range members {
body := string(member.Body)
for _, line := range strings.Split(body, "\n") {
line = strings.TrimSpace(line)
if !strings.HasPrefix(line, "<a ") {
continue
}
href := extractHref(line)
text := extractLinkText(line)
if text == "" {
continue
}
if _, exists := links[text]; exists {
continue
}
if proxyBaseURL != "" && href != "" {
href = fmt.Sprintf("%s/api/v1/remote/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
member.RemoteName,
strings.TrimLeft(href, "/"))
}
links[text] = href
}
}
keys := make([]string, 0, len(links))
for k := range links {
keys = append(keys, k)
}
sort.Strings(keys)
var sb strings.Builder
sb.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, name := range keys {
sb.WriteString(fmt.Sprintf(" <a href=\"%s\">%s</a>\n", links[name], name))
}
sb.WriteString("</body></html>\n")
return []byte(sb.String()), nil
}
func extractHref(tag string) string {
idx := strings.Index(tag, `href="`)
if idx == -1 {
return ""
}
rest := tag[idx+6:]
end := strings.Index(rest, `"`)
if end == -1 {
return rest
}
return rest[:end]
}
func extractLinkText(tag string) string {
start := strings.Index(tag, ">")
if start == -1 {
return ""
}
rest := tag[start+1:]
end := strings.Index(rest, "</a>")
if end == -1 {
return strings.TrimSpace(rest)
}
return strings.TrimSpace(rest[:end])
}
+98
View File
@@ -0,0 +1,98 @@
package virtual_test
import (
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/virtual"
)
func TestPyPIMerger_BasicMerge(t *testing.T) {
m := &virtual.PyPIMerger{}
member1 := virtual.MemberIndex{
RemoteName: "pypi-a",
Body: []byte(`<!DOCTYPE html>
<html><body>
<a href="/simple/requests/">requests</a>
<a href="/simple/flask/">flask</a>
</body></html>`),
}
member2 := virtual.MemberIndex{
RemoteName: "pypi-b",
Body: []byte(`<!DOCTYPE html>
<html><body>
<a href="/simple/django/">django</a>
</body></html>`),
}
result, err := m.MergeIndexes([]virtual.MemberIndex{member1, member2}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
body := string(result)
if !strings.Contains(body, "requests") {
t.Error("expected requests")
}
if !strings.Contains(body, "flask") {
t.Error("expected flask")
}
if !strings.Contains(body, "django") {
t.Error("expected django")
}
if !strings.Contains(body, "proxy.example.com/api/v1/remote/pypi-a") {
t.Error("expected proxy URL for pypi-a")
}
}
func TestPyPIMerger_Dedup(t *testing.T) {
m := &virtual.PyPIMerger{}
idx := []byte(`<html><body>
<a href="/simple/requests/">requests</a>
</body></html>`)
members := []virtual.MemberIndex{
{RemoteName: "a", Body: idx},
{RemoteName: "b", Body: idx},
}
result, err := m.MergeIndexes(members, "")
if err != nil {
t.Fatal(err)
}
count := strings.Count(string(result), "<a ")
if count != 1 {
t.Errorf("expected 1 <a> tag for deduplicated requests, got %d\n%s", count, result)
}
}
func TestPyPIMerger_Sorted(t *testing.T) {
m := &virtual.PyPIMerger{}
member := virtual.MemberIndex{
RemoteName: "pypi",
Body: []byte(`<html><body>
<a href="/z/">zebra</a>
<a href="/a/">alpha</a>
<a href="/m/">middle</a>
</body></html>`),
}
result, err := m.MergeIndexes([]virtual.MemberIndex{member}, "")
if err != nil {
t.Fatal(err)
}
body := string(result)
alphaIdx := strings.Index(body, "alpha")
middleIdx := strings.Index(body, "middle")
zebraIdx := strings.Index(body, "zebra")
if alphaIdx > middleIdx || middleIdx > zebraIdx {
t.Error("expected sorted output")
}
}