unkin/artifactapi
2
0
Fork 0
Code Issues 5 Pull Requests 2 Actions Packages Projects Releases 1 Wiki Activity

feat: virtual PyPI repos can merge local + remote members
ci/woodpecker/pr/pre-commit Pipeline was successful
Details
ci/woodpecker/pr/build Pipeline was successful
Details
ci/woodpecker/pr/test Pipeline was successful
Details

Browse Source 
The virtual engine now detects local members (repo_type=local) and
generates their package index in-memory instead of trying to fetch
from a non-existent upstream.

- MemberIndex gains RepoType field so mergers use correct URL prefix
  (/api/v1/local/ vs /api/v1/remote/)
- Virtual engine accepts a LocalIndexGenerator interface for producing
  local PyPI indexes
- LocalHandler implements GeneratePyPIPackageHTML for reuse by both
  the direct serving path and the virtual merger
- Includes local PyPI upload support (cherry-picked from benvin/local-pypi)

Tested e2e: local wheel upload + virtual merge + uv pip install from
both direct local and virtual URLs
This commit is contained in:
Ben Vincent
2026-06-23 22:13:12 +10:00
parent de96637122
commit 79bf0de110
5 changed files with 81 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/v2/local.go
+15 -9
View File
@@ -320,10 +320,19 @@ func (h *LocalHandler) ServePyPIPackageIndex(w http.ResponseWriter, r *http.Requ
return return
} }
body := h.generatePyPIPackageHTML(normalized, files) var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
normalized, filename, hash, filename)
}
b.WriteString("</body></html>\n")
w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
w.Write(body) io.WriteString(w, b.String())
} }
func (h *LocalHandler) GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error) { func (h *LocalHandler) GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error) {
@@ -333,20 +342,17 @@ func (h *LocalHandler) GeneratePyPIPackageHTML(ctx context.Context, repoName, pa
if err != nil { if err != nil {
return nil, err return nil, err
} }
return h.generatePyPIPackageHTML(normalized, files), nil
}
func (h *LocalHandler) generatePyPIPackageHTML(packageName string, files []database.LocalFile) []byte {
var b strings.Builder var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n") b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range files { for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, packageName+"/") filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:") hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n", fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
packageName, filename, hash, filename) normalized, filename, hash, filename)
} }
b.WriteString("</body></html>\n") b.WriteString("</body></html>\n")
return []byte(b.String()) return []byte(b.String()), nil
} }
func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) { func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) {
internal/server/server.go
+23 -22
View File
@@ -34,14 +34,15 @@ import (
) )
type Server struct { type Server struct {
cfg *config.Config cfg *config.Config
router chi.Router router chi.Router
db *database.DB db *database.DB
cache *cache.Redis cache *cache.Redis
store *storage.S3 store *storage.S3
engine *proxy.Engine engine *proxy.Engine
virtEngine *virtual.Engine virtEngine *virtual.Engine
gc *gc.Collector localHandler *v2.LocalHandler
gc *gc.Collector
} }
func New(cfg *config.Config) (*Server, error) { func New(cfg *config.Config) (*Server, error) {
@@ -61,17 +62,19 @@ func New(cfg *config.Config) (*Server, error) {
} }
engine := proxy.NewEngine(db, redis, s3) engine := proxy.NewEngine(db, redis, s3)
virtEngine := virtual.NewEngine(db, engine) localHandler := v2.NewLocalHandler(db, s3)
virtEngine := virtual.NewEngine(db, engine, localHandler)
collector := gc.New(db, s3, 1*time.Hour) collector := gc.New(db, s3, 1*time.Hour)
s := &Server{ s := &Server{
cfg: cfg, cfg: cfg,
db: db, db: db,
cache: redis, cache: redis,
store: s3, store: s3,
engine: engine, engine: engine,
virtEngine: virtEngine, virtEngine: virtEngine,
gc: collector, localHandler: localHandler,
gc: collector,
} }
s.router = s.routes() s.router = s.routes()
@@ -91,9 +94,7 @@ func (s *Server) routes() chi.Router {
r.Get("/health", s.handleHealth) r.Get("/health", s.handleHealth)
r.Get("/", s.handleRoot) r.Get("/", s.handleRoot)
localHandler := v2.NewLocalHandler(s.db, s.store) proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, localHandler)
r.Mount("/api/v1", proxyHandler.Routes()) r.Mount("/api/v1", proxyHandler.Routes())
remotesHandler := v2.NewRemotesHandler(s.db) remotesHandler := v2.NewRemotesHandler(s.db)
@@ -118,9 +119,9 @@ func (s *Server) routes() chi.Router {
}) })
r.Route("/remotes/{name}/files", func(r chi.Router) { r.Route("/remotes/{name}/files", func(r chi.Router) {
r.Put("/*", localHandler.Routes().ServeHTTP) r.Put("/*", s.localHandler.Routes().ServeHTTP)
r.Get("/*", localHandler.Routes().ServeHTTP) r.Get("/*", s.localHandler.Routes().ServeHTTP)
r.Delete("/*", localHandler.Routes().ServeHTTP) r.Delete("/*", s.localHandler.Routes().ServeHTTP)
}) })
}) })
internal/virtual/engine.go
+36 -3
View File
@@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"io" "io"
"log/slog" "log/slog"
"strings"
"sync" "sync"
"git.unkin.net/unkin/artifactapi/internal/database" "git.unkin.net/unkin/artifactapi/internal/database"
@@ -13,13 +14,18 @@ import (
"git.unkin.net/unkin/artifactapi/pkg/models" "git.unkin.net/unkin/artifactapi/pkg/models"
) )
type LocalIndexGenerator interface {
GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error)
}
type Engine struct { type Engine struct {
db *database.DB db *database.DB
proxyEngine *proxy.Engine proxyEngine *proxy.Engine
localGen LocalIndexGenerator
} }
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine { func NewEngine(db *database.DB, proxyEngine *proxy.Engine, localGen LocalIndexGenerator) *Engine {
return &Engine{db: db, proxyEngine: proxyEngine} return &Engine{db: db, proxyEngine: proxyEngine, localGen: localGen}
} }
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) { func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
@@ -73,6 +79,16 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
return return
} }
if remote.RepoType == models.RepoTypeLocal {
body, err := e.fetchLocalIndex(ctx, *remote, virt.PackageType, path)
if err != nil {
results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)}
return
}
results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, Body: body}}
return
}
prov, err := provider.Get(remote.PackageType) prov, err := provider.Get(remote.PackageType)
if err != nil { if err != nil {
results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)} results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)}
@@ -92,7 +108,7 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
return return
} }
results[idx] = result{index: MemberIndex{RemoteName: name, Body: body}} results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, Body: body}}
}(i, memberName) }(i, memberName)
} }
@@ -109,3 +125,20 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
return members, nil return members, nil
} }
func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, packageType models.PackageType, path string) ([]byte, error) {
switch packageType {
case models.PackagePyPI:
if e.localGen == nil {
return nil, fmt.Errorf("no local index generator configured")
}
parts := strings.SplitN(strings.TrimPrefix(path, "simple/"), "/", 2)
pkgName := strings.TrimSuffix(parts[0], "/")
if pkgName == "" {
return nil, fmt.Errorf("cannot determine package name from path %q", path)
}
return e.localGen.GeneratePyPIPackageHTML(ctx, remote.Name, pkgName)
default:
return nil, fmt.Errorf("local index generation not supported for %q", packageType)
}
}
internal/virtual/merger.go
+1
View File
@@ -8,6 +8,7 @@ import (
type MemberIndex struct { type MemberIndex struct {
RemoteName string RemoteName string
RepoType models.RepoType
Body []byte Body []byte
} }
internal/virtual/pypi_merger.go
+6 -1
View File
@@ -36,8 +36,13 @@ func (m *PyPIMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
} }
if proxyBaseURL != "" && href != "" { if proxyBaseURL != "" && href != "" {
href = fmt.Sprintf("%s/api/v1/remote/%s/%s", routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
}
href = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"), strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName, member.RemoteName,
strings.TrimLeft(href, "/")) strings.TrimLeft(href, "/"))
} }