feat: virtual PyPI repos can merge local + remote members

The virtual engine now detects local members (repo_type=local) and generates their package index in-memory instead of trying to fetch from a non-existent upstream. - MemberIndex gains RepoType field so mergers use correct URL prefix (/api/v1/local/ vs /api/v1/remote/) - Virtual engine accepts a LocalIndexGenerator interface for producing local PyPI indexes - LocalHandler implements GeneratePyPIPackageHTML for reuse by both the direct serving path and the virtual merger - Includes local PyPI upload support (cherry-picked from benvin/local-pypi) Tested e2e: local wheel upload + virtual merge + uv pip install from both direct local and virtual URLs
2026-06-23 22:13:12 +10:00
parent de96637122
commit 79bf0de110
5 changed files with 81 additions and 35 deletions
@@ -320,10 +320,19 @@ func (h *LocalHandler) ServePyPIPackageIndex(w http.ResponseWriter, r *http.Requ
 		return
 	}

-	body := h.generatePyPIPackageHTML(normalized, files)
+	var b strings.Builder
+	b.WriteString("<!DOCTYPE html>\n<html><body>\n")
+	for _, f := range files {
+		filename := strings.TrimPrefix(f.FilePath, normalized+"/")
+		hash := strings.TrimPrefix(f.ContentHash, "sha256:")
+		fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
+			normalized, filename, hash, filename)
+	}
+	b.WriteString("</body></html>\n")
+
 	w.Header().Set("Content-Type", "text/html")
 	w.WriteHeader(http.StatusOK)
-	w.Write(body)
+	io.WriteString(w, b.String())
 }

 func (h *LocalHandler) GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error) {
@@ -333,20 +342,17 @@ func (h *LocalHandler) GeneratePyPIPackageHTML(ctx context.Context, repoName, pa
 	if err != nil {
 		return nil, err
 	}
-	return h.generatePyPIPackageHTML(normalized, files), nil
-}

-func (h *LocalHandler) generatePyPIPackageHTML(packageName string, files []database.LocalFile) []byte {
 	var b strings.Builder
 	b.WriteString("<!DOCTYPE html>\n<html><body>\n")
 	for _, f := range files {
-		filename := strings.TrimPrefix(f.FilePath, packageName+"/")
+		filename := strings.TrimPrefix(f.FilePath, normalized+"/")
 		hash := strings.TrimPrefix(f.ContentHash, "sha256:")
-		fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
-			packageName, filename, hash, filename)
+		fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
+			normalized, filename, hash, filename)
 	}
 	b.WriteString("</body></html>\n")
-	return []byte(b.String())
+	return []byte(b.String()), nil
 }

 func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) {
@@ -34,14 +34,15 @@ import (
 )

 type Server struct {
-	cfg        *config.Config
-	router     chi.Router
-	db         *database.DB
-	cache      *cache.Redis
-	store      *storage.S3
-	engine     *proxy.Engine
-	virtEngine *virtual.Engine
-	gc         *gc.Collector
+	cfg          *config.Config
+	router       chi.Router
+	db           *database.DB
+	cache        *cache.Redis
+	store        *storage.S3
+	engine       *proxy.Engine
+	virtEngine   *virtual.Engine
+	localHandler *v2.LocalHandler
+	gc           *gc.Collector
 }

 func New(cfg *config.Config) (*Server, error) {
@@ -61,17 +62,19 @@ func New(cfg *config.Config) (*Server, error) {
 	}

 	engine := proxy.NewEngine(db, redis, s3)
-	virtEngine := virtual.NewEngine(db, engine)
+	localHandler := v2.NewLocalHandler(db, s3)
+	virtEngine := virtual.NewEngine(db, engine, localHandler)
 	collector := gc.New(db, s3, 1*time.Hour)

 	s := &Server{
-		cfg:        cfg,
-		db:         db,
-		cache:      redis,
-		store:      s3,
-		engine:     engine,
-		virtEngine: virtEngine,
-		gc:         collector,
+		cfg:          cfg,
+		db:           db,
+		cache:        redis,
+		store:        s3,
+		engine:       engine,
+		virtEngine:   virtEngine,
+		localHandler: localHandler,
+		gc:           collector,
 	}

 	s.router = s.routes()
@@ -91,9 +94,7 @@ func (s *Server) routes() chi.Router {
 	r.Get("/health", s.handleHealth)
 	r.Get("/", s.handleRoot)

-	localHandler := v2.NewLocalHandler(s.db, s.store)
-
-	proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, localHandler)
+	proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
 	r.Mount("/api/v1", proxyHandler.Routes())

 	remotesHandler := v2.NewRemotesHandler(s.db)
@@ -118,9 +119,9 @@ func (s *Server) routes() chi.Router {
 		})

 		r.Route("/remotes/{name}/files", func(r chi.Router) {
-			r.Put("/*", localHandler.Routes().ServeHTTP)
-			r.Get("/*", localHandler.Routes().ServeHTTP)
-			r.Delete("/*", localHandler.Routes().ServeHTTP)
+			r.Put("/*", s.localHandler.Routes().ServeHTTP)
+			r.Get("/*", s.localHandler.Routes().ServeHTTP)
+			r.Delete("/*", s.localHandler.Routes().ServeHTTP)
 		})
 	})

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"log/slog"
+	"strings"
 	"sync"

 	"git.unkin.net/unkin/artifactapi/internal/database"
@@ -13,13 +14,18 @@ import (
 	"git.unkin.net/unkin/artifactapi/pkg/models"
 )

+type LocalIndexGenerator interface {
+	GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error)
+}
+
 type Engine struct {
 	db          *database.DB
 	proxyEngine *proxy.Engine
+	localGen    LocalIndexGenerator
 }

-func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
-	return &Engine{db: db, proxyEngine: proxyEngine}
+func NewEngine(db *database.DB, proxyEngine *proxy.Engine, localGen LocalIndexGenerator) *Engine {
+	return &Engine{db: db, proxyEngine: proxyEngine, localGen: localGen}
 }

 func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
@@ -73,6 +79,16 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
 				return
 			}

+			if remote.RepoType == models.RepoTypeLocal {
+				body, err := e.fetchLocalIndex(ctx, *remote, virt.PackageType, path)
+				if err != nil {
+					results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)}
+					return
+				}
+				results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, Body: body}}
+				return
+			}
+
 			prov, err := provider.Get(remote.PackageType)
 			if err != nil {
 				results[idx] = result{err: fmt.Errorf("provider %q: %w", remote.PackageType, err)}
@@ -92,7 +108,7 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
 				return
 			}

-			results[idx] = result{index: MemberIndex{RemoteName: name, Body: body}}
+			results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, Body: body}}
 		}(i, memberName)
 	}

@@ -109,3 +125,20 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa

 	return members, nil
 }
+
+func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, packageType models.PackageType, path string) ([]byte, error) {
+	switch packageType {
+	case models.PackagePyPI:
+		if e.localGen == nil {
+			return nil, fmt.Errorf("no local index generator configured")
+		}
+		parts := strings.SplitN(strings.TrimPrefix(path, "simple/"), "/", 2)
+		pkgName := strings.TrimSuffix(parts[0], "/")
+		if pkgName == "" {
+			return nil, fmt.Errorf("cannot determine package name from path %q", path)
+		}
+		return e.localGen.GeneratePyPIPackageHTML(ctx, remote.Name, pkgName)
+	default:
+		return nil, fmt.Errorf("local index generation not supported for %q", packageType)
+	}
+}
@@ -8,6 +8,7 @@ import (

 type MemberIndex struct {
 	RemoteName string
+	RepoType   models.RepoType
 	Body       []byte
 }

@@ -36,8 +36,13 @@ func (m *PyPIMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
 			}

 			if proxyBaseURL != "" && href != "" {
-				href = fmt.Sprintf("%s/api/v1/remote/%s/%s",
+				routePrefix := "remote"
+				if member.RepoType == "local" {
+					routePrefix = "local"
+				}
+				href = fmt.Sprintf("%s/api/v1/%s/%s/%s",
 					strings.TrimRight(proxyBaseURL, "/"),
+					routePrefix,
 					member.RemoteName,
 					strings.TrimLeft(href, "/"))
 			}