From 5b830fc3defa131e1588b06e9bb68e3f1ea9f048 Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Tue, 23 Jun 2026 22:41:29 +1000 Subject: [PATCH] refactor: modular local provider interfaces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move package-type-specific local repo logic into provider packages via optional interfaces, eliminating switch statements from handlers. - provider.LocalUploader: ValidateUpload + UploadResponse - provider.LocalIndexer: ServeLocalIndex + GenerateLocalIndex - provider.FileStore: interface for querying local files (implemented by database.DB) PyPI and Terraform providers now implement both interfaces. The local handler and v1 proxy use type assertions to dispatch — adding a new local repo type only requires implementing the interfaces in its provider package, no handler changes needed. local.go: 468 → 163 lines (removed all PyPI/Terraform specifics) proxy.go: 211 → 136 lines (removed switch + helper methods) engine.go: removed LocalIndexGenerator, uses provider.LocalIndexer --- internal/api/v1/proxy.go | 58 +---- internal/api/v2/local.go | 305 ++--------------------- internal/database/local_files.go | 18 ++ internal/provider/provider.go | 20 ++ internal/provider/pypi/pypi.go | 180 +++++++++++++ internal/provider/terraform/terraform.go | 149 +++++++++++ internal/server/server.go | 2 +- internal/virtual/engine.go | 37 ++- 8 files changed, 400 insertions(+), 369 deletions(-) diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 057c429..f37719a 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -6,8 +6,6 @@ import ( "io" "log/slog" "net/http" - "regexp" - "strings" "github.com/go-chi/chi/v5" @@ -17,11 +15,8 @@ import ( "git.unkin.net/unkin/artifactapi/internal/proxy" "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/virtual" - "git.unkin.net/unkin/artifactapi/pkg/models" ) -var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`) - type ProxyHandler struct { engine *proxy.Engine virtualEngine *virtual.Engine @@ -115,13 +110,9 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) { return } - switch remote.PackageType { - case models.PackageTerraform: - if h.serveTerraformMirror(w, r, remote, path) { - return - } - case models.PackagePyPI: - if h.servePyPIMirror(w, r, remote, path) { + prov, _ := provider.Get(remote.PackageType) + if indexer, ok := prov.(provider.LocalIndexer); ok { + if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) { return } } @@ -129,49 +120,6 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) { h.serveLocalFile(w, r, localName, path) } -func (h *ProxyHandler) serveTerraformMirror(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) bool { - parts := strings.Split(path, "/") - if len(parts) < 3 { - return false - } - - namespace, typeName := parts[0], parts[1] - tail := parts[2] - - if tail == "index.json" { - h.local.ServeTerraformIndex(w, r, remote.Name, namespace, typeName) - return true - } - - if strings.HasSuffix(tail, ".json") { - version := strings.TrimSuffix(tail, ".json") - if semverRe.MatchString(version) { - h.local.ServeTerraformVersionDoc(w, r, remote.Name, namespace, typeName, version) - return true - } - } - - return false -} - -func (h *ProxyHandler) servePyPIMirror(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) bool { - if path == "simple" || path == "simple/" { - h.local.ServePyPIIndex(w, r, remote.Name) - return true - } - - if strings.HasPrefix(path, "simple/") { - pkg := strings.TrimPrefix(path, "simple/") - pkg = strings.TrimSuffix(pkg, "/") - if pkg != "" && !strings.Contains(pkg, "/") { - h.local.ServePyPIPackageIndex(w, r, remote.Name, pkg) - return true - } - } - - return false -} - func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) { file, err := h.db.GetLocalFile(r.Context(), repoName, path) if err != nil { diff --git a/internal/api/v2/local.go b/internal/api/v2/local.go index 17b0f1b..fe704fe 100644 --- a/internal/api/v2/local.go +++ b/internal/api/v2/local.go @@ -1,57 +1,19 @@ package v2 import ( - "context" - "encoding/json" "errors" "fmt" "io" "net/http" - "regexp" - "strings" "github.com/go-chi/chi/v5" "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/pkg/models" ) -var pypiFileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`) - -var pypiNormalizeRe = regexp.MustCompile(`[-_.]+`) - -func pypiNormalize(name string) string { - return strings.ToLower(pypiNormalizeRe.ReplaceAllString(name, "-")) -} - -func pypiPackageFromWheel(filename string) string { - parts := strings.SplitN(filename, "-", 3) - if len(parts) < 2 { - return "" - } - return pypiNormalize(parts[0]) -} - -func pypiPackageFromSdist(filename string) string { - name := filename - for _, suffix := range []string{".tar.gz", ".zip"} { - if strings.HasSuffix(name, suffix) { - name = strings.TrimSuffix(name, suffix) - break - } - } - idx := strings.LastIndex(name, "-") - if idx <= 0 { - return "" - } - return pypiNormalize(name[:idx]) -} - -var providerZipRe = regexp.MustCompile( - `^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`, -) - type LocalHandler struct { db *database.DB store *storage.S3 @@ -93,45 +55,22 @@ func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) { return } - switch remote.PackageType { - case models.PackageTerraform: - h.uploadTerraformProvider(w, r, remote, filePath) - return - case models.PackagePyPI: - h.uploadPyPI(w, r, remote, filePath) + prov, _ := provider.Get(remote.PackageType) + + if uploader, ok := prov.(provider.LocalUploader); ok { + h.uploadValidated(w, r, remote, filePath, uploader) return } h.uploadGeneric(w, r, remote, filePath) } -func (h *LocalHandler) uploadTerraformProvider(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) { - parts := strings.Split(filePath, "/") - if len(parts) != 3 { - http.Error(w, "path must be {namespace}/{type}/{filename}.zip", http.StatusBadRequest) +func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, uploader provider.LocalUploader) { + storagePath, contentType, err := uploader.ValidateUpload(filePath) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) return } - namespace, typeName, filename := parts[0], parts[1], parts[2] - - m := providerZipRe.FindStringSubmatch(filename) - if m == nil { - http.Error(w, fmt.Sprintf( - "filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", - filename, - ), http.StatusBadRequest) - return - } - fileType, version, os, arch := m[1], m[2], m[3], m[4] - - if fileType != typeName { - http.Error(w, fmt.Sprintf( - "provider type in filename %q does not match path type %q", - fileType, typeName, - ), http.StatusBadRequest) - return - } - - storagePath := fmt.Sprintf("%s/%s/%s", namespace, typeName, filename) existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath) if err != nil { @@ -139,20 +78,17 @@ func (h *LocalHandler) uploadTerraformProvider(w http.ResponseWriter, r *http.Re return } if existing != nil { - http.Error(w, fmt.Sprintf( - "provider %s/%s version %s for %s_%s already exists; overwrites are not allowed", - namespace, typeName, version, os, arch, - ), http.StatusConflict) + http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict) return } - result, err := h.cas.Store(r.Context(), r.Body, "application/zip") + result, err := h.cas.Store(r.Context(), r.Body, contentType) if err != nil { http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError) return } - if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, "application/zip"); err != nil { + if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil { http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError) return } @@ -166,15 +102,7 @@ func (h *LocalHandler) uploadTerraformProvider(w http.ResponseWriter, r *http.Re return } - writeJSON(w, http.StatusCreated, map[string]any{ - "namespace": namespace, - "type": typeName, - "version": version, - "os": os, - "arch": arch, - "content_hash": result.ContentHash, - "size_bytes": result.SizeBytes, - }) + writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes)) } func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) { @@ -220,141 +148,6 @@ func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, rem }) } -func (h *LocalHandler) uploadPyPI(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) { - filename := filePath - if idx := strings.LastIndex(filePath, "/"); idx >= 0 { - filename = filePath[idx+1:] - } - - if !pypiFileRe.MatchString(filename) { - http.Error(w, fmt.Sprintf("filename %q must be a .whl, .tar.gz, or .zip file", filename), http.StatusBadRequest) - return - } - - var pkgName string - if strings.HasSuffix(filename, ".whl") { - pkgName = pypiPackageFromWheel(filename) - } else { - pkgName = pypiPackageFromSdist(filename) - } - if pkgName == "" { - http.Error(w, fmt.Sprintf("cannot parse package name from %q", filename), http.StatusBadRequest) - return - } - - storagePath := pkgName + "/" + filename - - existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - if existing != nil { - http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict) - return - } - - contentType := "application/zip" - if strings.HasSuffix(filename, ".tar.gz") { - contentType = "application/gzip" - } - - result, err := h.cas.Store(r.Context(), r.Body, contentType) - if err != nil { - http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError) - return - } - - if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil { - http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError) - return - } - - if err := h.db.CreateLocalFile(r.Context(), remote.Name, storagePath, result.ContentHash); err != nil { - if errors.Is(err, database.ErrAlreadyExists) { - http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict) - return - } - http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError) - return - } - - writeJSON(w, http.StatusCreated, map[string]any{ - "package": pkgName, - "filename": filename, - "content_hash": result.ContentHash, - "size_bytes": result.SizeBytes, - }) -} - -func (h *LocalHandler) ServePyPIIndex(w http.ResponseWriter, r *http.Request, repoName string) { - packages, err := h.db.ListLocalFilePackages(r.Context(), repoName) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - var b strings.Builder - b.WriteString("\n\n") - for _, pkg := range packages { - fmt.Fprintf(&b, "%s\n", pkg, pkg) - } - b.WriteString("\n") - - w.Header().Set("Content-Type", "text/html") - w.WriteHeader(http.StatusOK) - io.WriteString(w, b.String()) -} - -func (h *LocalHandler) ServePyPIPackageIndex(w http.ResponseWriter, r *http.Request, repoName, packageName string) { - normalized := pypiNormalize(packageName) - prefix := normalized + "/" - files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - if len(files) == 0 { - http.Error(w, "not found", http.StatusNotFound) - return - } - - var b strings.Builder - b.WriteString("\n\n") - for _, f := range files { - filename := strings.TrimPrefix(f.FilePath, normalized+"/") - hash := strings.TrimPrefix(f.ContentHash, "sha256:") - fmt.Fprintf(&b, "%s\n", - normalized, filename, hash, filename) - } - b.WriteString("\n") - - w.Header().Set("Content-Type", "text/html") - w.WriteHeader(http.StatusOK) - io.WriteString(w, b.String()) -} - -func (h *LocalHandler) GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error) { - normalized := pypiNormalize(packageName) - prefix := normalized + "/" - files, err := h.db.ListLocalFilesByPrefix(ctx, repoName, prefix) - if err != nil { - return nil, err - } - - var b strings.Builder - b.WriteString("\n\n") - for _, f := range files { - filename := strings.TrimPrefix(f.FilePath, normalized+"/") - hash := strings.TrimPrefix(f.ContentHash, "sha256:") - fmt.Fprintf(&b, "%s\n", - normalized, filename, hash, filename) - } - b.WriteString("\n") - return []byte(b.String()), nil -} - func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) { repoName := chi.URLParam(r, "name") filePath := chi.URLParam(r, "*") @@ -394,74 +187,6 @@ func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) } -type terraformIndex struct { - Versions map[string]json.RawMessage `json:"versions"` -} - -type terraformVersionDoc struct { - Archives map[string]terraformArchive `json:"archives"` -} - -type terraformArchive struct { - URL string `json:"url"` - Hashes []string `json:"hashes,omitempty"` -} - -func (h *LocalHandler) ServeTerraformIndex(w http.ResponseWriter, r *http.Request, repoName, namespace, typeName string) { - prefix := fmt.Sprintf("%s/%s/", namespace, typeName) - files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - versions := map[string]json.RawMessage{} - for _, f := range files { - filename := strings.TrimPrefix(f.FilePath, prefix) - m := providerZipRe.FindStringSubmatch(filename) - if m == nil { - continue - } - versions[m[2]] = json.RawMessage(`{}`) - } - - if len(versions) == 0 { - http.Error(w, "not found", http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(terraformIndex{Versions: versions}) -} - -func (h *LocalHandler) ServeTerraformVersionDoc(w http.ResponseWriter, r *http.Request, repoName, namespace, typeName, version string) { - prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version) - files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - - archives := map[string]terraformArchive{} - for _, f := range files { - filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName)) - m := providerZipRe.FindStringSubmatch(filename) - if m == nil || m[2] != version { - continue - } - platform := m[3] + "_" + m[4] - archive := terraformArchive{URL: filename} - if f.ContentHash != "" { - archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")} - } - archives[platform] = archive - } - - if len(archives) == 0 { - http.Error(w, "not found", http.StatusNotFound) - return - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives}) +func (h *LocalHandler) DB() *database.DB { + return h.db } diff --git a/internal/database/local_files.go b/internal/database/local_files.go index 7039eef..dd00f71 100644 --- a/internal/database/local_files.go +++ b/internal/database/local_files.go @@ -8,6 +8,8 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" + + "git.unkin.net/unkin/artifactapi/internal/provider" ) type LocalFile struct { @@ -122,6 +124,22 @@ func (db *DB) ListLocalFilePackages(ctx context.Context, repoName string) ([]str return packages, rows.Err() } +func (db *DB) ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]provider.FileEntry, error) { + files, err := db.ListLocalFilesByPrefix(ctx, repoName, prefix) + if err != nil { + return nil, err + } + result := make([]provider.FileEntry, len(files)) + for i, f := range files { + result[i] = provider.FileEntry{FilePath: f.FilePath, ContentHash: f.ContentHash} + } + return result, nil +} + +func (db *DB) ListPackages(ctx context.Context, repoName string) ([]string, error) { + return db.ListLocalFilePackages(ctx, repoName) +} + func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error { _, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) return err diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 10fce92..27d6358 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -24,6 +24,26 @@ type Provider interface { AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error) } +type FileEntry struct { + FilePath string + ContentHash string +} + +type FileStore interface { + ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error) + ListPackages(ctx context.Context, repoName string) ([]string, error) +} + +type LocalUploader interface { + ValidateUpload(filePath string) (storagePath, contentType string, err error) + UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any +} + +type LocalIndexer interface { + ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool + GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error) +} + type IndexMerger interface { MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) } diff --git a/internal/provider/pypi/pypi.go b/internal/provider/pypi/pypi.go index 8956247..e87e9c1 100644 --- a/internal/provider/pypi/pypi.go +++ b/internal/provider/pypi/pypi.go @@ -2,7 +2,10 @@ package pypi import ( "context" + "fmt" + "io" "net/http" + "regexp" "strings" "git.unkin.net/unkin/artifactapi/internal/auth" @@ -14,6 +17,9 @@ func init() { provider.Register(&Provider{}) } +var fileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`) +var normalizeRe = regexp.MustCompile(`[-_.]+`) + type Provider struct{} func (p *Provider) Type() models.PackageType { return models.PackagePyPI } @@ -60,3 +66,177 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { return auth.BasicHeaders(remote), nil } + +func normalize(name string) string { + return strings.ToLower(normalizeRe.ReplaceAllString(name, "-")) +} + +func packageFromWheel(filename string) string { + parts := strings.SplitN(filename, "-", 3) + if len(parts) < 2 { + return "" + } + return normalize(parts[0]) +} + +func packageFromSdist(filename string) string { + name := filename + for _, suffix := range []string{".tar.gz", ".zip"} { + if strings.HasSuffix(name, suffix) { + name = strings.TrimSuffix(name, suffix) + break + } + } + idx := strings.LastIndex(name, "-") + if idx <= 0 { + return "" + } + return normalize(name[:idx]) +} + +func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { + filename := filePath + if idx := strings.LastIndex(filePath, "/"); idx >= 0 { + filename = filePath[idx+1:] + } + + if !fileRe.MatchString(filename) { + return "", "", fmt.Errorf("filename %q must be a .whl, .tar.gz, or .zip file", filename) + } + + var pkgName string + if strings.HasSuffix(filename, ".whl") { + pkgName = packageFromWheel(filename) + } else { + pkgName = packageFromSdist(filename) + } + if pkgName == "" { + return "", "", fmt.Errorf("cannot parse package name from %q", filename) + } + + ct := "application/zip" + if strings.HasSuffix(filename, ".tar.gz") { + ct = "application/gzip" + } + + return pkgName + "/" + filename, ct, nil +} + +func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any { + parts := strings.SplitN(storagePath, "/", 2) + filename := storagePath + if len(parts) == 2 { + filename = parts[1] + } + return map[string]any{ + "package": parts[0], + "filename": filename, + "content_hash": contentHash, + "size_bytes": sizeBytes, + } +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool { + if path == "simple" || path == "simple/" { + p.servePackageList(w, r, files, repoName) + return true + } + + if strings.HasPrefix(path, "simple/") { + pkg := strings.TrimPrefix(path, "simple/") + pkg = strings.TrimSuffix(pkg, "/") + if pkg != "" && !strings.Contains(pkg, "/") { + p.servePackageFiles(w, r, files, repoName, pkg) + return true + } + } + + return false +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) { + if !strings.HasPrefix(path, "simple/") { + return nil, fmt.Errorf("unsupported index path: %q", path) + } + + pkg := strings.TrimPrefix(path, "simple/") + pkg = strings.TrimSuffix(pkg, "/") + if pkg == "" { + return p.generatePackageListHTML(ctx, files, repoName) + } + return p.generatePackageFilesHTML(ctx, files, repoName, pkg) +} + +func (p *Provider) servePackageList(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName string) { + body, err := p.generatePackageListHTML(r.Context(), files, repoName) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + w.Write(body) +} + +func (p *Provider) servePackageFiles(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, packageName string) { + normalized := normalize(packageName) + prefix := normalized + "/" + entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if len(entries) == 0 { + http.Error(w, "not found", http.StatusNotFound) + return + } + + var b strings.Builder + b.WriteString("\n\n") + for _, f := range entries { + filename := strings.TrimPrefix(f.FilePath, normalized+"/") + hash := strings.TrimPrefix(f.ContentHash, "sha256:") + fmt.Fprintf(&b, "%s\n", + normalized, filename, hash, filename) + } + b.WriteString("\n") + + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + io.WriteString(w, b.String()) +} + +func (p *Provider) generatePackageListHTML(ctx context.Context, files provider.FileStore, repoName string) ([]byte, error) { + packages, err := files.ListPackages(ctx, repoName) + if err != nil { + return nil, err + } + + var b strings.Builder + b.WriteString("\n\n") + for _, pkg := range packages { + fmt.Fprintf(&b, "%s\n", pkg, pkg) + } + b.WriteString("\n") + return []byte(b.String()), nil +} + +func (p *Provider) generatePackageFilesHTML(ctx context.Context, files provider.FileStore, repoName, packageName string) ([]byte, error) { + normalized := normalize(packageName) + prefix := normalized + "/" + entries, err := files.ListFilesByPrefix(ctx, repoName, prefix) + if err != nil { + return nil, err + } + + var b strings.Builder + b.WriteString("\n\n") + for _, f := range entries { + filename := strings.TrimPrefix(f.FilePath, normalized+"/") + hash := strings.TrimPrefix(f.ContentHash, "sha256:") + fmt.Fprintf(&b, "%s\n", + normalized, filename, hash, filename) + } + b.WriteString("\n") + return []byte(b.String()), nil +} diff --git a/internal/provider/terraform/terraform.go b/internal/provider/terraform/terraform.go index 9e0009f..f2633de 100644 --- a/internal/provider/terraform/terraform.go +++ b/internal/provider/terraform/terraform.go @@ -3,6 +3,7 @@ package terraform import ( "context" "encoding/json" + "fmt" "net/http" "net/url" "regexp" @@ -19,6 +20,12 @@ func init() { var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`) +var providerZipRe = regexp.MustCompile( + `^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`, +) + +var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`) + type Provider struct{} func (p *Provider) Type() models.PackageType { return models.PackageTerraform } @@ -86,3 +93,145 @@ func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { return auth.BasicHeaders(remote), nil } + +func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { + parts := strings.Split(filePath, "/") + if len(parts) != 3 { + return "", "", fmt.Errorf("path must be {namespace}/{type}/{filename}.zip") + } + namespace, typeName, filename := parts[0], parts[1], parts[2] + + m := providerZipRe.FindStringSubmatch(filename) + if m == nil { + return "", "", fmt.Errorf("filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", filename) + } + + if m[1] != typeName { + return "", "", fmt.Errorf("provider type in filename %q does not match path type %q", m[1], typeName) + } + + return fmt.Sprintf("%s/%s/%s", namespace, typeName, filename), "application/zip", nil +} + +func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any { + parts := strings.Split(storagePath, "/") + if len(parts) != 3 { + return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes} + } + + m := providerZipRe.FindStringSubmatch(parts[2]) + if m == nil { + return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes} + } + + return map[string]any{ + "namespace": parts[0], + "type": parts[1], + "version": m[2], + "os": m[3], + "arch": m[4], + "content_hash": contentHash, + "size_bytes": sizeBytes, + } +} + +type terraformIndex struct { + Versions map[string]json.RawMessage `json:"versions"` +} + +type terraformVersionDoc struct { + Archives map[string]terraformArchive `json:"archives"` +} + +type terraformArchive struct { + URL string `json:"url"` + Hashes []string `json:"hashes,omitempty"` +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool { + parts := strings.Split(path, "/") + if len(parts) < 3 { + return false + } + + namespace, typeName := parts[0], parts[1] + tail := parts[2] + + if tail == "index.json" { + p.serveIndex(w, r, files, repoName, namespace, typeName) + return true + } + + if strings.HasSuffix(tail, ".json") { + version := strings.TrimSuffix(tail, ".json") + if semverRe.MatchString(version) { + p.serveVersionDoc(w, r, files, repoName, namespace, typeName, version) + return true + } + } + + return false +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) { + return nil, fmt.Errorf("terraform local index generation for virtual repos not supported") +} + +func (p *Provider) serveIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName string) { + prefix := fmt.Sprintf("%s/%s/", namespace, typeName) + entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + versions := map[string]json.RawMessage{} + for _, f := range entries { + filename := strings.TrimPrefix(f.FilePath, prefix) + m := providerZipRe.FindStringSubmatch(filename) + if m == nil { + continue + } + versions[m[2]] = json.RawMessage(`{}`) + } + + if len(versions) == 0 { + http.Error(w, "not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(terraformIndex{Versions: versions}) +} + +func (p *Provider) serveVersionDoc(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName, version string) { + prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version) + entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + archives := map[string]terraformArchive{} + for _, f := range entries { + filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName)) + m := providerZipRe.FindStringSubmatch(filename) + if m == nil || m[2] != version { + continue + } + platform := m[3] + "_" + m[4] + archive := terraformArchive{URL: filename} + if f.ContentHash != "" { + archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")} + } + archives[platform] = archive + } + + if len(archives) == 0 { + http.Error(w, "not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives}) +} diff --git a/internal/server/server.go b/internal/server/server.go index 3b9ffff..b28e6b9 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -63,7 +63,7 @@ func New(cfg *config.Config) (*Server, error) { engine := proxy.NewEngine(db, redis, s3) localHandler := v2.NewLocalHandler(db, s3) - virtEngine := virtual.NewEngine(db, engine, localHandler) + virtEngine := virtual.NewEngine(db, engine) collector := gc.New(db, s3, 1*time.Hour) s := &Server{ diff --git a/internal/virtual/engine.go b/internal/virtual/engine.go index ab7a89f..b80393b 100644 --- a/internal/virtual/engine.go +++ b/internal/virtual/engine.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "log/slog" - "strings" "sync" "git.unkin.net/unkin/artifactapi/internal/database" @@ -14,18 +13,13 @@ import ( "git.unkin.net/unkin/artifactapi/pkg/models" ) -type LocalIndexGenerator interface { - GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error) -} - type Engine struct { db *database.DB proxyEngine *proxy.Engine - localGen LocalIndexGenerator } -func NewEngine(db *database.DB, proxyEngine *proxy.Engine, localGen LocalIndexGenerator) *Engine { - return &Engine{db: db, proxyEngine: proxyEngine, localGen: localGen} +func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine { + return &Engine{db: db, proxyEngine: proxyEngine} } func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) { @@ -80,7 +74,7 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa } if remote.RepoType == models.RepoTypeLocal { - body, err := e.fetchLocalIndex(ctx, *remote, virt.PackageType, path) + body, err := e.fetchLocalIndex(ctx, *remote, path) if err != nil { results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)} return @@ -126,19 +120,16 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa return members, nil } -func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, packageType models.PackageType, path string) ([]byte, error) { - switch packageType { - case models.PackagePyPI: - if e.localGen == nil { - return nil, fmt.Errorf("no local index generator configured") - } - parts := strings.SplitN(strings.TrimPrefix(path, "simple/"), "/", 2) - pkgName := strings.TrimSuffix(parts[0], "/") - if pkgName == "" { - return nil, fmt.Errorf("cannot determine package name from path %q", path) - } - return e.localGen.GeneratePyPIPackageHTML(ctx, remote.Name, pkgName) - default: - return nil, fmt.Errorf("local index generation not supported for %q", packageType) +func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, path string) ([]byte, error) { + prov, err := provider.Get(remote.PackageType) + if err != nil { + return nil, fmt.Errorf("no provider for %q: %w", remote.PackageType, err) } + + indexer, ok := prov.(provider.LocalIndexer) + if !ok { + return nil, fmt.Errorf("provider %q does not support local index generation", remote.PackageType) + } + + return indexer.GenerateLocalIndex(ctx, e.db, remote.Name, path) }