From 2f1e3830da8b0b8867f69ada0fb0caef2b64a91d Mon Sep 17 00:00:00 2001 From: Ben Vincent Date: Sun, 21 Jun 2026 23:01:59 +1000 Subject: [PATCH] feat: add terraform-local package type for local registry - New package type "terraform-local" for hosting terraform providers without an upstream registry - Upload via PUT /api/v2/remotes/{name}/files/{path}, returns 409 on duplicate (overwrites not allowed) - Download via GET on same path, or via v1 proxy route - Provider registers as fully immutable, no upstream rewriting - v1 proxy handler serves local files directly for local repo types - Database CRUD for local_files table --- internal/api/v1/proxy.go | 39 ++++- internal/api/v2/local.go | 142 ++++++++++++++++++ internal/database/local_files.go | 81 ++++++++++ .../provider/terraformlocal/terraformlocal.go | 49 ++++++ internal/server/server.go | 10 +- pkg/models/package_type.go | 14 +- 6 files changed, 328 insertions(+), 7 deletions(-) create mode 100644 internal/api/v2/local.go create mode 100644 internal/database/local_files.go create mode 100644 internal/provider/terraformlocal/terraformlocal.go diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index e1d3e1e..c85d784 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -12,6 +12,7 @@ import ( "git.unkin.net/unkin/artifactapi/internal/database" "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/internal/proxy" + "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/virtual" ) @@ -19,10 +20,11 @@ type ProxyHandler struct { engine *proxy.Engine virtualEngine *virtual.Engine db *database.DB + store *storage.S3 } -func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB) *ProxyHandler { - return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db} +func NewProxyHandler(engine *proxy.Engine, virtualEngine *virtual.Engine, db *database.DB, store *storage.S3) *ProxyHandler { + return &ProxyHandler{engine: engine, virtualEngine: virtualEngine, db: db, store: store} } func (h *ProxyHandler) Routes() chi.Router { @@ -42,6 +44,11 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) { return } + if remote.PackageType.IsLocal() { + h.serveLocal(w, r, remoteName, path) + return + } + prov, err := provider.Get(remote.PackageType) if err != nil { http.Error(w, fmt.Sprintf("no provider for %q", remote.PackageType), http.StatusInternalServerError) @@ -95,6 +102,34 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) { w.Write(body) } +func (h *ProxyHandler) serveLocal(w http.ResponseWriter, r *http.Request, repoName, path string) { + file, err := h.db.GetLocalFile(r.Context(), repoName, path) + if err != nil { + slog.Error("local file lookup failed", "repo", repoName, "path", path, "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + if file == nil { + http.Error(w, "not found", http.StatusNotFound) + return + } + + s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):]) + reader, info, err := h.store.Download(r.Context(), s3Key) + if err != nil { + slog.Error("local file download failed", "repo", repoName, "path", path, "error", err) + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + defer reader.Close() + + w.Header().Set("Content-Type", info.ContentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size)) + w.Header().Set("X-Artifact-Source", "local") + w.WriteHeader(http.StatusOK) + io.Copy(w, reader) +} + func scheme(r *http.Request) string { if r.TLS != nil { return "https" diff --git a/internal/api/v2/local.go b/internal/api/v2/local.go new file mode 100644 index 0000000..1206657 --- /dev/null +++ b/internal/api/v2/local.go @@ -0,0 +1,142 @@ +package v2 + +import ( + "errors" + "fmt" + "io" + "net/http" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/storage" +) + +type LocalHandler struct { + db *database.DB + store *storage.S3 + cas *storage.CAS +} + +func NewLocalHandler(db *database.DB, store *storage.S3) *LocalHandler { + return &LocalHandler{ + db: db, + store: store, + cas: storage.NewCAS(store), + } +} + +func (h *LocalHandler) Routes() chi.Router { + r := chi.NewRouter() + r.Put("/*", h.upload) + r.Get("/*", h.download) + r.Delete("/*", h.remove) + return r +} + +func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) { + repoName := chi.URLParam(r, "name") + filePath := chi.URLParam(r, "*") + + if filePath == "" { + http.Error(w, "file path required", http.StatusBadRequest) + return + } + + remote, err := h.db.GetRemote(r.Context(), repoName) + if err != nil { + http.Error(w, fmt.Sprintf("remote %q not found", repoName), http.StatusNotFound) + return + } + if !remote.PackageType.IsLocal() { + http.Error(w, "upload only allowed for local repository types", http.StatusBadRequest) + return + } + + existing, err := h.db.GetLocalFile(r.Context(), repoName, filePath) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if existing != nil { + http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict) + return + } + + prov, err := provider.Get(remote.PackageType) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + contentType := prov.ContentType(filePath) + if ct := r.Header.Get("Content-Type"); ct != "" && ct != "application/octet-stream" { + contentType = ct + } + + result, err := h.cas.Store(r.Context(), r.Body, contentType) + if err != nil { + http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError) + return + } + + if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil { + http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError) + return + } + + if err := h.db.CreateLocalFile(r.Context(), repoName, filePath, result.ContentHash); err != nil { + if errors.Is(err, database.ErrAlreadyExists) { + http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", filePath), http.StatusConflict) + return + } + http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError) + return + } + + writeJSON(w, http.StatusCreated, map[string]any{ + "path": filePath, + "content_hash": result.ContentHash, + "size_bytes": result.SizeBytes, + }) +} + +func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) { + repoName := chi.URLParam(r, "name") + filePath := chi.URLParam(r, "*") + + file, err := h.db.GetLocalFile(r.Context(), repoName, filePath) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if file == nil { + http.Error(w, "not found", http.StatusNotFound) + return + } + + s3Key := storage.BlobKey(file.ContentHash[len("sha256:"):]) + reader, info, err := h.store.Download(r.Context(), s3Key) + if err != nil { + http.Error(w, fmt.Sprintf("download failed: %v", err), http.StatusInternalServerError) + return + } + defer reader.Close() + + w.Header().Set("Content-Type", info.ContentType) + w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size)) + w.WriteHeader(http.StatusOK) + io.Copy(w, reader) +} + +func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) { + repoName := chi.URLParam(r, "name") + filePath := chi.URLParam(r, "*") + + if err := h.db.DeleteLocalFile(r.Context(), repoName, filePath); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/database/local_files.go b/internal/database/local_files.go new file mode 100644 index 0000000..54857f2 --- /dev/null +++ b/internal/database/local_files.go @@ -0,0 +1,81 @@ +package database + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type LocalFile struct { + ID int64 `json:"id"` + RepoName string `json:"repo_name"` + FilePath string `json:"file_path"` + ContentHash string `json:"content_hash"` + CreatedAt string `json:"created_at"` +} + +var ErrAlreadyExists = fmt.Errorf("file already exists") + +func (db *DB) CreateLocalFile(ctx context.Context, repoName, filePath, contentHash string) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO local_files (repo_name, file_path, content_hash) + VALUES ($1, $2, $3) + `, repoName, filePath, contentHash) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return ErrAlreadyExists + } + return err + } + return nil +} + +func (db *DB) GetLocalFile(ctx context.Context, repoName, filePath string) (*LocalFile, error) { + row := db.Pool.QueryRow(ctx, ` + SELECT id, repo_name, file_path, content_hash, created_at + FROM local_files + WHERE repo_name = $1 AND file_path = $2 + `, repoName, filePath) + + var f LocalFile + if err := row.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + return nil, err + } + return &f, nil +} + +func (db *DB) ListLocalFiles(ctx context.Context, repoName string, limit, offset int) ([]LocalFile, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT id, repo_name, file_path, content_hash, created_at + FROM local_files + WHERE repo_name = $1 + ORDER BY file_path + LIMIT $2 OFFSET $3 + `, repoName, limit, offset) + if err != nil { + return nil, err + } + defer rows.Close() + + var files []LocalFile + for rows.Next() { + var f LocalFile + if err := rows.Scan(&f.ID, &f.RepoName, &f.FilePath, &f.ContentHash, &f.CreatedAt); err != nil { + return nil, err + } + files = append(files, f) + } + return files, rows.Err() +} + +func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} diff --git a/internal/provider/terraformlocal/terraformlocal.go b/internal/provider/terraformlocal/terraformlocal.go new file mode 100644 index 0000000..1a9d0dc --- /dev/null +++ b/internal/provider/terraformlocal/terraformlocal.go @@ -0,0 +1,49 @@ +package terraformlocal + +import ( + "context" + "net/http" + "path" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(&Provider{}) +} + +type Provider struct{} + +func (p *Provider) Type() models.PackageType { return models.PackageTerraformLocal } + +func (p *Provider) Classify(_ string) provider.Mutability { + return provider.Immutable +} + +var contentTypeMap = map[string]string{ + ".zip": "application/zip", + ".json": "application/json", + ".sig": "application/octet-stream", +} + +func (p *Provider) ContentType(filePath string) string { + ext := path.Ext(strings.ToLower(filePath)) + if ct, ok := contentTypeMap[ext]; ok { + return ct + } + return "application/octet-stream" +} + +func (p *Provider) UpstreamURL(_ models.Remote, _ string) string { + return "" +} + +func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) { + return nil, nil +} + +func (p *Provider) AuthHeaders(_ context.Context, _ models.Remote) (http.Header, error) { + return http.Header{}, nil +} diff --git a/internal/server/server.go b/internal/server/server.go index f79da71..448fe3b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -28,6 +28,7 @@ import ( _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi" _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform" + _ "git.unkin.net/unkin/artifactapi/internal/provider/terraformlocal" "git.unkin.net/unkin/artifactapi/internal/proxy" "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/virtual" @@ -91,7 +92,7 @@ func (s *Server) routes() chi.Router { r.Get("/health", s.handleHealth) r.Get("/", s.handleRoot) - proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db) + proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store) r.Mount("/api/v1", proxyHandler.Routes()) remotesHandler := v2.NewRemotesHandler(s.db) @@ -114,6 +115,13 @@ func (s *Server) routes() chi.Router { r.Get("/", objHandler.Routes().ServeHTTP) r.Delete("/*", objHandler.Routes().ServeHTTP) }) + + localHandler := v2.NewLocalHandler(s.db, s.store) + r.Route("/remotes/{name}/files", func(r chi.Router) { + r.Put("/*", localHandler.Routes().ServeHTTP) + r.Get("/*", localHandler.Routes().ServeHTTP) + r.Delete("/*", localHandler.Routes().ServeHTTP) + }) }) return r diff --git a/pkg/models/package_type.go b/pkg/models/package_type.go index ea316a8..0dee075 100644 --- a/pkg/models/package_type.go +++ b/pkg/models/package_type.go @@ -13,8 +13,9 @@ const ( PackageRPM PackageType = "rpm" PackageAlpine PackageType = "alpine" PackagePuppet PackageType = "puppet" - PackageTerraform PackageType = "terraform" - PackageGoProxy PackageType = "goproxy" + PackageTerraform PackageType = "terraform" + PackageTerraformLocal PackageType = "terraform-local" + PackageGoProxy PackageType = "goproxy" ) var validPackageTypes = map[PackageType]bool{ @@ -26,14 +27,19 @@ var validPackageTypes = map[PackageType]bool{ PackageRPM: true, PackageAlpine: true, PackagePuppet: true, - PackageTerraform: true, - PackageGoProxy: true, + PackageTerraform: true, + PackageTerraformLocal: true, + PackageGoProxy: true, } func (p PackageType) Valid() bool { return validPackageTypes[p] } +func (p PackageType) IsLocal() bool { + return p == PackageTerraformLocal +} + func (p PackageType) String() string { return string(p) }