Compare commits

..

4 Commits

Author SHA1 Message Date
unkinben 74d9c0fa84 chore: add pre-commit config and update CI pipeline (#55)
ci/woodpecker/tag/docker Pipeline was successful
## Summary
- New `.pre-commit-config.yaml` with standard Go hooks (gofmt, go vet, go mod tidy) plus file hygiene checks (trailing whitespace, end-of-file, yaml, large files, merge conflicts)
- go vet runs as a local hook with `./...` since the dnephin per-file hook doesn't work with Go module layouts
- Woodpecker pre-commit pipeline updated to use `almalinux9-gobuilder` image with `uvx pre-commit run --all-files`
- Pre-commit hooks installed into the repo

Reviewed-on: #55
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
2026-06-23 23:21:09 +10:00
unkinben 097fbf0016 feat: UI separates locals, remotes, and virtuals (#54)
## Summary
- New "Locals" sidebar nav item with list + detail + browse pages
- Remotes page filters out local repos (repo_type=local hidden)
- LocalDetail: simplified view — just name, type, description + "Browse Files" button
- Virtuals: member links resolve to /locals/ or /remotes/ based on repo_type
- Objects page detects context for correct back-navigation

## Test plan
- [ ] Visual check: locals page shows only local repos
- [ ] Remotes page hides local repos
- [ ] Virtual member links point to correct pages
- [ ] Browse files works from local detail page

Reviewed-on: #54
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
2026-06-23 23:20:18 +10:00
unkinben 6f8e70c27a feat: add local RPM repository with on-demand repodata (#53)
## Summary
- Upload RPMs to local repos, metadata parsed async via cavaliergopher/rpm
- Repodata (repomd.xml, primary/filelists/other.xml.gz) generated on-demand from DB — nothing stored in S3
- RPM provider implements LocalUploader, PostUploadHook, and LocalIndexer
- New rpm_metadata table for parsed RPM header data (name, version, deps, etc.)
- New provider interfaces: PostUploadHook, BlobReader, MetadataStore, RPMMetadataReader

## Test plan
- [x] Upload cowsay RPM from epel → async metadata parse confirmed in logs
- [x] repomd.xml generated with correct hashes → primary.xml.gz has correct metadata
- [x] `dnf install` from local repo: download + install successful
- [x] Bad file rejection (.txt → 400), overwrite rejection (409)

Reviewed-on: #53
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
2026-06-23 23:20:05 +10:00
unkinben 3a6721c2a7 refactor: modular local provider interfaces (#52)
ci/woodpecker/tag/docker Pipeline was successful
## Summary
Move package-type-specific local repo logic out of centralized handlers into provider packages via optional Go interfaces.

**New interfaces in `provider` package:**
- \`LocalUploader\`: \`ValidateUpload(filePath) → (storagePath, contentType, error)\` + \`UploadResponse(...)\`
- \`LocalIndexer\`: \`ServeLocalIndex(w, r, files, repoName, path) → bool\` + \`GenerateLocalIndex(ctx, files, repoName, path) → ([]byte, error)\`
- \`FileStore\`: \`ListFilesByPrefix\` + \`ListPackages\` (implemented by database.DB)

**Providers implement these interfaces:**
- PyPI: upload validation (wheel/sdist naming), simple index serving + generation
- Terraform: upload validation (provider zip naming), mirror protocol serving

**Handlers simplified to generic dispatch:**
- \`local.go\`: type-asserts to \`LocalUploader\`, falls back to generic upload
- \`proxy.go\`: type-asserts to \`LocalIndexer\`, falls back to raw file serving
- \`engine.go\`: type-asserts to \`LocalIndexer\` for local virtual members

Adding a new local repo type (e.g. RPM) = implement the interfaces in its provider package. Zero handler changes.

## Test plan
- [x] Build + unit tests pass
- [x] E2E: PyPI local upload → simple index → uv pip install (smoke test after refactor)

Reviewed-on: #52
Co-authored-by: Ben Vincent <ben@unkin.net>
Co-committed-by: Ben Vincent <ben@unkin.net>
2026-06-23 22:56:25 +10:00
21 changed files with 1240 additions and 381 deletions
+24
View File
@@ -0,0 +1,24 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
- id: check-merge-conflict
- repo: https://github.com/dnephin/pre-commit-golang
rev: v0.5.1
hooks:
- id: go-fmt
- id: go-mod-tidy
- repo: local
hooks:
- id: go-vet
name: go vet
entry: go vet ./...
language: system
types: [go]
pass_filenames: false
+11 -3
View File
@@ -3,7 +3,15 @@ when:
steps:
- name: pre-commit
image: golang:1.25
image: git.unkin.net/unkin/almalinux9-gobuilder:20260606
commands:
- test -z "$(gofmt -l .)"
- go vet ./...
- uvx pre-commit run --all-files
backend_options:
kubernetes:
resources:
requests:
memory: 512Mi
cpu: 1
limits:
memory: 2Gi
cpu: 2
+1
View File
@@ -3,6 +3,7 @@ module git.unkin.net/unkin/artifactapi
go 1.25.9
require (
github.com/cavaliergopher/rpm v1.3.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/go-chi/chi/v5 v5.3.0
+2
View File
@@ -12,6 +12,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cavaliergopher/rpm v1.3.0 h1:UHX46sasX8MesUXXQ+UbkFLUX4eUWTlEcX8jcnRBIgI=
github.com/cavaliergopher/rpm v1.3.0/go.mod h1:vEumo1vvtrHM1Ov86f6+k8j7zNKOxQfHDCAIcR/36ZI=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+3 -55
View File
@@ -6,8 +6,6 @@ import (
"io"
"log/slog"
"net/http"
"regexp"
"strings"
"github.com/go-chi/chi/v5"
@@ -17,11 +15,8 @@ import (
"git.unkin.net/unkin/artifactapi/internal/proxy"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/internal/virtual"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
type ProxyHandler struct {
engine *proxy.Engine
virtualEngine *virtual.Engine
@@ -115,13 +110,9 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
return
}
switch remote.PackageType {
case models.PackageTerraform:
if h.serveTerraformMirror(w, r, remote, path) {
return
}
case models.PackagePyPI:
if h.servePyPIMirror(w, r, remote, path) {
prov, _ := provider.Get(remote.PackageType)
if indexer, ok := prov.(provider.LocalIndexer); ok {
if indexer.ServeLocalIndex(w, r, h.db, remote.Name, path) {
return
}
}
@@ -129,49 +120,6 @@ func (h *ProxyHandler) handleLocal(w http.ResponseWriter, r *http.Request) {
h.serveLocalFile(w, r, localName, path)
}
func (h *ProxyHandler) serveTerraformMirror(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) bool {
parts := strings.Split(path, "/")
if len(parts) < 3 {
return false
}
namespace, typeName := parts[0], parts[1]
tail := parts[2]
if tail == "index.json" {
h.local.ServeTerraformIndex(w, r, remote.Name, namespace, typeName)
return true
}
if strings.HasSuffix(tail, ".json") {
version := strings.TrimSuffix(tail, ".json")
if semverRe.MatchString(version) {
h.local.ServeTerraformVersionDoc(w, r, remote.Name, namespace, typeName, version)
return true
}
}
return false
}
func (h *ProxyHandler) servePyPIMirror(w http.ResponseWriter, r *http.Request, remote *models.Remote, path string) bool {
if path == "simple" || path == "simple/" {
h.local.ServePyPIIndex(w, r, remote.Name)
return true
}
if strings.HasPrefix(path, "simple/") {
pkg := strings.TrimPrefix(path, "simple/")
pkg = strings.TrimSuffix(pkg, "/")
if pkg != "" && !strings.Contains(pkg, "/") {
h.local.ServePyPIPackageIndex(w, r, remote.Name, pkg)
return true
}
}
return false
}
func (h *ProxyHandler) serveLocalFile(w http.ResponseWriter, r *http.Request, repoName, path string) {
file, err := h.db.GetLocalFile(r.Context(), repoName, path)
if err != nil {
+23 -285
View File
@@ -2,56 +2,19 @@ package v2
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var pypiFileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`)
var pypiNormalizeRe = regexp.MustCompile(`[-_.]+`)
func pypiNormalize(name string) string {
return strings.ToLower(pypiNormalizeRe.ReplaceAllString(name, "-"))
}
func pypiPackageFromWheel(filename string) string {
parts := strings.SplitN(filename, "-", 3)
if len(parts) < 2 {
return ""
}
return pypiNormalize(parts[0])
}
func pypiPackageFromSdist(filename string) string {
name := filename
for _, suffix := range []string{".tar.gz", ".zip"} {
if strings.HasSuffix(name, suffix) {
name = strings.TrimSuffix(name, suffix)
break
}
}
idx := strings.LastIndex(name, "-")
if idx <= 0 {
return ""
}
return pypiNormalize(name[:idx])
}
var providerZipRe = regexp.MustCompile(
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
)
type LocalHandler struct {
db *database.DB
store *storage.S3
@@ -93,45 +56,22 @@ func (h *LocalHandler) upload(w http.ResponseWriter, r *http.Request) {
return
}
switch remote.PackageType {
case models.PackageTerraform:
h.uploadTerraformProvider(w, r, remote, filePath)
return
case models.PackagePyPI:
h.uploadPyPI(w, r, remote, filePath)
prov, _ := provider.Get(remote.PackageType)
if uploader, ok := prov.(provider.LocalUploader); ok {
h.uploadValidated(w, r, remote, filePath, prov, uploader)
return
}
h.uploadGeneric(w, r, remote, filePath)
}
func (h *LocalHandler) uploadTerraformProvider(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
parts := strings.Split(filePath, "/")
if len(parts) != 3 {
http.Error(w, "path must be {namespace}/{type}/{filename}.zip", http.StatusBadRequest)
func (h *LocalHandler) uploadValidated(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string, prov provider.Provider, uploader provider.LocalUploader) {
storagePath, contentType, err := uploader.ValidateUpload(filePath)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
namespace, typeName, filename := parts[0], parts[1], parts[2]
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
http.Error(w, fmt.Sprintf(
"filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip",
filename,
), http.StatusBadRequest)
return
}
fileType, version, os, arch := m[1], m[2], m[3], m[4]
if fileType != typeName {
http.Error(w, fmt.Sprintf(
"provider type in filename %q does not match path type %q",
fileType, typeName,
), http.StatusBadRequest)
return
}
storagePath := fmt.Sprintf("%s/%s/%s", namespace, typeName, filename)
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath)
if err != nil {
@@ -139,20 +79,17 @@ func (h *LocalHandler) uploadTerraformProvider(w http.ResponseWriter, r *http.Re
return
}
if existing != nil {
http.Error(w, fmt.Sprintf(
"provider %s/%s version %s for %s_%s already exists; overwrites are not allowed",
namespace, typeName, version, os, arch,
), http.StatusConflict)
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
return
}
result, err := h.cas.Store(r.Context(), r.Body, "application/zip")
result, err := h.cas.Store(r.Context(), r.Body, contentType)
if err != nil {
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
return
}
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, "application/zip"); err != nil {
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
return
}
@@ -166,15 +103,11 @@ func (h *LocalHandler) uploadTerraformProvider(w http.ResponseWriter, r *http.Re
return
}
writeJSON(w, http.StatusCreated, map[string]any{
"namespace": namespace,
"type": typeName,
"version": version,
"os": os,
"arch": arch,
"content_hash": result.ContentHash,
"size_bytes": result.SizeBytes,
})
if hook, ok := prov.(provider.PostUploadHook); ok {
go hook.AfterUpload(context.Background(), remote.Name, storagePath, result.ContentHash, h, h.db)
}
writeJSON(w, http.StatusCreated, uploader.UploadResponse(storagePath, result.ContentHash, result.SizeBytes))
}
func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
@@ -220,141 +153,6 @@ func (h *LocalHandler) uploadGeneric(w http.ResponseWriter, r *http.Request, rem
})
}
func (h *LocalHandler) uploadPyPI(w http.ResponseWriter, r *http.Request, remote *models.Remote, filePath string) {
filename := filePath
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
filename = filePath[idx+1:]
}
if !pypiFileRe.MatchString(filename) {
http.Error(w, fmt.Sprintf("filename %q must be a .whl, .tar.gz, or .zip file", filename), http.StatusBadRequest)
return
}
var pkgName string
if strings.HasSuffix(filename, ".whl") {
pkgName = pypiPackageFromWheel(filename)
} else {
pkgName = pypiPackageFromSdist(filename)
}
if pkgName == "" {
http.Error(w, fmt.Sprintf("cannot parse package name from %q", filename), http.StatusBadRequest)
return
}
storagePath := pkgName + "/" + filename
existing, err := h.db.GetLocalFile(r.Context(), remote.Name, storagePath)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if existing != nil {
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
return
}
contentType := "application/zip"
if strings.HasSuffix(filename, ".tar.gz") {
contentType = "application/gzip"
}
result, err := h.cas.Store(r.Context(), r.Body, contentType)
if err != nil {
http.Error(w, fmt.Sprintf("store failed: %v", err), http.StatusInternalServerError)
return
}
if err := h.db.UpsertBlob(r.Context(), result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil {
http.Error(w, fmt.Sprintf("record blob: %v", err), http.StatusInternalServerError)
return
}
if err := h.db.CreateLocalFile(r.Context(), remote.Name, storagePath, result.ContentHash); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
http.Error(w, fmt.Sprintf("file %q already exists; overwrites are not allowed", storagePath), http.StatusConflict)
return
}
http.Error(w, fmt.Sprintf("record file: %v", err), http.StatusInternalServerError)
return
}
writeJSON(w, http.StatusCreated, map[string]any{
"package": pkgName,
"filename": filename,
"content_hash": result.ContentHash,
"size_bytes": result.SizeBytes,
})
}
func (h *LocalHandler) ServePyPIIndex(w http.ResponseWriter, r *http.Request, repoName string) {
packages, err := h.db.ListLocalFilePackages(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, pkg := range packages {
fmt.Fprintf(&b, "<a href=\"%s/\">%s</a>\n", pkg, pkg)
}
b.WriteString("</body></html>\n")
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK)
io.WriteString(w, b.String())
}
func (h *LocalHandler) ServePyPIPackageIndex(w http.ResponseWriter, r *http.Request, repoName, packageName string) {
normalized := pypiNormalize(packageName)
prefix := normalized + "/"
files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(files) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
normalized, filename, hash, filename)
}
b.WriteString("</body></html>\n")
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK)
io.WriteString(w, b.String())
}
func (h *LocalHandler) GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error) {
normalized := pypiNormalize(packageName)
prefix := normalized + "/"
files, err := h.db.ListLocalFilesByPrefix(ctx, repoName, prefix)
if err != nil {
return nil, err
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
normalized, filename, hash, filename)
}
b.WriteString("</body></html>\n")
return []byte(b.String()), nil
}
func (h *LocalHandler) download(w http.ResponseWriter, r *http.Request) {
repoName := chi.URLParam(r, "name")
filePath := chi.URLParam(r, "*")
@@ -394,74 +192,14 @@ func (h *LocalHandler) remove(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNoContent)
}
type terraformIndex struct {
Versions map[string]json.RawMessage `json:"versions"`
func (h *LocalHandler) DB() *database.DB {
return h.db
}
type terraformVersionDoc struct {
Archives map[string]terraformArchive `json:"archives"`
}
type terraformArchive struct {
URL string `json:"url"`
Hashes []string `json:"hashes,omitempty"`
}
func (h *LocalHandler) ServeTerraformIndex(w http.ResponseWriter, r *http.Request, repoName, namespace, typeName string) {
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix)
func (h *LocalHandler) Download(ctx context.Context, key string) (io.ReadCloser, int64, error) {
reader, info, err := h.store.Download(ctx, key)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
return nil, 0, err
}
versions := map[string]json.RawMessage{}
for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, prefix)
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
continue
}
versions[m[2]] = json.RawMessage(`{}`)
}
if len(versions) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
}
func (h *LocalHandler) ServeTerraformVersionDoc(w http.ResponseWriter, r *http.Request, repoName, namespace, typeName, version string) {
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
files, err := h.db.ListLocalFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
archives := map[string]terraformArchive{}
for _, f := range files {
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
m := providerZipRe.FindStringSubmatch(filename)
if m == nil || m[2] != version {
continue
}
platform := m[3] + "_" + m[4]
archive := terraformArchive{URL: filename}
if f.ContentHash != "" {
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
}
archives[platform] = archive
}
if len(archives) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
return reader, info.Size, nil
}
+18
View File
@@ -8,6 +8,8 @@ import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
type LocalFile struct {
@@ -122,6 +124,22 @@ func (db *DB) ListLocalFilePackages(ctx context.Context, repoName string) ([]str
return packages, rows.Err()
}
func (db *DB) ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]provider.FileEntry, error) {
files, err := db.ListLocalFilesByPrefix(ctx, repoName, prefix)
if err != nil {
return nil, err
}
result := make([]provider.FileEntry, len(files))
for i, f := range files {
result[i] = provider.FileEntry{FilePath: f.FilePath, ContentHash: f.ContentHash}
}
return result, nil
}
func (db *DB) ListPackages(ctx context.Context, repoName string) ([]string, error) {
return db.ListLocalFilePackages(ctx, repoName)
}
func (db *DB) DeleteLocalFile(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM local_files WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
+31
View File
@@ -124,6 +124,37 @@ func (db *DB) migrate() error {
CREATE INDEX IF NOT EXISTS idx_access_log_remote_time ON access_log(remote_name, created_at);
ALTER TABLE remotes ADD COLUMN IF NOT EXISTS repo_type TEXT DEFAULT 'remote';
CREATE TABLE IF NOT EXISTS rpm_metadata (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL,
name TEXT NOT NULL,
epoch INTEGER DEFAULT 0,
version TEXT NOT NULL,
release TEXT NOT NULL,
arch TEXT NOT NULL,
summary TEXT DEFAULT '',
description TEXT DEFAULT '',
rpm_size BIGINT DEFAULT 0,
installed_size BIGINT DEFAULT 0,
license TEXT DEFAULT '',
vendor TEXT DEFAULT '',
build_group TEXT DEFAULT '',
build_host TEXT DEFAULT '',
source_rpm TEXT DEFAULT '',
url TEXT DEFAULT '',
packager TEXT DEFAULT '',
requires JSONB DEFAULT '[]',
provides JSONB DEFAULT '[]',
files JSONB DEFAULT '[]',
changelogs JSONB DEFAULT '[]',
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE INDEX IF NOT EXISTS idx_rpm_metadata_repo ON rpm_metadata(repo_name);
`)
return err
}
+129
View File
@@ -0,0 +1,129 @@
package database
import (
"context"
"encoding/json"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertRPMMetadata(ctx context.Context, meta *provider.RPMMetadata) error {
requiresJSON, _ := json.Marshal(meta.Requires)
providesJSON, _ := json.Marshal(meta.Provides)
filesJSON, _ := json.Marshal(meta.Files)
changelogsJSON, _ := json.Marshal(meta.Changelogs)
_, err := db.Pool.Exec(ctx, `
INSERT INTO rpm_metadata (
repo_name, file_path, content_hash,
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23)
ON CONFLICT (repo_name, file_path) DO NOTHING
`,
meta.RepoName, meta.FilePath, meta.ContentHash,
meta.Name, meta.Epoch, meta.Version, meta.Release, meta.Arch,
meta.Summary, meta.Description, meta.RPMSize, meta.InstalledSize,
meta.License, meta.Vendor, meta.Group, meta.BuildHost, meta.SourceRPM, meta.URL, meta.Packager,
requiresJSON, providesJSON, filesJSON, changelogsJSON,
)
return err
}
type RPMMetadataRow struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires json.RawMessage
Provides json.RawMessage
Files json.RawMessage
Changelogs json.RawMessage
}
func (db *DB) ListRPMMetadataEntries(ctx context.Context, repoName string) ([]provider.RPMMetadata, error) {
rows, err := db.ListRPMMetadata(ctx, repoName)
if err != nil {
return nil, err
}
result := make([]provider.RPMMetadata, len(rows))
for i, r := range rows {
meta := provider.RPMMetadata{
RepoName: r.RepoName,
FilePath: r.FilePath,
ContentHash: r.ContentHash,
Name: r.Name,
Epoch: r.Epoch,
Version: r.Version,
Release: r.Release,
Arch: r.Arch,
Summary: r.Summary,
Description: r.Description,
RPMSize: r.RPMSize,
InstalledSize: r.InstalledSize,
License: r.License,
Vendor: r.Vendor,
Group: r.Group,
BuildHost: r.BuildHost,
SourceRPM: r.SourceRPM,
URL: r.URL,
Packager: r.Packager,
}
json.Unmarshal(r.Requires, &meta.Requires)
json.Unmarshal(r.Provides, &meta.Provides)
json.Unmarshal(r.Files, &meta.Files)
json.Unmarshal(r.Changelogs, &meta.Changelogs)
result[i] = meta
}
return result, nil
}
func (db *DB) ListRPMMetadata(ctx context.Context, repoName string) ([]RPMMetadataRow, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash,
name, epoch, version, release, arch,
summary, description, rpm_size, installed_size,
license, vendor, build_group, build_host, source_rpm, url, packager,
requires, provides, files, changelogs
FROM rpm_metadata
WHERE repo_name = $1
ORDER BY name, epoch, version, release, arch
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []RPMMetadataRow
for rows.Next() {
var r RPMMetadataRow
if err := rows.Scan(
&r.RepoName, &r.FilePath, &r.ContentHash,
&r.Name, &r.Epoch, &r.Version, &r.Release, &r.Arch,
&r.Summary, &r.Description, &r.RPMSize, &r.InstalledSize,
&r.License, &r.Vendor, &r.Group, &r.BuildHost, &r.SourceRPM, &r.URL, &r.Packager,
&r.Requires, &r.Provides, &r.Files, &r.Changelogs,
); err != nil {
return nil, err
}
result = append(result, r)
}
return result, rows.Err()
}
+82
View File
@@ -3,6 +3,7 @@ package provider
import (
"context"
"fmt"
"io"
"net/http"
"git.unkin.net/unkin/artifactapi/pkg/models"
@@ -24,6 +25,87 @@ type Provider interface {
AuthHeaders(ctx context.Context, remote models.Remote) (http.Header, error)
}
type FileEntry struct {
FilePath string
ContentHash string
}
type FileStore interface {
ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error)
ListPackages(ctx context.Context, repoName string) ([]string, error)
}
type LocalUploader interface {
ValidateUpload(filePath string) (storagePath, contentType string, err error)
UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any
}
type LocalIndexer interface {
ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool
GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error)
}
type BlobReader interface {
Download(ctx context.Context, key string) (io.ReadCloser, int64, error)
}
type PostUploadHook interface {
AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore)
}
type MetadataStore interface {
InsertRPMMetadata(ctx context.Context, meta *RPMMetadata) error
}
type RPMMetadataReader interface {
ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error)
}
type RPMMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Epoch int
Version string
Release string
Arch string
Summary string
Description string
RPMSize int64
InstalledSize int64
License string
Vendor string
Group string
BuildHost string
SourceRPM string
URL string
Packager string
Requires []RPMDep
Provides []RPMDep
Files []RPMFile
Changelogs []RPMChangelog
}
type RPMDep struct {
Name string `json:"name"`
Flags string `json:"flags,omitempty"`
Epoch string `json:"epoch,omitempty"`
Version string `json:"version,omitempty"`
Release string `json:"release,omitempty"`
}
type RPMFile struct {
Path string `json:"path"`
Type string `json:"type,omitempty"`
}
type RPMChangelog struct {
Author string `json:"author"`
Date int64 `json:"date"`
Text string `json:"text"`
}
type IndexMerger interface {
MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error)
}
+180
View File
@@ -2,7 +2,10 @@ package pypi
import (
"context"
"fmt"
"io"
"net/http"
"regexp"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
@@ -14,6 +17,9 @@ func init() {
provider.Register(&Provider{})
}
var fileRe = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9._-]*\.(whl|tar\.gz|zip)$`)
var normalizeRe = regexp.MustCompile(`[-_.]+`)
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackagePyPI }
@@ -60,3 +66,177 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
func normalize(name string) string {
return strings.ToLower(normalizeRe.ReplaceAllString(name, "-"))
}
func packageFromWheel(filename string) string {
parts := strings.SplitN(filename, "-", 3)
if len(parts) < 2 {
return ""
}
return normalize(parts[0])
}
func packageFromSdist(filename string) string {
name := filename
for _, suffix := range []string{".tar.gz", ".zip"} {
if strings.HasSuffix(name, suffix) {
name = strings.TrimSuffix(name, suffix)
break
}
}
idx := strings.LastIndex(name, "-")
if idx <= 0 {
return ""
}
return normalize(name[:idx])
}
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := filePath
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
filename = filePath[idx+1:]
}
if !fileRe.MatchString(filename) {
return "", "", fmt.Errorf("filename %q must be a .whl, .tar.gz, or .zip file", filename)
}
var pkgName string
if strings.HasSuffix(filename, ".whl") {
pkgName = packageFromWheel(filename)
} else {
pkgName = packageFromSdist(filename)
}
if pkgName == "" {
return "", "", fmt.Errorf("cannot parse package name from %q", filename)
}
ct := "application/zip"
if strings.HasSuffix(filename, ".tar.gz") {
ct = "application/gzip"
}
return pkgName + "/" + filename, ct, nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
parts := strings.SplitN(storagePath, "/", 2)
filename := storagePath
if len(parts) == 2 {
filename = parts[1]
}
return map[string]any{
"package": parts[0],
"filename": filename,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
if path == "simple" || path == "simple/" {
p.servePackageList(w, r, files, repoName)
return true
}
if strings.HasPrefix(path, "simple/") {
pkg := strings.TrimPrefix(path, "simple/")
pkg = strings.TrimSuffix(pkg, "/")
if pkg != "" && !strings.Contains(pkg, "/") {
p.servePackageFiles(w, r, files, repoName, pkg)
return true
}
}
return false
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
if !strings.HasPrefix(path, "simple/") {
return nil, fmt.Errorf("unsupported index path: %q", path)
}
pkg := strings.TrimPrefix(path, "simple/")
pkg = strings.TrimSuffix(pkg, "/")
if pkg == "" {
return p.generatePackageListHTML(ctx, files, repoName)
}
return p.generatePackageFilesHTML(ctx, files, repoName, pkg)
}
func (p *Provider) servePackageList(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName string) {
body, err := p.generatePackageListHTML(r.Context(), files, repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK)
w.Write(body)
}
func (p *Provider) servePackageFiles(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, packageName string) {
normalized := normalize(packageName)
prefix := normalized + "/"
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(entries) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"../../%s/%s#sha256=%s\">%s</a>\n",
normalized, filename, hash, filename)
}
b.WriteString("</body></html>\n")
w.Header().Set("Content-Type", "text/html")
w.WriteHeader(http.StatusOK)
io.WriteString(w, b.String())
}
func (p *Provider) generatePackageListHTML(ctx context.Context, files provider.FileStore, repoName string) ([]byte, error) {
packages, err := files.ListPackages(ctx, repoName)
if err != nil {
return nil, err
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, pkg := range packages {
fmt.Fprintf(&b, "<a href=\"%s/\">%s</a>\n", pkg, pkg)
}
b.WriteString("</body></html>\n")
return []byte(b.String()), nil
}
func (p *Provider) generatePackageFilesHTML(ctx context.Context, files provider.FileStore, repoName, packageName string) ([]byte, error) {
normalized := normalize(packageName)
prefix := normalized + "/"
entries, err := files.ListFilesByPrefix(ctx, repoName, prefix)
if err != nil {
return nil, err
}
var b strings.Builder
b.WriteString("<!DOCTYPE html>\n<html><body>\n")
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, normalized+"/")
hash := strings.TrimPrefix(f.ContentHash, "sha256:")
fmt.Fprintf(&b, "<a href=\"%s/%s#sha256=%s\">%s</a>\n",
normalized, filename, hash, filename)
}
b.WriteString("</body></html>\n")
return []byte(b.String()), nil
}
+387
View File
@@ -1,13 +1,24 @@
package rpm
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/xml"
"fmt"
"log/slog"
"net/http"
"regexp"
"strings"
"time"
rpmlib "github.com/cavaliergopher/rpm"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -55,3 +66,379 @@ func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte,
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := filePath
if idx := strings.LastIndex(filePath, "/"); idx >= 0 {
filename = filePath[idx+1:]
}
if !strings.HasSuffix(strings.ToLower(filename), ".rpm") {
return "", "", fmt.Errorf("file must be an .rpm package")
}
return "Packages/" + filename, "application/x-rpm", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
filename := strings.TrimPrefix(storagePath, "Packages/")
return map[string]any{
"filename": filename,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
s3Key := storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))
reader, blobSize, err := blobs.Download(ctx, s3Key)
if err != nil {
slog.Error("rpm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer reader.Close()
pkg, err := rpmlib.Read(reader)
if err != nil {
slog.Error("rpm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
meta := &provider.RPMMetadata{
RepoName: repoName,
FilePath: storagePath,
ContentHash: contentHash,
Name: pkg.Name(),
Epoch: pkg.Epoch(),
Version: pkg.Version(),
Release: pkg.Release(),
Arch: pkg.Architecture(),
Summary: pkg.Summary(),
Description: pkg.Description(),
RPMSize: blobSize,
InstalledSize: int64(pkg.Size()),
License: pkg.License(),
Vendor: pkg.Vendor(),
Group: firstGroup(pkg.Groups()),
BuildHost: pkg.BuildHost(),
SourceRPM: pkg.SourceRPM(),
URL: pkg.URL(),
Packager: pkg.Packager(),
}
for _, req := range pkg.Requires() {
meta.Requires = append(meta.Requires, rpmDepFromEntry(req))
}
for _, prov := range pkg.Provides() {
meta.Provides = append(meta.Provides, rpmDepFromEntry(prov))
}
if meta.Requires == nil {
meta.Requires = []provider.RPMDep{}
}
if meta.Provides == nil {
meta.Provides = []provider.RPMDep{}
}
meta.Files = []provider.RPMFile{}
meta.Changelogs = []provider.RPMChangelog{}
if err := db.InsertRPMMetadata(ctx, meta); err != nil {
slog.Error("rpm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("rpm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
}
func rpmDepFromEntry(e rpmlib.Dependency) provider.RPMDep {
dep := provider.RPMDep{Name: e.Name()}
if e.Flags() != 0 {
dep.Flags = rpmFlagString(e.Flags())
dep.Version = e.Version()
dep.Release = e.Release()
if e.Epoch() > 0 {
dep.Epoch = fmt.Sprintf("%d", e.Epoch())
}
}
return dep
}
func rpmFlagString(f int) string {
switch {
case f&0x08 != 0 && f&0x04 != 0:
return "GE"
case f&0x02 != 0 && f&0x04 != 0:
return "LE"
case f&0x08 != 0:
return "GT"
case f&0x02 != 0:
return "LT"
case f&0x04 != 0:
return "EQ"
default:
return ""
}
}
func firstGroup(groups []string) string {
if len(groups) > 0 {
return groups[0]
}
return "Unspecified"
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
if !strings.HasPrefix(path, "repodata/") {
return false
}
rpmReader, ok := files.(provider.RPMMetadataReader)
if !ok {
http.Error(w, "rpm metadata not available", http.StatusInternalServerError)
return true
}
tail := strings.TrimPrefix(path, "repodata/")
switch {
case tail == "repomd.xml":
p.serveRepomd(w, r, rpmReader, repoName)
case strings.HasSuffix(tail, "-primary.xml.gz"):
p.servePrimary(w, r, rpmReader, repoName)
case strings.HasSuffix(tail, "-filelists.xml.gz"):
p.serveFilelists(w, r, rpmReader, repoName)
case strings.HasSuffix(tail, "-other.xml.gz"):
p.serveOther(w, r, rpmReader, repoName)
default:
http.Error(w, "not found", http.StatusNotFound)
}
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("rpm local index generation for virtual repos not supported")
}
func (p *Provider) serveRepomd(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
primary := generatePrimaryXMLGZ(metas)
filelists := generateFilelistsXMLGZ(metas)
other := generateOtherXMLGZ(metas)
primaryHash := sha256Hex(primary)
filelistsHash := sha256Hex(filelists)
otherHash := sha256Hex(other)
repomd := generateRepomd(primaryHash, len(primary), filelistsHash, len(filelists), otherHash, len(other))
w.Header().Set("Content-Type", "application/xml")
w.WriteHeader(http.StatusOK)
w.Write(repomd)
}
func (p *Provider) servePrimary(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(generatePrimaryXMLGZ(metas))
}
func (p *Provider) serveFilelists(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(generateFilelistsXMLGZ(metas))
}
func (p *Provider) serveOther(w http.ResponseWriter, r *http.Request, reader provider.RPMMetadataReader, repoName string) {
metas, err := reader.ListRPMMetadataEntries(r.Context(), repoName)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/gzip")
w.WriteHeader(http.StatusOK)
w.Write(generateOtherXMLGZ(metas))
}
func generateRepomd(primaryHash string, primarySize int, filelistsHash string, filelistsSize int, otherHash string, otherSize int) []byte {
ts := fmt.Sprintf("%d", time.Now().Unix())
var b bytes.Buffer
b.WriteString(xml.Header)
b.WriteString(`<repomd xmlns="http://linux.duke.edu/metadata/repo" xmlns:rpm="http://linux.duke.edu/metadata/rpm">` + "\n")
fmt.Fprintf(&b, " <revision>%s</revision>\n", ts)
writeRepomdData(&b, "primary", primaryHash, primarySize, ts)
writeRepomdData(&b, "filelists", filelistsHash, filelistsSize, ts)
writeRepomdData(&b, "other", otherHash, otherSize, ts)
b.WriteString("</repomd>\n")
return b.Bytes()
}
func writeRepomdData(b *bytes.Buffer, dtype, hash string, size int, ts string) {
fmt.Fprintf(b, " <data type=\"%s\">\n", dtype)
fmt.Fprintf(b, " <checksum type=\"sha256\">%s</checksum>\n", hash)
fmt.Fprintf(b, " <location href=\"repodata/%s-%s.xml.gz\"/>\n", hash, dtype)
fmt.Fprintf(b, " <timestamp>%s</timestamp>\n", ts)
fmt.Fprintf(b, " <size>%d</size>\n", size)
fmt.Fprintf(b, " </data>\n")
}
func generatePrimaryXMLGZ(metas []provider.RPMMetadata) []byte {
var xmlBuf bytes.Buffer
xmlBuf.WriteString(xml.Header)
fmt.Fprintf(&xmlBuf, "<metadata xmlns=\"http://linux.duke.edu/metadata/common\" xmlns:rpm=\"http://linux.duke.edu/metadata/rpm\" packages=\"%d\">\n", len(metas))
for _, m := range metas {
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
fmt.Fprintf(&xmlBuf, "<package type=\"rpm\">\n")
fmt.Fprintf(&xmlBuf, " <name>%s</name>\n", xmlEscape(m.Name))
fmt.Fprintf(&xmlBuf, " <arch>%s</arch>\n", xmlEscape(m.Arch))
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
fmt.Fprintf(&xmlBuf, " <checksum type=\"sha256\" pkgid=\"YES\">%s</checksum>\n", pkgHash)
fmt.Fprintf(&xmlBuf, " <summary>%s</summary>\n", xmlEscape(m.Summary))
fmt.Fprintf(&xmlBuf, " <description>%s</description>\n", xmlEscape(m.Description))
if m.Packager != "" {
fmt.Fprintf(&xmlBuf, " <packager>%s</packager>\n", xmlEscape(m.Packager))
}
if m.URL != "" {
fmt.Fprintf(&xmlBuf, " <url>%s</url>\n", xmlEscape(m.URL))
}
fmt.Fprintf(&xmlBuf, " <time file=\"%d\" build=\"0\"/>\n", time.Now().Unix())
fmt.Fprintf(&xmlBuf, " <size package=\"%d\" installed=\"%d\" archive=\"0\"/>\n", m.RPMSize, m.InstalledSize)
fmt.Fprintf(&xmlBuf, " <location href=\"%s\"/>\n", xmlEscape(m.FilePath))
fmt.Fprintf(&xmlBuf, " <format>\n")
if m.License != "" {
fmt.Fprintf(&xmlBuf, " <rpm:license>%s</rpm:license>\n", xmlEscape(m.License))
}
if m.Vendor != "" {
fmt.Fprintf(&xmlBuf, " <rpm:vendor>%s</rpm:vendor>\n", xmlEscape(m.Vendor))
}
fmt.Fprintf(&xmlBuf, " <rpm:group>%s</rpm:group>\n", xmlEscape(m.Group))
if m.BuildHost != "" {
fmt.Fprintf(&xmlBuf, " <rpm:buildhost>%s</rpm:buildhost>\n", xmlEscape(m.BuildHost))
}
if m.SourceRPM != "" {
fmt.Fprintf(&xmlBuf, " <rpm:sourcerpm>%s</rpm:sourcerpm>\n", xmlEscape(m.SourceRPM))
}
if len(m.Provides) > 0 {
xmlBuf.WriteString(" <rpm:provides>\n")
for _, d := range m.Provides {
writeRPMEntry(&xmlBuf, d)
}
xmlBuf.WriteString(" </rpm:provides>\n")
}
if len(m.Requires) > 0 {
xmlBuf.WriteString(" <rpm:requires>\n")
for _, d := range m.Requires {
writeRPMEntry(&xmlBuf, d)
}
xmlBuf.WriteString(" </rpm:requires>\n")
}
fmt.Fprintf(&xmlBuf, " </format>\n")
fmt.Fprintf(&xmlBuf, "</package>\n")
}
xmlBuf.WriteString("</metadata>\n")
return gzipBytes(xmlBuf.Bytes())
}
func generateFilelistsXMLGZ(metas []provider.RPMMetadata) []byte {
var xmlBuf bytes.Buffer
xmlBuf.WriteString(xml.Header)
fmt.Fprintf(&xmlBuf, "<filelists xmlns=\"http://linux.duke.edu/metadata/filelists\" packages=\"%d\">\n", len(metas))
for _, m := range metas {
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
for _, f := range m.Files {
if f.Type != "" {
fmt.Fprintf(&xmlBuf, " <file type=\"%s\">%s</file>\n", f.Type, xmlEscape(f.Path))
} else {
fmt.Fprintf(&xmlBuf, " <file>%s</file>\n", xmlEscape(f.Path))
}
}
xmlBuf.WriteString("</package>\n")
}
xmlBuf.WriteString("</filelists>\n")
return gzipBytes(xmlBuf.Bytes())
}
func generateOtherXMLGZ(metas []provider.RPMMetadata) []byte {
var xmlBuf bytes.Buffer
xmlBuf.WriteString(xml.Header)
fmt.Fprintf(&xmlBuf, "<otherdata xmlns=\"http://linux.duke.edu/metadata/other\" packages=\"%d\">\n", len(metas))
for _, m := range metas {
pkgHash := strings.TrimPrefix(m.ContentHash, "sha256:")
fmt.Fprintf(&xmlBuf, "<package pkgid=\"%s\" name=\"%s\" arch=\"%s\">\n", pkgHash, xmlEscape(m.Name), xmlEscape(m.Arch))
fmt.Fprintf(&xmlBuf, " <version epoch=\"%d\" ver=\"%s\" rel=\"%s\"/>\n", m.Epoch, xmlEscape(m.Version), xmlEscape(m.Release))
for _, cl := range m.Changelogs {
fmt.Fprintf(&xmlBuf, " <changelog author=\"%s\" date=\"%d\">%s</changelog>\n",
xmlEscape(cl.Author), cl.Date, xmlEscape(cl.Text))
}
xmlBuf.WriteString("</package>\n")
}
xmlBuf.WriteString("</otherdata>\n")
return gzipBytes(xmlBuf.Bytes())
}
func writeRPMEntry(b *bytes.Buffer, d provider.RPMDep) {
if d.Flags != "" {
fmt.Fprintf(b, " <rpm:entry name=\"%s\" flags=\"%s\"", xmlEscape(d.Name), d.Flags)
if d.Epoch != "" {
fmt.Fprintf(b, " epoch=\"%s\"", d.Epoch)
}
if d.Version != "" {
fmt.Fprintf(b, " ver=\"%s\"", xmlEscape(d.Version))
}
if d.Release != "" {
fmt.Fprintf(b, " rel=\"%s\"", xmlEscape(d.Release))
}
b.WriteString("/>\n")
} else {
fmt.Fprintf(b, " <rpm:entry name=\"%s\"/>\n", xmlEscape(d.Name))
}
}
func xmlEscape(s string) string {
var b bytes.Buffer
xml.EscapeText(&b, []byte(s))
return b.String()
}
func gzipBytes(data []byte) []byte {
var buf bytes.Buffer
gz := gzip.NewWriter(&buf)
gz.Write(data)
gz.Close()
return buf.Bytes()
}
func sha256Hex(data []byte) string {
h := sha256.Sum256(data)
return hex.EncodeToString(h[:])
}
+149
View File
@@ -3,6 +3,7 @@ package terraform
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
@@ -19,6 +20,12 @@ func init() {
var versionsRe = regexp.MustCompile(`[^/]+/[^/]+/versions$`)
var providerZipRe = regexp.MustCompile(
`^terraform-provider-([a-zA-Z0-9_-]+)_([0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?)_([a-z0-9]+)_([a-z0-9]+)\.zip$`,
)
var semverRe = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(?:-[a-zA-Z0-9.]+)?$`)
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageTerraform }
@@ -86,3 +93,145 @@ func rewriteDownloadURL(originalURL, releasesRemote, proxyBaseURL string) string
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
parts := strings.Split(filePath, "/")
if len(parts) != 3 {
return "", "", fmt.Errorf("path must be {namespace}/{type}/{filename}.zip")
}
namespace, typeName, filename := parts[0], parts[1], parts[2]
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
return "", "", fmt.Errorf("filename %q does not match terraform-provider-{type}_{version}_{os}_{arch}.zip", filename)
}
if m[1] != typeName {
return "", "", fmt.Errorf("provider type in filename %q does not match path type %q", m[1], typeName)
}
return fmt.Sprintf("%s/%s/%s", namespace, typeName, filename), "application/zip", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
parts := strings.Split(storagePath, "/")
if len(parts) != 3 {
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
}
m := providerZipRe.FindStringSubmatch(parts[2])
if m == nil {
return map[string]any{"path": storagePath, "content_hash": contentHash, "size_bytes": sizeBytes}
}
return map[string]any{
"namespace": parts[0],
"type": parts[1],
"version": m[2],
"os": m[3],
"arch": m[4],
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
type terraformIndex struct {
Versions map[string]json.RawMessage `json:"versions"`
}
type terraformVersionDoc struct {
Archives map[string]terraformArchive `json:"archives"`
}
type terraformArchive struct {
URL string `json:"url"`
Hashes []string `json:"hashes,omitempty"`
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
parts := strings.Split(path, "/")
if len(parts) < 3 {
return false
}
namespace, typeName := parts[0], parts[1]
tail := parts[2]
if tail == "index.json" {
p.serveIndex(w, r, files, repoName, namespace, typeName)
return true
}
if strings.HasSuffix(tail, ".json") {
version := strings.TrimSuffix(tail, ".json")
if semverRe.MatchString(version) {
p.serveVersionDoc(w, r, files, repoName, namespace, typeName, version)
return true
}
}
return false
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("terraform local index generation for virtual repos not supported")
}
func (p *Provider) serveIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName string) {
prefix := fmt.Sprintf("%s/%s/", namespace, typeName)
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
versions := map[string]json.RawMessage{}
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, prefix)
m := providerZipRe.FindStringSubmatch(filename)
if m == nil {
continue
}
versions[m[2]] = json.RawMessage(`{}`)
}
if len(versions) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformIndex{Versions: versions})
}
func (p *Provider) serveVersionDoc(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, namespace, typeName, version string) {
prefix := fmt.Sprintf("%s/%s/terraform-provider-%s_%s_", namespace, typeName, typeName, version)
entries, err := files.ListFilesByPrefix(r.Context(), repoName, prefix)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
archives := map[string]terraformArchive{}
for _, f := range entries {
filename := strings.TrimPrefix(f.FilePath, fmt.Sprintf("%s/%s/", namespace, typeName))
m := providerZipRe.FindStringSubmatch(filename)
if m == nil || m[2] != version {
continue
}
platform := m[3] + "_" + m[4]
archive := terraformArchive{URL: filename}
if f.ContentHash != "" {
archive.Hashes = []string{"zh:" + strings.TrimPrefix(f.ContentHash, "sha256:")}
}
archives[platform] = archive
}
if len(archives) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return
}
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(terraformVersionDoc{Archives: archives})
}
+1 -1
View File
@@ -63,7 +63,7 @@ func New(cfg *config.Config) (*Server, error) {
engine := proxy.NewEngine(db, redis, s3)
localHandler := v2.NewLocalHandler(db, s3)
virtEngine := virtual.NewEngine(db, engine, localHandler)
virtEngine := virtual.NewEngine(db, engine)
collector := gc.New(db, s3, 1*time.Hour)
s := &Server{
+14 -23
View File
@@ -5,7 +5,6 @@ import (
"fmt"
"io"
"log/slog"
"strings"
"sync"
"git.unkin.net/unkin/artifactapi/internal/database"
@@ -14,18 +13,13 @@ import (
"git.unkin.net/unkin/artifactapi/pkg/models"
)
type LocalIndexGenerator interface {
GeneratePyPIPackageHTML(ctx context.Context, repoName, packageName string) ([]byte, error)
}
type Engine struct {
db *database.DB
proxyEngine *proxy.Engine
localGen LocalIndexGenerator
}
func NewEngine(db *database.DB, proxyEngine *proxy.Engine, localGen LocalIndexGenerator) *Engine {
return &Engine{db: db, proxyEngine: proxyEngine, localGen: localGen}
func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine {
return &Engine{db: db, proxyEngine: proxyEngine}
}
func (e *Engine) Fetch(ctx context.Context, virt models.Virtual, path string, proxyBaseURL string) ([]byte, string, error) {
@@ -80,7 +74,7 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
}
if remote.RepoType == models.RepoTypeLocal {
body, err := e.fetchLocalIndex(ctx, *remote, virt.PackageType, path)
body, err := e.fetchLocalIndex(ctx, *remote, path)
if err != nil {
results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)}
return
@@ -126,19 +120,16 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
return members, nil
}
func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, packageType models.PackageType, path string) ([]byte, error) {
switch packageType {
case models.PackagePyPI:
if e.localGen == nil {
return nil, fmt.Errorf("no local index generator configured")
}
parts := strings.SplitN(strings.TrimPrefix(path, "simple/"), "/", 2)
pkgName := strings.TrimSuffix(parts[0], "/")
if pkgName == "" {
return nil, fmt.Errorf("cannot determine package name from path %q", path)
}
return e.localGen.GeneratePyPIPackageHTML(ctx, remote.Name, pkgName)
default:
return nil, fmt.Errorf("local index generation not supported for %q", packageType)
func (e *Engine) fetchLocalIndex(ctx context.Context, remote models.Remote, path string) ([]byte, error) {
prov, err := provider.Get(remote.PackageType)
if err != nil {
return nil, fmt.Errorf("no provider for %q: %w", remote.PackageType, err)
}
indexer, ok := prov.(provider.LocalIndexer)
if !ok {
return nil, fmt.Errorf("provider %q does not support local index generation", remote.PackageType)
}
return indexer.GenerateLocalIndex(ctx, e.db, remote.Name, path)
}
+6
View File
@@ -2,6 +2,8 @@ import { Routes, Route, NavLink } from 'react-router-dom';
import { Dashboard } from './pages/Dashboard';
import { Remotes } from './pages/Remotes';
import { RemoteDetail } from './pages/RemoteDetail';
import { Locals } from './pages/Locals';
import { LocalDetail } from './pages/LocalDetail';
import { Virtuals } from './pages/Virtuals';
import { Objects } from './pages/Objects';
import { Probe } from './pages/Probe';
@@ -18,6 +20,7 @@ export function App() {
<div className="sidebar-nav">
<NavLink to="/" end>Dashboard</NavLink>
<NavLink to="/remotes">Remotes</NavLink>
<NavLink to="/locals">Locals</NavLink>
<NavLink to="/virtuals">Virtuals</NavLink>
<NavLink to="/probe">Test Remote</NavLink>
</div>
@@ -31,6 +34,9 @@ export function App() {
<Route path="/remotes" element={<Remotes />} />
<Route path="/remotes/:name" element={<RemoteDetail />} />
<Route path="/remotes/:name/objects" element={<Objects />} />
<Route path="/locals" element={<Locals />} />
<Route path="/locals/:name" element={<LocalDetail />} />
<Route path="/locals/:name/objects" element={<Objects />} />
<Route path="/virtuals" element={<Virtuals />} />
<Route path="/probe" element={<Probe />} />
</Routes>
+46
View File
@@ -0,0 +1,46 @@
import { useEffect, useState } from 'react';
import { useParams, Link } from 'react-router-dom';
import { api } from '../api/client';
import type { Remote } from '../api/types';
import { Badge } from '../components/Badge';
import './RemoteDetail.css';
export function LocalDetail() {
const { name } = useParams<{ name: string }>();
const [remote, setRemote] = useState<Remote | null>(null);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
if (!name) return;
api.getRemote(name)
.then(setRemote)
.catch(e => setError(e.message));
}, [name]);
if (error) return <div className="error-banner">{error}</div>;
if (!remote) return <div className="loading">Loading...</div>;
return (
<div>
<div className="detail-header">
<Link to="/locals" className="back-link">&larr; Locals</Link>
<h1 className="page-title">{remote.name}</h1>
<div className="detail-badges">
<Badge variant="blue">{remote.package_type}</Badge>
<Badge variant="default">local</Badge>
{remote.managed_by && <Badge variant="green">managed by {remote.managed_by}</Badge>}
</div>
</div>
{remote.description && (
<p className="detail-description">{remote.description}</p>
)}
<div className="detail-actions">
<Link to={`/locals/${remote.name}/objects`} className="btn btn-primary">
Browse Files
</Link>
</div>
</div>
);
}
+93
View File
@@ -0,0 +1,93 @@
import { useEffect, useState } from 'react';
import { useNavigate } from 'react-router-dom';
import { api } from '../api/client';
import type { Remote } from '../api/types';
import { Badge } from '../components/Badge';
import { DataTable } from '../components/DataTable';
import './Remotes.css';
const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'> = {
docker: 'blue',
helm: 'green',
rpm: 'yellow',
pypi: 'blue',
npm: 'red',
generic: 'default',
alpine: 'green',
puppet: 'yellow',
terraform: 'blue',
goproxy: 'green',
};
export function Locals() {
const navigate = useNavigate();
const [remotes, setRemotes] = useState<Remote[]>([]);
const [filter, setFilter] = useState('');
const [loading, setLoading] = useState(true);
useEffect(() => {
api.listRemotes()
.then(r => setRemotes((r || []).filter(x => x.repo_type === 'local')))
.finally(() => setLoading(false));
}, []);
const filtered = remotes.filter(r => {
if (filter && !r.name.toLowerCase().includes(filter.toLowerCase())) return false;
return true;
});
return (
<div>
<h1 className="page-title">Local Repositories</h1>
<div className="remotes-toolbar">
<input
className="search-input"
placeholder="Filter by name..."
value={filter}
onChange={e => setFilter(e.target.value)}
/>
<span className="result-count">{filtered.length} locals</span>
</div>
{loading ? (
<div className="loading">Loading...</div>
) : (
<DataTable
columns={[
{
key: 'name',
header: 'Name',
render: (r: Remote) => <span className="mono">{r.name}</span>,
},
{
key: 'type',
header: 'Type',
render: (r: Remote) => (
<Badge variant={typeColors[r.package_type] || 'default'}>
{r.package_type}
</Badge>
),
width: '110px',
},
{
key: 'description',
header: 'Description',
render: (r: Remote) => r.description || <span className="text-muted"></span>,
},
{
key: 'managed',
header: 'Managed',
render: (r: Remote) =>
r.managed_by ? <Badge variant="blue">{r.managed_by}</Badge> : <span className="text-muted"></span>,
width: '100px',
},
]}
data={filtered}
emptyMessage="No local repositories configured"
onRowClick={(r) => navigate(`/locals/${r.name}`)}
/>
)}
</div>
);
}
+5 -2
View File
@@ -1,5 +1,5 @@
import { useEffect, useState, useCallback, useMemo } from 'react';
import { useParams, Link } from 'react-router-dom';
import { useParams, useLocation, Link } from 'react-router-dom';
import { api } from '../api/client';
import type { Artifact } from '../api/types';
import { formatBytes, timeAgo, truncateHash } from '../components/format';
@@ -171,6 +171,9 @@ function TreeRow({ node, depth, expanded, onToggle, onEvict }: TreeRowProps) {
export function Objects() {
const { name } = useParams<{ name: string }>();
const location = useLocation();
const isLocal = location.pathname.startsWith('/locals/');
const backLink = isLocal ? `/locals/${name}` : `/remotes/${name}`;
const [artifacts, setArtifacts] = useState<Artifact[]>([]);
const [loading, setLoading] = useState(true);
const [filter, setFilter] = useState('');
@@ -233,7 +236,7 @@ export function Objects() {
return (
<div>
<div className="detail-header">
<Link to={`/remotes/${name}`} className="back-link">&larr; {name}</Link>
<Link to={backLink} className="back-link">&larr; {name}</Link>
<h1 className="page-title">Cached Objects</h1>
</div>
+3 -2
View File
@@ -32,9 +32,10 @@ export function Remotes() {
.finally(() => setLoading(false));
}, []);
const types = [...new Set(remotes.map(r => r.package_type))].sort();
const remoteOnly = remotes.filter(r => r.repo_type !== 'local');
const types = [...new Set(remoteOnly.map(r => r.package_type))].sort();
const filtered = remotes.filter(r => {
const filtered = remoteOnly.filter(r => {
if (typeFilter && r.package_type !== typeFilter) return false;
if (filter && !r.name.toLowerCase().includes(filter.toLowerCase())) return false;
return true;
+32 -10
View File
@@ -1,21 +1,38 @@
import { useEffect, useState } from 'react';
import { Link } from 'react-router-dom';
import { api } from '../api/client';
import type { Virtual } from '../api/types';
import type { Remote, Virtual } from '../api/types';
import { Badge } from '../components/Badge';
import { DataTable } from '../components/DataTable';
import './Virtuals.css';
export function Virtuals() {
const [virtuals, setVirtuals] = useState<Virtual[]>([]);
const [remoteMap, setRemoteMap] = useState<Record<string, Remote>>({});
const [loading, setLoading] = useState(true);
const [expanded, setExpanded] = useState<string | null>(null);
useEffect(() => {
api.listVirtuals()
.then(v => setVirtuals(v || []))
Promise.all([api.listVirtuals(), api.listRemotes()])
.then(([v, r]) => {
setVirtuals(v || []);
const map: Record<string, Remote> = {};
for (const remote of r || []) {
map[remote.name] = remote;
}
setRemoteMap(map);
})
.finally(() => setLoading(false));
}, []);
function memberLink(name: string) {
const remote = remoteMap[name];
if (remote?.repo_type === 'local') {
return `/locals/${name}`;
}
return `/remotes/${name}`;
}
return (
<div>
<h1 className="page-title">Virtual Repositories</h1>
@@ -40,7 +57,7 @@ export function Virtuals() {
key: 'members',
header: 'Members',
render: (v: Virtual) => (
<span className="member-count">{v.members?.length || 0} remotes</span>
<span className="member-count">{v.members?.length || 0} repos</span>
),
width: '110px',
},
@@ -69,12 +86,17 @@ export function Virtuals() {
<ul className="member-list">
{virtuals
.find(v => v.name === expanded)
?.members?.map((m, i) => (
<li key={m}>
<span className="member-priority">{i + 1}</span>
<a href={`/remotes/${m}`} className="mono">{m}</a>
</li>
))}
?.members?.map((m, i) => {
const remote = remoteMap[m];
const typeLabel = remote?.repo_type === 'local' ? 'local' : 'remote';
return (
<li key={m}>
<span className="member-priority">{i + 1}</span>
<Link to={memberLink(m)} className="mono">{m}</Link>
<Badge variant={typeLabel === 'local' ? 'yellow' : 'default'}>{typeLabel}</Badge>
</li>
);
})}
</ul>
</div>
)}