From 9b91aa39a60a5b7bc5bd83747bbaa50d3e26e61c Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 13:17:51 +1100 Subject: [PATCH 1/4] Add local puppet forge (#145) Puppet modules can only be proxied from a remote Forge; there is nowhere to publish internal modules for puppet module install, r10k and g10k. - add puppet local uploads at --.tar.gz, rejecting existing versions - record each release's metadata.json, md5 and sha256 in a puppet_metadata table, streaming the tarball under size caps - serve Forge v3 /v3/modules, paginated /v3/releases, /v3/releases/ and /v3/files - add an e2e-docker case driving real r10k and puppet module install - document local puppet in README and UI usage snippets Reviewed-on: https://git.unkin.net/unkin/artifactapi/pulls/145 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- README.md | 15 + e2e-docker/README.md | 3 +- e2e-docker/puppet_local_test.go | 88 ++++++ internal/database/puppet_metadata.go | 46 +++ internal/provider/provider.go | 26 ++ internal/provider/puppet/local.go | 393 ++++++++++++++++++++++++ internal/provider/puppet/local_test.go | 301 ++++++++++++++++++ migrations/0004_puppet_metadata.sql | 15 + ui/src/components/UsageInstructions.tsx | 14 +- 9 files changed, 898 insertions(+), 3 deletions(-) create mode 100644 e2e-docker/puppet_local_test.go create mode 100644 internal/database/puppet_metadata.go create mode 100644 internal/provider/puppet/local.go create mode 100644 internal/provider/puppet/local_test.go create mode 100644 migrations/0004_puppet_metadata.sql diff --git a/README.md b/README.md index 5b4a4e9..2d76346 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,21 @@ there's nothing to provision. To bring your own key instead, point `TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. `TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. +### Local puppet forge + +A local `puppet` repo is a Forge: `PUT` a module tarball to +`/api/v2/remotes//files/--.tar.gz` and the Forge +v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each +release's `metadata.json` at `/api/v1/local/`. Versions are immutable. + +```bash +puppet module build ./hello +curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ + https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz +puppet module install unkin-hello \ + --module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ +``` + ### Local docker registry A local `docker` repo is a real container registry, not a mirror: it serves the diff --git a/e2e-docker/README.md b/e2e-docker/README.md index ec2ee0c..facd807 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -26,7 +26,8 @@ already-running stack. helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. -- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` +- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`), + generic (upload/download), pypi (wheel + generated `simple/` index), rpm (real package + **automatic repodata** generation). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members. diff --git a/e2e-docker/puppet_local_test.go b/e2e-docker/puppet_local_test.go new file mode 100644 index 0000000..9c5acaa --- /dev/null +++ b/e2e-docker/puppet_local_test.go @@ -0,0 +1,88 @@ +//go:build dockere2e + +package e2edocker + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}` + +func puppetModuleTarball(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range map[string]string{ + "unkin-hello-0.1.0/metadata.json": puppetHelloMetadata, + "unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n", + } { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(body)) + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +// TestLocalPuppetRealClient publishes a module to a puppet local and installs +// it with the real r10k and puppet module tool. The stdlib dependency is not on +// a local-only forge, so puppet runs with --ignore-dependencies. +func TestLocalPuppetRealClient(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`) + defer deleteRepo(t, "local-puppet") + + upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz") + defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "") + tarball := puppetModuleTarball(t) + if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated { + t.Fatalf("upload: %d %s", resp.StatusCode, body) + } + if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict { + t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode) + } + if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) { + t.Fatalf("release never indexed: %d %s", resp.StatusCode, body) + } + + forge := internal + "/api/v1/local/local-puppet" + run := func(image, script string) string { + t.Helper() + out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput() + if err != nil { + t.Fatalf("%s failed: %v\n%s", image, err, out) + } + return string(out) + } + + out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+ + "printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+ + "r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json") + if !strings.Contains(out, `"unkin-hello"`) { + t.Fatalf("r10k did not install unkin-hello:\n%s", out) + } + + out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods") + if !strings.Contains(out, "unkin-hello (v0.1.0)") { + t.Fatalf("puppet module install did not install unkin-hello:\n%s", out) + } +} diff --git a/internal/database/puppet_metadata.go b/internal/database/puppet_metadata.go new file mode 100644 index 0000000..036c35d --- /dev/null +++ b/internal/database/puppet_metadata.go @@ -0,0 +1,46 @@ +package database + +import ( + "context" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) + ON CONFLICT (repo_name, file_path) DO NOTHING + `, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5) + return err +} + +func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} + +// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to +// one module slug (empty slug lists every module). +func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at + FROM puppet_metadata + WHERE repo_name = $1 AND ($2 = '' OR slug = $2) + ORDER BY slug, file_path + `, repoName, slug) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.PuppetMetadata + for rows.Next() { + var m provider.PuppetMetadata + if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil { + return nil, err + } + result = append(result, m) + } + return result, rows.Err() +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 4c005f3..aeecde8 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -247,3 +247,29 @@ func Get(t models.PackageType) (Provider, error) { func All() map[models.PackageType]Provider { return registry } + +type PuppetMetadataStore interface { + InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error +} + +type PuppetMetadataDeleter interface { + DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error +} + +type PuppetMetadataReader interface { + ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error) +} + +// PuppetMetadata is a local module release; MetadataJSON is the tarball's +// metadata.json verbatim so Forge release objects carry every field. +type PuppetMetadata struct { + RepoName string + FilePath string + ContentHash string + Slug string + Version string + MetadataJSON string + Size int64 + MD5 string + CreatedAt time.Time +} diff --git a/internal/provider/puppet/local.go b/internal/provider/puppet/local.go new file mode 100644 index 0000000..b9a3a4c --- /dev/null +++ b/internal/provider/puppet/local.go @@ -0,0 +1,393 @@ +package puppet + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "path" + "regexp" + "slices" + "strconv" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/storage" +) + +const filesPrefix = "v3/files/" + +// releaseRe matches --, the Forge release slug and the +// tarball basename that `puppet module build` produces. +var releaseRe = regexp.MustCompile(`^([a-z0-9]+)-([a-z][a-z0-9_]*)-(\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?)$`) + +func parseRelease(s string) (slug, version string, ok bool) { + m := releaseRe.FindStringSubmatch(strings.ToLower(s)) + if m == nil { + return "", "", false + } + return m[1] + "-" + m[2], m[3], true +} + +func normalizeSlug(s string) string { + return strings.ToLower(strings.ReplaceAll(s, "/", "-")) +} + +// ValidateUpload stores tarballs under v3/files/ so Forge file_uri downloads +// resolve straight to the stored file. +func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { + filename := strings.ToLower(path.Base(path.Clean("/" + filePath))) + if _, _, ok := parseRelease(strings.TrimSuffix(filename, ".tar.gz")); !ok || !strings.HasSuffix(filename, ".tar.gz") { + return "", "", fmt.Errorf("file must be named --.tar.gz") + } + return filesPrefix + filename, "application/gzip", nil +} + +func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any { + return map[string]any{ + "filename": path.Base(storagePath), + "content_hash": contentHash, + "size_bytes": sizeBytes, + } +} + +func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) { + reader, size, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))) + if err != nil { + slog.Error("puppet metadata: download failed", "repo", repoName, "path", storagePath, "error", err) + return + } + defer func() { _ = reader.Close() }() + if size > maxModuleTarball { + slog.Error("puppet metadata: tarball too large", "repo", repoName, "path", storagePath, "size", size) + return + } + + hash := md5.New() + src := io.TeeReader(io.LimitReader(reader, maxModuleTarball), hash) + meta, err := parseModule(src) + if err != nil { + slog.Error("puppet metadata: parse failed", "repo", repoName, "path", storagePath, "error", err) + return + } + if _, err := io.Copy(io.Discard, src); err != nil { + slog.Error("puppet metadata: read failed", "repo", repoName, "path", storagePath, "error", err) + return + } + slug, version, _ := parseRelease(strings.TrimSuffix(path.Base(storagePath), ".tar.gz")) + if meta.Slug != slug || meta.Version != version { + slog.Error("puppet metadata: metadata.json does not match filename", "repo", repoName, "path", storagePath, "name", meta.Slug, "version", meta.Version) + return + } + meta.RepoName = repoName + meta.FilePath = storagePath + meta.ContentHash = contentHash + meta.Size = size + meta.MD5 = hex.EncodeToString(hash.Sum(nil)) + + store, ok := db.(provider.PuppetMetadataStore) + if !ok { + slog.Error("puppet metadata: store does not support puppet metadata", "repo", repoName) + return + } + if err := store.InsertPuppetMetadata(ctx, meta); err != nil { + slog.Error("puppet metadata: insert failed", "repo", repoName, "path", storagePath, "error", err) + return + } + slog.Info("puppet metadata: parsed", "repo", repoName, "module", meta.Slug, "version", meta.Version) +} + +func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error { + deleter, ok := db.(provider.PuppetMetadataDeleter) + if !ok { + return nil + } + return deleter.DeletePuppetMetadata(ctx, repoName, storagePath) +} + +const maxMetadataJSON = 1 << 20 + +var ( + maxModuleTarball int64 = 100 << 20 + maxModuleScan int64 = 200 << 20 +) + +// parseModule reads the top-level /metadata.json from a module tarball, +// scanning at most maxModuleScan decompressed bytes. +func parseModule(r io.Reader) (*provider.PuppetMetadata, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + scan := &io.LimitedReader{R: zr, N: maxModuleScan} + tr := tar.NewReader(scan) + for { + hdr, err := tr.Next() + if scan.N <= 0 { + return nil, errors.New("module tarball exceeds decompressed scan limit before metadata.json") + } + if err == io.EOF { + return nil, errors.New("no metadata.json in module tarball") + } + if err != nil { + return nil, err + } + dir, file := path.Split(strings.TrimPrefix(path.Clean(hdr.Name), "./")) + if hdr.Typeflag != tar.TypeReg || file != "metadata.json" || strings.Count(dir, "/") != 1 { + continue + } + body, err := io.ReadAll(io.LimitReader(tr, maxMetadataJSON)) + if err != nil { + return nil, err + } + var md struct { + Name string `json:"name"` + Version string `json:"version"` + } + if err := json.Unmarshal(body, &md); err != nil { + return nil, fmt.Errorf("parse metadata.json: %w", err) + } + if md.Name == "" || md.Version == "" { + return nil, errors.New("metadata.json missing name or version") + } + return &provider.PuppetMetadata{Slug: normalizeSlug(md.Name), Version: md.Version, MetadataJSON: string(body)}, nil + } +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool { + clean := strings.TrimPrefix(path.Clean("/"+reqPath), "/") + var list func(slug string) ([]provider.PuppetMetadata, error) + if reader, ok := files.(provider.PuppetMetadataReader); ok { + list = func(slug string) ([]provider.PuppetMetadata, error) { + metas, err := reader.ListPuppetMetadataEntries(r.Context(), repoName, slug) + sortReleases(metas) + return metas, err + } + } else { + list = func(string) ([]provider.PuppetMetadata, error) { + return nil, errors.New("puppet metadata not available") + } + } + + switch { + case clean == "v3/releases": + serveReleaseList(w, r, list) + case strings.HasPrefix(clean, "v3/releases/"): + slug, version, ok := parseRelease(strings.TrimPrefix(clean, "v3/releases/")) + if !ok { + notFound(w) + return true + } + metas, err := list(slug) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + for _, m := range metas { + if m.Version == version { + writeJSON(w, releaseObject(m)) + return true + } + } + notFound(w) + case strings.HasPrefix(clean, "v3/modules/"): + slug := normalizeSlug(strings.TrimPrefix(clean, "v3/modules/")) + metas, err := list(slug) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + if len(metas) == 0 { + notFound(w) + return true + } + writeJSON(w, moduleObject(metas)) + default: + return false + } + return true +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) { + return nil, fmt.Errorf("puppet local index generation for virtual repos not supported") +} + +func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) ([]provider.PuppetMetadata, error)) { + q := r.URL.Query() + slug := normalizeSlug(q.Get("module")) + limit, err := strconv.Atoi(q.Get("limit")) + if err != nil || limit <= 0 { + limit = 20 + } + limit = min(limit, 100) + offset, err := strconv.Atoi(q.Get("offset")) + if err != nil || offset < 0 { + offset = 0 + } + + metas, err := list(slug) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + offset = min(offset, len(metas)) + results := []map[string]any{} + for _, m := range metas[offset : offset+min(limit, len(metas)-offset)] { + results = append(results, releaseObject(m)) + } + + page := func(off int) any { + if off < 0 || off >= len(metas) && off != 0 { + return nil + } + v := url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}} + if slug != "" { + v.Set("module", slug) + } + return "/v3/releases?" + v.Encode() + } + var previous any + if offset > 0 { + previous = page(max(offset-limit, 0)) + } + writeJSON(w, map[string]any{ + "pagination": map[string]any{ + "limit": limit, + "offset": offset, + "first": page(0), + "previous": previous, + "current": page(offset), + "next": page(offset + limit), + "total": len(metas), + }, + "results": results, + }) +} + +func owner(slug string) map[string]any { + name, _, _ := strings.Cut(slug, "-") + return map[string]any{"uri": "/v3/users/" + name, "slug": name, "username": name} +} + +func moduleRef(slug string) map[string]any { + _, name, _ := strings.Cut(slug, "-") + return map[string]any{"uri": "/v3/modules/" + slug, "slug": slug, "name": name, "deprecated_at": nil, "owner": owner(slug)} +} + +const forgeTime = "2006-01-02 15:04:05 -0700" + +func releaseObject(m provider.PuppetMetadata) map[string]any { + slug := m.Slug + "-" + m.Version + return map[string]any{ + "uri": "/v3/releases/" + slug, + "slug": slug, + "module": moduleRef(m.Slug), + "version": m.Version, + "metadata": json.RawMessage(m.MetadataJSON), + "tags": []string{}, + "supported": false, + "pdk": false, + "file_uri": "/" + m.FilePath, + "file_size": m.Size, + "file_md5": m.MD5, + "file_sha256": strings.TrimPrefix(m.ContentHash, "sha256:"), + "downloads": 0, + "readme": nil, + "changelog": nil, + "license": nil, + "created_at": m.CreatedAt.Format(forgeTime), + "updated_at": m.CreatedAt.Format(forgeTime), + "deleted_at": nil, + } +} + +// moduleObject expects metas sorted newest version first. +func moduleObject(metas []provider.PuppetMetadata) map[string]any { + obj := moduleRef(metas[0].Slug) + releases := make([]map[string]any, 0, len(metas)) + for _, m := range metas { + slug := m.Slug + "-" + m.Version + releases = append(releases, map[string]any{ + "uri": "/v3/releases/" + slug, "slug": slug, "version": m.Version, + "file_uri": "/" + m.FilePath, "file_size": m.Size, "supported": false, + "created_at": m.CreatedAt.Format(forgeTime), "deleted_at": nil, + }) + } + obj["current_release"] = releaseObject(metas[0]) + obj["releases"] = releases + obj["downloads"] = 0 + obj["created_at"] = metas[len(metas)-1].CreatedAt.Format(forgeTime) + obj["updated_at"] = metas[0].CreatedAt.Format(forgeTime) + return obj +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func notFound(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"message":"404 Not Found","errors":["The requested resource could not be found"]}`) +} + +// sortReleases orders by module slug, then newest semver first. +func sortReleases(metas []provider.PuppetMetadata) { + slices.SortStableFunc(metas, func(a, b provider.PuppetMetadata) int { + if c := strings.Compare(a.Slug, b.Slug); c != 0 { + return c + } + return compareSemver(b.Version, a.Version) + }) +} + +func compareSemver(a, b string) int { + a, _, _ = strings.Cut(a, "+") + b, _, _ = strings.Cut(b, "+") + aCore, aPre, aHasPre := strings.Cut(a, "-") + bCore, bPre, bHasPre := strings.Cut(b, "-") + if c := compareIdents(strings.Split(aCore, "."), strings.Split(bCore, ".")); c != 0 { + return c + } + switch { + case aHasPre && !bHasPre: + return -1 + case !aHasPre && bHasPre: + return 1 + } + return compareIdents(strings.Split(aPre, "."), strings.Split(bPre, ".")) +} + +func compareIdents(a, b []string) int { + for i := 0; i < len(a) && i < len(b); i++ { + an, aErr := strconv.Atoi(a[i]) + bn, bErr := strconv.Atoi(b[i]) + var c int + switch { + case aErr == nil && bErr == nil: + c = an - bn + case aErr == nil: + c = -1 + case bErr == nil: + c = 1 + default: + c = strings.Compare(a[i], b[i]) + } + if c != 0 { + return c + } + } + return len(a) - len(b) +} diff --git a/internal/provider/puppet/local_test.go b/internal/provider/puppet/local_test.go new file mode 100644 index 0000000..57927d3 --- /dev/null +++ b/internal/provider/puppet/local_test.go @@ -0,0 +1,301 @@ +package puppet + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func moduleTarball(t *testing.T, dir, metadata string) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range map[string]string{dir + "/metadata.json": metadata, dir + "/manifests/init.pp": "class hello {}\n"} { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(body)) + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +const helloMetadata = `{"name":"unkin-hello","version":"0.1.0","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0"}]}` + +type fakeBlobs struct{ data []byte } + +func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil +} + +type fakeStore struct{ inserted *provider.PuppetMetadata } + +func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil } +func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil } +func (f *fakeStore) InsertPuppetMetadata(_ context.Context, m *provider.PuppetMetadata) error { + f.inserted = m + return nil +} + +type fakeReader struct{ metas []provider.PuppetMetadata } + +func (f fakeReader) ListPuppetMetadataEntries(_ context.Context, _, slug string) ([]provider.PuppetMetadata, error) { + var out []provider.PuppetMetadata + for _, m := range f.metas { + if slug == "" || m.Slug == slug { + out = append(out, m) + } + } + return out, nil +} +func (fakeReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (fakeReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestValidateUpload(t *testing.T) { + p := &Provider{} + sp, ct, err := p.ValidateUpload("some/dir/Unkin-hello-0.1.0.tar.gz") + if err != nil || sp != "v3/files/unkin-hello-0.1.0.tar.gz" || ct != "application/gzip" { + t.Errorf("sp=%q ct=%q err=%v", sp, ct, err) + } + for _, bad := range []string{"hello-0.1.0.tar.gz", "unkin-hello-0.1.tar.gz", "unkin-hello-0.1.0.zip", "unkin-Hello_x-0.1.0.tgz"} { + if _, _, err := p.ValidateUpload(bad); err == nil { + t.Errorf("%q accepted", bad) + } + } +} + +func TestAfterUploadRecordsMetadata(t *testing.T) { + raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata) + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + m := store.inserted + if m == nil { + t.Fatal("nothing inserted") + } + sum := md5.Sum(raw) + if m.Slug != "unkin-hello" || m.Version != "0.1.0" || m.MD5 != hex.EncodeToString(sum[:]) || m.Size != int64(len(raw)) || m.MetadataJSON != helloMetadata { + t.Errorf("got %+v", m) + } +} + +func TestAfterUploadRejectsMismatchedFilename(t *testing.T) { + raw := moduleTarball(t, "unkin-hello-0.2.0", `{"name":"unkin-hello","version":"0.2.0"}`) + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + if store.inserted != nil { + t.Error("mismatched version indexed") + } +} + +func TestParseModuleRequiresTopLevelMetadata(t *testing.T) { + if _, err := parseModule(bytes.NewReader(moduleTarball(t, "a/b", helloMetadata))); err == nil { + t.Error("nested metadata.json accepted") + } + if _, err := parseModule(bytes.NewReader(moduleTarball(t, "x", `{"name":"unkin-hello"}`))); err == nil { + t.Error("missing version accepted") + } +} + +func tarballOf(t *testing.T, entries ...*tar.Header) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for _, h := range entries { + if err := tw.WriteHeader(h); err != nil { + t.Fatal(err) + } + if h.Typeflag == tar.TypeReg { + body := helloMetadata + if !strings.HasSuffix(h.Name, "metadata.json") { + body = strings.Repeat("x", int(h.Size)) + } + _, _ = tw.Write([]byte(body)) + } + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +func TestParseModuleSkipsNonRegularMetadata(t *testing.T) { + raw := tarballOf(t, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"}, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json/", Typeflag: tar.TypeDir, Mode: 0o755}, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))}, + ) + meta, err := parseModule(bytes.NewReader(raw)) + if err != nil || meta.Slug != "unkin-hello" || meta.Version != "0.1.0" { + t.Fatalf("got %v, %v", meta, err) + } +} + +func TestParseModuleCapsDecompressedScan(t *testing.T) { + defer func(n int64) { maxModuleScan = n }(maxModuleScan) + maxModuleScan = 64 << 10 + raw := tarballOf(t, + &tar.Header{Name: "unkin-hello-0.1.0/big", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1 << 20}, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))}, + ) + if _, err := parseModule(bytes.NewReader(raw)); err == nil || !strings.Contains(err.Error(), "scan limit") { + t.Fatalf("expected scan limit error, got %v", err) + } + maxModuleScan = 2 << 20 + if _, err := parseModule(bytes.NewReader(raw)); err != nil { + t.Fatalf("under cap: %v", err) + } +} + +func TestAfterUploadCapsTarballSize(t *testing.T) { + defer func(n int64) { maxModuleTarball = n }(maxModuleTarball) + raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata) + maxModuleTarball = int64(len(raw)) - 1 + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + if store.inserted != nil { + t.Error("oversized tarball indexed") + } + maxModuleTarball = int64(len(raw)) + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + if store.inserted == nil { + t.Error("tarball at cap not indexed") + } +} + +func TestReleaseListClampsPagination(t *testing.T) { + files := releases("0.1.0", "0.2.0", "0.3.0") + for _, q := range []string{"offset=9223372036854775807", "offset=9223372036854775807&limit=9223372036854775807", "offset=-5&limit=-5", "limit=9223372036854775807"} { + code, body := get(t, files, "v3/releases?module=unkin-hello&"+q) + if code != http.StatusOK { + t.Fatalf("%s: code %d", q, code) + } + pg := body["pagination"].(map[string]any) + if l := pg["limit"].(float64); l < 1 || l > 100 { + t.Errorf("%s: limit %v", q, l) + } + if o := pg["offset"].(float64); o < 0 || o > 3 { + t.Errorf("%s: offset %v", q, o) + } + } + _, body := get(t, files, "v3/releases?module=unkin-hello&offset=9223372036854775807") + if len(body["results"].([]any)) != 0 || body["pagination"].(map[string]any)["next"] != nil { + t.Errorf("past-end page: %v", body) + } +} + +func get(t *testing.T, files provider.FileStore, target string) (int, map[string]any) { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/x/"+target, nil) + path, _, _ := strings.Cut(target, "?") + if !(&Provider{}).ServeLocalIndex(rec, req, files, "forge", path) { + return 0, nil + } + var body map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &body) + return rec.Code, body +} + +func releases(versions ...string) fakeReader { + var f fakeReader + for i, v := range versions { + f.metas = append(f.metas, provider.PuppetMetadata{ + FilePath: "v3/files/unkin-hello-" + v + ".tar.gz", ContentHash: "sha256:feed", Slug: "unkin-hello", + Version: v, MetadataJSON: `{"name":"unkin-hello","version":"` + v + `"}`, MD5: "m" + v, + CreatedAt: time.Unix(int64(i), 0), + }) + } + f.metas = append(f.metas, provider.PuppetMetadata{Slug: "other-mod", Version: "9.9.9", MetadataJSON: `{}`}) + return f +} + +func TestReleaseListPaginatesNewestFirst(t *testing.T) { + files := releases("0.1.0", "0.10.0", "0.2.0", "1.0.0-rc.1", "1.0.0") + code, body := get(t, files, "v3/releases?module=unkin/hello&limit=2") + if code != http.StatusOK { + t.Fatalf("code %d", code) + } + results := body["results"].([]any) + pg := body["pagination"].(map[string]any) + if len(results) != 2 || results[0].(map[string]any)["version"] != "1.0.0" || results[1].(map[string]any)["version"] != "1.0.0-rc.1" { + t.Fatalf("page 1: %v", results) + } + if pg["total"] != float64(5) || pg["next"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" || pg["previous"] != nil { + t.Errorf("pagination: %v", pg) + } + + _, body = get(t, files, "v3/releases?module=unkin-hello&limit=2&offset=4") + results = body["results"].([]any) + pg = body["pagination"].(map[string]any) + if len(results) != 1 || results[0].(map[string]any)["version"] != "0.1.0" || pg["next"] != nil || pg["previous"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" { + t.Errorf("last page: %v %v", results, pg) + } + + _, body = get(t, files, "v3/releases?module=nobody-none") + if len(body["results"].([]any)) != 0 { + t.Error("unknown module returned results") + } +} + +func TestReleaseObject(t *testing.T) { + code, body := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.1.0") + if code != http.StatusOK { + t.Fatalf("code %d", code) + } + if body["file_uri"] != "/v3/files/unkin-hello-0.1.0.tar.gz" || body["file_md5"] != "m0.1.0" || body["file_sha256"] != "feed" { + t.Errorf("release: %v", body) + } + if body["metadata"].(map[string]any)["name"] != "unkin-hello" || body["module"].(map[string]any)["owner"].(map[string]any)["username"] != "unkin" { + t.Errorf("release refs: %v", body) + } + if code, _ := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.2.0"); code != http.StatusNotFound { + t.Errorf("missing version: %d", code) + } +} + +func TestModuleObject(t *testing.T) { + code, body := get(t, releases("0.9.0", "0.10.0"), "v3/modules/unkin-hello") + if code != http.StatusOK { + t.Fatalf("code %d", code) + } + if body["slug"] != "unkin-hello" || body["name"] != "hello" || body["current_release"].(map[string]any)["version"] != "0.10.0" || len(body["releases"].([]any)) != 2 { + t.Errorf("module: %v", body) + } + if code, _ := get(t, releases(), "v3/modules/unkin-missing"); code != http.StatusNotFound { + t.Errorf("missing module: %d", code) + } +} + +func TestFilesFallThrough(t *testing.T) { + if code, _ := get(t, releases(), "v3/files/unkin-hello-0.1.0.tar.gz"); code != 0 { + t.Error("file downloads must fall through to the stored file") + } +} + +func TestCompareSemver(t *testing.T) { + ordered := []string{"0.1.0", "0.2.0", "0.10.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0", "1.0.1+build"} + for i := 1; i < len(ordered); i++ { + if compareSemver(ordered[i-1], ordered[i]) >= 0 || compareSemver(ordered[i], ordered[i-1]) <= 0 { + t.Errorf("%s should sort before %s", ordered[i-1], ordered[i]) + } + } + if compareSemver("1.0.0+a", "1.0.0+b") != 0 { + t.Error("build metadata must not affect precedence") + } +} diff --git a/migrations/0004_puppet_metadata.sql b/migrations/0004_puppet_metadata.sql new file mode 100644 index 0000000..a3bd85e --- /dev/null +++ b/migrations/0004_puppet_metadata.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS puppet_metadata ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL, + slug TEXT NOT NULL, + version TEXT NOT NULL, + metadata_json TEXT NOT NULL, + file_size BIGINT NOT NULL DEFAULT 0, + file_md5 TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_puppet_metadata_repo_slug ON puppet_metadata(repo_name, slug); diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index 92c1e57..dd66d22 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -262,11 +262,21 @@ index = "sparse+${proxy}/"`, case 'puppet': return [ { - title: 'Install a module from the Forge proxy', + title: isLocal ? 'Install a module from this forge' : 'Install a module from the Forge proxy', language: 'bash', code: `puppet module install - \\ - --module_repository ${proxy}`, + --module_repository ${isLocal ? `${url}/api/v1/local/${name}` : proxy}`, }, + ...(isLocal + ? [ + { + title: 'Publish a module (versions are immutable)', + language: 'bash', + code: `puppet module build && curl -fsSL --upload-file pkg/--.tar.gz \\ + ${url}/api/v2/remotes/${name}/files/--.tar.gz`, + }, + ] + : []), ]; case 'deb': From 0150b5ebb48eee5bd6425cd001d9f91853607001 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 13:28:15 +1100 Subject: [PATCH 2/4] Share the virtual merge cache across index types (#147) The rpm virtual's merge cache, singleflight and two-generation retention are hard-wired to RPMRepo, so other merged index types (deb next) can't reuse them. - add a mergedRepo interface (index + referenced files) implemented by RPMRepo - key the merge cache by type-scoped string and serve files from both retained generations via cachedFile - extract parallel fail-closed member fetching into a generic fetchMembers - add tests for generation retention and per-key isolation Reviewed-on: https://git.unkin.net/unkin/artifactapi/pulls/147 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- internal/virtual/engine.go | 90 ++++++++++++++++++++------------- internal/virtual/engine_test.go | 50 ++++++++++++++++++ 2 files changed, 105 insertions(+), 35 deletions(-) diff --git a/internal/virtual/engine.go b/internal/virtual/engine.go index d4c52d0..1f8bf7c 100644 --- a/internal/virtual/engine.go +++ b/internal/virtual/engine.go @@ -29,15 +29,24 @@ type Engine struct { mergeTTL time.Duration - sf singleflight.Group - mu sync.Mutex - rpm map[string]*rpmGen + sf singleflight.Group + mu sync.Mutex + gens map[string]*repoGen } -// rpmGen holds a virtual's current merge plus the previous one, so a client -// holding the previous repomd can still fetch its content-hashed data files. -type rpmGen struct { - cur, prev *RPMRepo +// mergedRepo is a merged index document plus the files it references. +type mergedRepo interface { + index() []byte + files() map[string][]byte +} + +func (r *RPMRepo) index() []byte { return r.Repomd } +func (r *RPMRepo) files() map[string][]byte { return r.Files } + +// repoGen holds a virtual's current merge plus the previous one, so a client +// holding the previous index can still fetch the files it references. +type repoGen struct { + cur, prev mergedRepo at time.Time } @@ -217,54 +226,57 @@ func (e *Engine) fetchRPM(ctx context.Context, virt models.Virtual, path string) if !strings.HasPrefix(path, "repodata/") { return nil, "", ErrNotFound } + key := "rpm\x00" + virt.Name if path != "repodata/repomd.xml" { - if body, ok := e.cachedRPMFile(virt.Name, path); ok { + if body, ok := e.cachedFile(key, path); ok { return body, "application/gzip", nil } } - repo, err := e.mergedRPM(ctx, virt) + repo, err := e.merged(ctx, key, func(ctx context.Context) (mergedRepo, error) { + return e.mergeRPM(ctx, virt) + }) if err != nil { return nil, "", err } if path == "repodata/repomd.xml" { - return repo.Repomd, "application/xml", nil + return repo.index(), "application/xml", nil } - if body, ok := repo.Files[path]; ok { + if body, ok := repo.files()[path]; ok { return body, "application/gzip", nil } return nil, "", ErrNotFound } -// mergedRPM returns the virtual's merge, reusing it for mergeTTL and -// collapsing concurrent merges of the same virtual into one. +// merged returns the merge cached under key, reusing it for mergeTTL and +// collapsing concurrent merges of the same key into one. // ponytail: per-replica cache; behind a non-sticky LB a data request landing on // another replica re-merges and 404s if a member changed in between. Move to the // shared redis cache if that shows up. -func (e *Engine) mergedRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, error) { +func (e *Engine) merged(ctx context.Context, key string, merge func(context.Context) (mergedRepo, error)) (mergedRepo, error) { e.mu.Lock() - g := e.rpm[virt.Name] + g := e.gens[key] if g != nil && time.Since(g.at) < e.mergeTTL { e.mu.Unlock() return g.cur, nil } e.mu.Unlock() - v, err, _ := e.sf.Do(virt.Name, func() (any, error) { - repo, err := e.mergeRPM(context.WithoutCancel(ctx), virt) + v, err, _ := e.sf.Do(key, func() (any, error) { + repo, err := merge(context.WithoutCancel(ctx)) if err != nil { return nil, err } e.mu.Lock() defer e.mu.Unlock() - if e.rpm == nil { - e.rpm = map[string]*rpmGen{} + if e.gens == nil { + e.gens = map[string]*repoGen{} } - g := e.rpm[virt.Name] + g := e.gens[key] switch { case g == nil: - e.rpm[virt.Name] = &rpmGen{cur: repo, at: time.Now()} - case string(g.cur.Repomd) == string(repo.Repomd): + e.gens[key] = &repoGen{cur: repo, at: time.Now()} + case string(g.cur.index()) == string(repo.index()): g.at = time.Now() repo = g.cur default: @@ -275,18 +287,31 @@ func (e *Engine) mergedRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, if err != nil { return nil, err } - return v.(*RPMRepo), nil + return v.(mergedRepo), nil } func (e *Engine) mergeRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, error) { - members := make([]RPMMember, len(virt.Members)) + members, err := fetchMembers(virt, func(name string) (*RPMMember, error) { return e.rpmMember(ctx, name) }) + if err != nil { + return nil, err + } + repo, err := MergeRPM(members) + if err != nil { + return nil, fmt.Errorf("merge rpm repodata: %w", err) + } + return repo, nil +} + +// fetchMembers reads every member in parallel, failing if any member fails. +func fetchMembers[T any](virt models.Virtual, fetch func(name string) (*T, error)) ([]T, error) { + members := make([]T, len(virt.Members)) errs := make([]error, len(virt.Members)) var wg sync.WaitGroup for i, name := range virt.Members { wg.Add(1) go func() { defer wg.Done() - m, err := e.rpmMember(ctx, name) + m, err := fetch(name) if err != nil { errs[i] = fmt.Errorf("member %q: %w", name, err) return @@ -299,26 +324,21 @@ func (e *Engine) mergeRPM(ctx context.Context, virt models.Virtual) (*RPMRepo, e if err := errors.Join(errs...); err != nil { return nil, fmt.Errorf("virtual %q: %v", virt.Name, err) } - - repo, err := MergeRPM(members) - if err != nil { - return nil, fmt.Errorf("merge rpm repodata: %w", err) - } - return repo, nil + return members, nil } -func (e *Engine) cachedRPMFile(virt, path string) ([]byte, bool) { +func (e *Engine) cachedFile(key, path string) ([]byte, bool) { e.mu.Lock() defer e.mu.Unlock() - g := e.rpm[virt] + g := e.gens[key] if g == nil { return nil, false } - for _, r := range []*RPMRepo{g.cur, g.prev} { + for _, r := range []mergedRepo{g.cur, g.prev} { if r == nil { continue } - if body, ok := r.Files[path]; ok { + if body, ok := r.files()[path]; ok { return body, true } } diff --git a/internal/virtual/engine_test.go b/internal/virtual/engine_test.go index d1c28c9..cb7a334 100644 --- a/internal/virtual/engine_test.go +++ b/internal/virtual/engine_test.go @@ -156,3 +156,53 @@ func TestMemberRedirect(t *testing.T) { } } } + +type fakeRepo struct { + idx string + fs map[string][]byte +} + +func (r *fakeRepo) index() []byte { return []byte(r.idx) } +func (r *fakeRepo) files() map[string][]byte { return r.fs } + +func genRepo(gen string) func(context.Context) (mergedRepo, error) { + return func(context.Context) (mergedRepo, error) { + return &fakeRepo{idx: gen, fs: map[string][]byte{gen + ".gz": []byte(gen), "fixed": []byte(gen)}}, nil + } +} + +func TestMergedRetainsPreviousGeneration(t *testing.T) { + e := fakeEngine(nil) + for _, gen := range []string{"g1", "g2", "g3"} { + if _, err := e.merged(context.Background(), "k", genRepo(gen)); err != nil { + t.Fatal(err) + } + } + for path, want := range map[string]string{"g3.gz": "g3", "g2.gz": "g2", "fixed": "g3"} { + if body, ok := e.cachedFile("k", path); !ok || string(body) != want { + t.Errorf("%s: got (%q, %v), want %q", path, body, ok, want) + } + } + if _, ok := e.cachedFile("k", "g1.gz"); ok { + t.Error("generation two merges back must be dropped") + } +} + +func TestMergedReusesWithinTTLAndIsolatesKeys(t *testing.T) { + e := fakeEngine(nil) + e.mergeTTL = time.Minute + for _, step := range []struct{ key, gen, want string }{ + {"a", "a1", "a1"}, {"b", "b1", "b1"}, {"a", "a2", "a1"}, + } { + repo, err := e.merged(context.Background(), step.key, genRepo(step.gen)) + if err != nil || string(repo.index()) != step.want { + t.Fatalf("%s: got %v (err %v), want %s", step.key, repo, err, step.want) + } + } + if _, ok := e.cachedFile("b", "a1.gz"); ok { + t.Error("key b served key a's file") + } + if _, ok := e.cachedFile("c", "a1.gz"); ok { + t.Error("unknown key served a file") + } +} From aaa923ffd1b89d6660a04be73bfed137b2e4aa20 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 15:14:36 +1100 Subject: [PATCH 3/4] Add helm local repositories (#140) In-house charts have nowhere to publish: a helm local stores uploads but serves no index.yaml, and the helm virtual merger drops every chart field except name/version/urls. - add helm local uploads (`PUT .../files/-.tgz`) recording each Chart.yaml in a new `helm_metadata` table; mismatched filenames stay out of the index - bound chart parsing (50MiB decompressed, 10k entries, 1MiB Chart.yaml) - generate a deterministic `index.yaml` with sha256 digests and path-escaped relative urls - keep all chart fields when merging helm virtual indexes, expanding YAML aliases - add a real-helm e2e-docker test, UI snippet and README section Reviewed-on: https://git.unkin.net/unkin/artifactapi/pulls/140 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- README.md | 16 ++ e2e-docker/helm_local_test.go | 71 +++++++ internal/database/helm_metadata.go | 44 ++++ internal/provider/helm/helm.go | 212 +++++++++++++++++++ internal/provider/helm/helm_local_test.go | 236 ++++++++++++++++++++++ internal/provider/provider.go | 24 +++ internal/provider/yamlalias.go | 46 +++++ internal/virtual/helm_merger.go | 100 +++++---- internal/virtual/helm_merger_test.go | 129 ++++++++++++ migrations/0005_helm_metadata.sql | 13 ++ ui/src/components/UsageInstructions.tsx | 15 +- 11 files changed, 863 insertions(+), 43 deletions(-) create mode 100644 e2e-docker/helm_local_test.go create mode 100644 internal/database/helm_metadata.go create mode 100644 internal/provider/helm/helm_local_test.go create mode 100644 internal/provider/yamlalias.go create mode 100644 migrations/0005_helm_metadata.sql diff --git a/README.md b/README.md index 2d76346..48e3545 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,22 @@ there's nothing to provision. To bring your own key instead, point `TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. `TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. +### Local helm repository + +A local `helm` repo is a real chart repository: `PUT` a packaged chart to +`/api/v2/remotes//files/-.tgz` and `index.yaml` is +generated from each chart's `Chart.yaml` (with sha256 digests) at +`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` +name and version is stored but left out of the index. Add the local as a +member of a helm virtual to serve it alongside remotes. + +```bash +helm package ./mychart +curl -fsSL --upload-file mychart-0.1.0.tgz \ + https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/mychart-0.1.0.tgz +helm repo add https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ +``` + ### Local puppet forge A local `puppet` repo is a Forge: `PUT` a module tarball to diff --git a/e2e-docker/helm_local_test.go b/e2e-docker/helm_local_test.go new file mode 100644 index 0000000..a0c9615 --- /dev/null +++ b/e2e-docker/helm_local_test.go @@ -0,0 +1,71 @@ +//go:build dockere2e + +package e2edocker + +import ( + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it +// to a helm local, then drives helm repo add/search/pull/template against the +// local and against a helm virtual that has the local as a member. +func TestLocalHelmRealClient(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`) + defer deleteRepo(t, "local-helm") + createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`) + defer deleteVirtual(t, "vhelm") + + helm := func(script string) string { + t.Helper() + out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", + "alpine/helm:3.16.4", "-c", "set -eu; "+script).Output() + if err != nil { + stderr := "" + if ee, ok := err.(*exec.ExitError); ok { + stderr = string(ee.Stderr) + } + t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr) + } + return string(out) + } + + chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz") + uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip") + + // Chart metadata is recorded asynchronously after upload; poll the index. + var body []byte + for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) { + var resp *http.Response + resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "") + if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") { + break + } + } + if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") { + t.Fatalf("index.yaml never listed the chart:\n%s", body) + } + + for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} { + out := helm("helm repo add e2e " + repoURL + " >/dev/null && " + + "helm search repo e2e/e2echart && " + + "cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " + + "helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'") + if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") { + t.Fatalf("helm via %s:\n%s", repoURL, out) + } + } +} diff --git a/internal/database/helm_metadata.go b/internal/database/helm_metadata.go new file mode 100644 index 0000000..6110f03 --- /dev/null +++ b/internal/database/helm_metadata.go @@ -0,0 +1,44 @@ +package database + +import ( + "context" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml) + VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (repo_name, file_path) DO NOTHING + `, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML) + return err +} + +func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} + +func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at + FROM helm_metadata + WHERE repo_name = $1 + ORDER BY name, version, file_path + `, repoName) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.HelmMetadata + for rows.Next() { + var m provider.HelmMetadata + if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil { + return nil, err + } + result = append(result, m) + } + return result, rows.Err() +} diff --git a/internal/provider/helm/helm.go b/internal/provider/helm/helm.go index d37765e..4904ff6 100644 --- a/internal/provider/helm/helm.go +++ b/internal/provider/helm/helm.go @@ -1,12 +1,24 @@ package helm import ( + "archive/tar" + "compress/gzip" "context" + "errors" + "fmt" + "io" + "log/slog" "net/http" + "net/url" + "path" "strings" + "time" + + "gopkg.in/yaml.v3" "git.unkin.net/unkin/artifactapi/internal/auth" "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -56,3 +68,203 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { return auth.BasicHeaders(remote), nil } + +// ValidateUpload stores charts flat by filename so index.yaml can reference +// them with relative urls, as `helm repo index` does. +func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { + filename := path.Base(path.Clean("/" + filePath)) + if !strings.HasSuffix(filename, ".tgz") || filename == ".tgz" { + return "", "", fmt.Errorf("file must be a .tgz chart package") + } + return filename, "application/gzip", nil +} + +func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any { + return map[string]any{ + "filename": storagePath, + "content_hash": contentHash, + "size_bytes": sizeBytes, + } +} + +func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) { + reader, _, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))) + if err != nil { + slog.Error("helm metadata: download failed", "repo", repoName, "path", storagePath, "error", err) + return + } + defer func() { _ = reader.Close() }() + + meta, err := parseChart(reader) + if err != nil { + slog.Error("helm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err) + return + } + if want := meta.Name + "-" + meta.Version + ".tgz"; storagePath != want { + slog.Error("helm metadata: filename does not match chart", "repo", repoName, "path", storagePath, "expected", want) + return + } + meta.RepoName = repoName + meta.FilePath = storagePath + meta.ContentHash = contentHash + + store, ok := db.(provider.HelmMetadataStore) + if !ok { + slog.Error("helm metadata: store does not support helm metadata", "repo", repoName) + return + } + if err := store.InsertHelmMetadata(ctx, meta); err != nil { + slog.Error("helm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err) + return + } + slog.Info("helm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version) +} + +func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error { + deleter, ok := db.(provider.HelmMetadataDeleter) + if !ok { + return nil + } + return deleter.DeleteHelmMetadata(ctx, repoName, storagePath) +} + +func isIndexPath(p string) bool { + return strings.TrimPrefix(path.Clean("/"+p), "/") == "index.yaml" +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool { + if !isIndexPath(reqPath) { + return false + } + body, err := p.GenerateLocalIndex(r.Context(), files, repoName, reqPath) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + w.Header().Set("Content-Type", "text/yaml") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + return true +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, reqPath string) ([]byte, error) { + if !isIndexPath(reqPath) { + return nil, fmt.Errorf("unsupported index path: %q", reqPath) + } + reader, ok := files.(provider.HelmMetadataReader) + if !ok { + return nil, errors.New("helm metadata not available") + } + metas, err := reader.ListHelmMetadataEntries(ctx, repoName) + if err != nil { + return nil, err + } + return generateIndex(metas) +} + +// generateIndex builds index.yaml from stored Chart.yaml documents; it is +// stamped from the newest upload so every replica serves identical bytes. +func generateIndex(metas []provider.HelmMetadata) ([]byte, error) { + entries := map[string][]map[string]yaml.Node{} + var newest time.Time + budget := provider.MaxAliasExpandedNodes + for _, m := range metas { + var raw map[string]yaml.Node + if err := yaml.Unmarshal([]byte(m.ChartYAML), &raw); err != nil { + return nil, fmt.Errorf("chart %s: %w", m.FilePath, err) + } + chart, err := provider.ExpandYAMLAliases(raw, &budget) + if err != nil { + slog.Warn("helm index: dropping chart", "chart", m.Name, "path", m.FilePath, "error", err) + continue + } + chart["name"] = node(m.Name) + chart["version"] = node(m.Version) + chart["urls"] = node([]string{url.PathEscape(m.FilePath)}) + chart["digest"] = node(strings.TrimPrefix(m.ContentHash, "sha256:")) + chart["created"] = node(m.CreatedAt.UTC().Format(time.RFC3339Nano)) + entries[m.Name] = append(entries[m.Name], chart) + if m.CreatedAt.After(newest) { + newest = m.CreatedAt + } + } + return yaml.Marshal(map[string]any{ + "apiVersion": "v1", + "entries": entries, + "generated": newest.UTC().Format(time.RFC3339Nano), + }) +} + +func node(v any) yaml.Node { + var n yaml.Node + _ = n.Encode(v) + return n +} + +const ( + maxChartYAML = 1 << 20 + maxChartBytes = 50 << 20 + maxChartEntries = 10000 +) + +var errChartTooLarge = fmt.Errorf("chart archive exceeds %d decompressed bytes", maxChartBytes) + +type capReader struct { + r io.Reader + n int64 +} + +func (c *capReader) Read(p []byte) (int, error) { + if c.n <= 0 { + return 0, errChartTooLarge + } + if int64(len(p)) > c.n { + p = p[:c.n] + } + n, err := c.r.Read(p) + c.n -= int64(n) + return n, err +} + +// parseChart reads the top-level /Chart.yaml from a packaged chart. +func parseChart(r io.Reader) (*provider.HelmMetadata, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + tr := tar.NewReader(&capReader{r: zr, n: maxChartBytes}) + for entries := 0; ; entries++ { + if entries >= maxChartEntries { + return nil, fmt.Errorf("chart archive exceeds %d entries", maxChartEntries) + } + hdr, err := tr.Next() + if err == io.EOF { + return nil, errors.New("no Chart.yaml in chart archive") + } + if err != nil { + return nil, err + } + dir, file := path.Split(path.Clean(hdr.Name)) + if file != "Chart.yaml" || strings.Count(dir, "/") != 1 { + continue + } + raw, err := io.ReadAll(io.LimitReader(tr, maxChartYAML+1)) + if err != nil { + return nil, err + } + if len(raw) > maxChartYAML { + return nil, fmt.Errorf("parse Chart.yaml: exceeds %d bytes", maxChartYAML) + } + var chart struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + } + if err := yaml.Unmarshal(raw, &chart); err != nil { + return nil, fmt.Errorf("parse Chart.yaml: %w", err) + } + if chart.Name == "" || chart.Version == "" { + return nil, errors.New("chart metadata missing name or version") + } + return &provider.HelmMetadata{Name: chart.Name, Version: chart.Version, ChartYAML: string(raw)}, nil + } +} diff --git a/internal/provider/helm/helm_local_test.go b/internal/provider/helm/helm_local_test.go new file mode 100644 index 0000000..abe2538 --- /dev/null +++ b/internal/provider/helm/helm_local_test.go @@ -0,0 +1,236 @@ +package helm + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func chartTgz(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range files { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write([]byte(body)); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +type fakeBlobs struct{ data []byte } + +func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil +} + +type fakeStore struct { + inserted *provider.HelmMetadata + metas []provider.HelmMetadata +} + +func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil } +func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil } +func (f *fakeStore) InsertHelmMetadata(_ context.Context, m *provider.HelmMetadata) error { + f.inserted = m + return nil +} +func (f *fakeStore) ListHelmMetadataEntries(context.Context, string) ([]provider.HelmMetadata, error) { + return f.metas, nil +} +func (f *fakeStore) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (f *fakeStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestValidateUpload(t *testing.T) { + p := &Provider{} + for in, want := range map[string]string{ + "mychart-0.1.0.tgz": "mychart-0.1.0.tgz", + "charts/mychart-0.1.0.tgz": "mychart-0.1.0.tgz", + "../../mychart-0.1.0.tgz": "mychart-0.1.0.tgz", + } { + sp, ct, err := p.ValidateUpload(in) + if err != nil || sp != want || ct != "application/gzip" { + t.Errorf("ValidateUpload(%q) = %q %q %v", in, sp, ct, err) + } + } + for _, bad := range []string{"index.yaml", "chart.tar.gz", ".tgz", ""} { + if _, _, err := p.ValidateUpload(bad); err == nil { + t.Errorf("ValidateUpload(%q) accepted", bad) + } + } +} + +func TestAfterUploadParsesTopLevelChart(t *testing.T) { + data := chartTgz(t, map[string]string{ + "mychart/charts/sub/Chart.yaml": "apiVersion: v2\nname: sub\nversion: 9.9.9\n", + "mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0\ndescription: hi\n", + "mychart/values.yaml": "a: 1\n", + }) + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "r", "mychart-1.0.tgz", "sha256:ab", fakeBlobs{data}, store) + m := store.inserted + if m == nil || m.Name != "mychart" || m.Version != "1.0" || m.FilePath != "mychart-1.0.tgz" || m.ContentHash != "sha256:ab" { + t.Fatalf("inserted %+v", m) + } +} + +func TestAfterUploadRejectsChartWithoutMetadata(t *testing.T) { + store := &fakeStore{} + data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\n"}) + (&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{data}, store) + if store.inserted != nil { + t.Fatal("chart without version was recorded") + } + (&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{[]byte("not gzip")}, store) + if store.inserted != nil { + t.Fatal("non-gzip upload was recorded") + } +} + +func TestServeLocalIndex(t *testing.T) { + t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + t2 := t1.Add(time.Hour) + store := &fakeStore{metas: []provider.HelmMetadata{ + {FilePath: "mychart-1.0.tgz", ContentHash: "sha256:aa", Name: "mychart", Version: "1.0", CreatedAt: t1, + ChartYAML: "apiVersion: v2\nname: mychart\nversion: 1.0\nappVersion: \"2\"\ndescription: hi\n"}, + {FilePath: "other-0.2.0.tgz", ContentHash: "sha256:bb", Name: "other", Version: "0.2.0", CreatedAt: t2, + ChartYAML: "apiVersion: v2\nname: other\nversion: 0.2.0\n"}, + }} + p := &Provider{} + + if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), store, "r", "mychart-1.0.tgz") { + t.Fatal("chart path handled as index") + } + rec := httptest.NewRecorder() + if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), store, "r", "./index.yaml") { + t.Fatal("index.yaml not handled") + } + if rec.Code != 200 || rec.Header().Get("Content-Type") != "text/yaml" { + t.Fatalf("code=%d ct=%q", rec.Code, rec.Header().Get("Content-Type")) + } + + var idx struct { + APIVersion string `yaml:"apiVersion"` + Generated string `yaml:"generated"` + Entries map[string][]map[string]any + } + if err := yaml.Unmarshal(rec.Body.Bytes(), &idx); err != nil { + t.Fatal(err) + } + if idx.APIVersion != "v1" || idx.Generated != t2.Format(time.RFC3339Nano) { + t.Errorf("header %+v", idx) + } + e := idx.Entries["mychart"][0] + if e["digest"] != "aa" || e["description"] != "hi" || e["appVersion"] != "2" || e["apiVersion"] != "v2" { + t.Errorf("entry %v", e) + } + if urls, _ := e["urls"].([]any); len(urls) != 1 || urls[0] != "mychart-1.0.tgz" { + t.Errorf("urls %v", e["urls"]) + } + if e["version"] != "1.0" || e["name"] != "mychart" { + t.Errorf("version must stay a string: %#v", e["version"]) + } + if len(idx.Entries["other"]) != 1 { + t.Errorf("other entries %v", idx.Entries["other"]) + } + + again, err := p.GenerateLocalIndex(context.Background(), store, "r", "index.yaml") + if err != nil || !bytes.Equal(again, rec.Body.Bytes()) { + t.Errorf("index not deterministic: %v", err) + } +} + +func TestAfterUploadRejectsFilenameMismatch(t *testing.T) { + store := &fakeStore{} + data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0.0\n"}) + (&Provider{}).AfterUpload(context.Background(), "r", "other-1.0.0.tgz", "sha256:ab", fakeBlobs{data}, store) + if store.inserted != nil { + t.Fatal("chart indexed under a filename that does not match name-version") + } +} + +func TestParseChartLimits(t *testing.T) { + chart := "apiVersion: v2\nname: c\nversion: 1.0.0\n" + for name, tc := range map[string]struct { + files map[string]string + want string + }{ + "decompressed": {map[string]string{"c/a.bin": strings.Repeat("\x00", maxChartBytes+1)}, "decompressed bytes"}, + "chart yaml": {map[string]string{"c/Chart.yaml": chart + "#" + strings.Repeat("x", maxChartYAML)}, "Chart.yaml: exceeds"}, + } { + _, err := parseChart(bytes.NewReader(chartTgz(t, tc.files))) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want %q", name, err, tc.want) + } + } + + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for i := 0; i <= maxChartEntries; i++ { + _ = tw.WriteHeader(&tar.Header{Name: "c/templates/" + strconv.Itoa(i), Mode: 0o644}) + } + _ = tw.WriteHeader(&tar.Header{Name: "c/Chart.yaml", Mode: 0o644, Size: int64(len(chart))}) + _, _ = tw.Write([]byte(chart)) + _ = tw.Close() + _ = zw.Close() + if _, err := parseChart(&buf); err == nil || !strings.Contains(err.Error(), "entries") { + t.Errorf("entries: err = %v", err) + } + + if m, err := parseChart(bytes.NewReader(chartTgz(t, map[string]string{"c/Chart.yaml": chart}))); err != nil || m.Name != "c" { + t.Errorf("valid chart: %v %v", m, err) + } +} + +func TestIndexEscapesURLs(t *testing.T) { + out, err := generateIndex([]provider.HelmMetadata{{FilePath: "my chart-1.0.0.tgz", Name: "my chart", Version: "1.0.0", ChartYAML: "name: x\n"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(out), "my%20chart-1.0.0.tgz") { + t.Errorf("url not escaped:\n%s", out) + } +} + +func TestIndexExpandsAliasesBeforeOverride(t *testing.T) { + body, err := generateIndex([]provider.HelmMetadata{{ + FilePath: "app-1.0.0.tgz", ContentHash: "sha256:aa", Name: "app", Version: "1.0.0", + ChartYAML: "apiVersion: v2\nname: app\nversion: &v 1.0.0\nappVersion: *v\n", + }}) + if err != nil { + t.Fatal(err) + } + var idx struct { + Entries map[string][]map[string]any + } + if err := yaml.Unmarshal(body, &idx); err != nil { + t.Fatalf("index does not parse: %v\n%s", err, body) + } + if e := idx.Entries["app"]; len(e) != 1 || e[0]["appVersion"] != "1.0.0" { + t.Errorf("entries %v", idx.Entries) + } +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index aeecde8..653fa25 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -248,6 +248,30 @@ func All() map[models.PackageType]Provider { return registry } +type HelmMetadataStore interface { + InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error +} + +type HelmMetadataDeleter interface { + DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error +} + +type HelmMetadataReader interface { + ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error) +} + +// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml +// entries carry every field helm reads. +type HelmMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + ChartYAML string + CreatedAt time.Time +} + type PuppetMetadataStore interface { InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error } diff --git a/internal/provider/yamlalias.go b/internal/provider/yamlalias.go new file mode 100644 index 0000000..af6f719 --- /dev/null +++ b/internal/provider/yamlalias.go @@ -0,0 +1,46 @@ +package provider + +import ( + "errors" + + "gopkg.in/yaml.v3" +) + +const MaxAliasExpandedNodes = 100000 + +// ExpandYAMLAliases inlines alias targets and drops anchors so a node map +// stays valid when copied away from the document that held its anchors; +// budget caps nodes reached through aliases against alias bombs. +func ExpandYAMLAliases(m map[string]yaml.Node, budget *int) (map[string]yaml.Node, error) { + var expand func(n *yaml.Node, viaAlias bool) (*yaml.Node, error) + expand = func(n *yaml.Node, viaAlias bool) (*yaml.Node, error) { + if viaAlias { + *budget-- + if *budget < 0 { + return nil, errors.New("yaml aliases expand to too many nodes") + } + } + if n.Kind == yaml.AliasNode { + return expand(n.Alias, true) + } + c := *n + c.Anchor = "" + c.Content = make([]*yaml.Node, len(n.Content)) + for i, ch := range n.Content { + var err error + if c.Content[i], err = expand(ch, viaAlias); err != nil { + return nil, err + } + } + return &c, nil + } + out := make(map[string]yaml.Node, len(m)) + for k, v := range m { + e, err := expand(&v, false) + if err != nil { + return nil, err + } + out[k] = *e + } + return out, nil +} diff --git a/internal/virtual/helm_merger.go b/internal/virtual/helm_merger.go index 94641d7..e7d9802 100644 --- a/internal/virtual/helm_merger.go +++ b/internal/virtual/helm_merger.go @@ -2,10 +2,14 @@ package virtual import ( "fmt" + "log/slog" + "maps" + "slices" "strings" "gopkg.in/yaml.v3" + "git.unkin.net/unkin/artifactapi/internal/provider" "git.unkin.net/unkin/artifactapi/pkg/models" ) @@ -16,22 +20,17 @@ func init() { type HelmMerger struct{} type helmIndex struct { - APIVersion string `yaml:"apiVersion"` - Entries map[string][]helmChartVersion `yaml:"entries"` - Generated string `yaml:"generated,omitempty"` -} - -type helmChartVersion struct { - Name string `yaml:"name"` - Version string `yaml:"version"` - URLs []string `yaml:"urls"` - rest map[string]any + APIVersion string `yaml:"apiVersion"` + Entries map[string][]map[string]yaml.Node `yaml:"entries"` + Generated string `yaml:"generated,omitempty"` } +// MergeIndexes keeps each chart version's fields as raw nodes so digest, +// apiVersion and the rest survive the merge untouched; only urls are rewritten. func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) { merged := &helmIndex{ APIVersion: "v1", - Entries: make(map[string][]helmChartVersion), + Entries: make(map[string][]map[string]yaml.Node), } seen := map[string]map[string]bool{} @@ -42,47 +41,38 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([ continue } - for chart, versions := range idx.Entries { + budget := provider.MaxAliasExpandedNodes + for _, chart := range slices.Sorted(maps.Keys(idx.Entries)) { + versions := idx.Entries[chart] if seen[chart] == nil { seen[chart] = map[string]bool{} } - for _, ver := range versions { - key := chart + ":" + ver.Version - if seen[chart][ver.Version] { + for _, raw := range versions { + ver, err := provider.ExpandYAMLAliases(raw, &budget) + if err != nil { + slog.Warn("helm merge: dropping chart entry", "member", member.RemoteName, "chart", chart, "error", err) continue } - seen[chart][ver.Version] = true + version := ver["version"].Value + if seen[chart][version] { + continue + } + seen[chart][version] = true - if proxyBaseURL != "" { - routePrefix := "remote" - if member.RepoType == "local" { - routePrefix = "local" + if urlsNode, ok := ver["urls"]; ok && proxyBaseURL != "" { + var urls []string + if err := urlsNode.Decode(&urls); err != nil { + continue } - baseHost := extractHost(member.BaseURL) - - for i, u := range ver.URLs { - if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") { - if baseHost != "" && extractHost(u) != baseHost { - continue - } - relPath := extractPathRelativeToBase(u, member.BaseURL) - ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s", - strings.TrimRight(proxyBaseURL, "/"), - routePrefix, - member.RemoteName, - relPath) - } else { - ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s", - strings.TrimRight(proxyBaseURL, "/"), - routePrefix, - member.RemoteName, - u) - } + rewriteHelmURLs(urls, member, proxyBaseURL) + var n yaml.Node + if err := n.Encode(urls); err != nil { + return nil, err } + ver["urls"] = n } merged.Entries[chart] = append(merged.Entries[chart], ver) - _ = key } } } @@ -90,6 +80,34 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([ return yaml.Marshal(merged) } +func rewriteHelmURLs(urls []string, member MemberIndex, proxyBaseURL string) { + routePrefix := "remote" + if member.RepoType == "local" { + routePrefix = "local" + } + baseHost := extractHost(member.BaseURL) + + for i, u := range urls { + if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") { + if baseHost != "" && extractHost(u) != baseHost { + continue + } + relPath := extractPathRelativeToBase(u, member.BaseURL) + urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s", + strings.TrimRight(proxyBaseURL, "/"), + routePrefix, + member.RemoteName, + relPath) + } else { + urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s", + strings.TrimRight(proxyBaseURL, "/"), + routePrefix, + member.RemoteName, + u) + } + } +} + func extractHost(rawURL string) string { idx := strings.Index(rawURL, "://") if idx == -1 { diff --git a/internal/virtual/helm_merger_test.go b/internal/virtual/helm_merger_test.go index 857074d..c7908be 100644 --- a/internal/virtual/helm_merger_test.go +++ b/internal/virtual/helm_merger_test.go @@ -1,9 +1,12 @@ package virtual_test import ( + "fmt" "strings" "testing" + "gopkg.in/yaml.v3" + "git.unkin.net/unkin/artifactapi/internal/virtual" ) @@ -122,3 +125,129 @@ entries: t.Error("expected fallback repo to be excluded for duplicate") } } + +func TestHelmMerger_PreservesChartFields(t *testing.T) { + m := &virtual.HelmMerger{} + body := []byte(`apiVersion: v1 +entries: + app: + - apiVersion: v2 + name: app + version: "1.0" + appVersion: "3.1" + description: in-house + digest: deadbeef + urls: + - app-1.0.tgz +`) + result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "charts", RepoType: "local", Body: body}}, "https://proxy.example.com") + if err != nil { + t.Fatal(err) + } + out := string(result) + for _, want := range []string{ + "digest: deadbeef", + "apiVersion: v2", + `appVersion: "3.1"`, + "description: in-house", + `version: "1.0"`, + "https://proxy.example.com/api/v1/local/charts/app-1.0.tgz", + } { + if !strings.Contains(out, want) { + t.Errorf("merged index missing %q:\n%s", want, out) + } + } +} + +func TestHelmMerger_ExpandsAliasesAcrossDedupe(t *testing.T) { + m := &virtual.HelmMerger{} + first := []byte(`apiVersion: v1 +entries: + app: + - version: "1.0" + urls: [app-1.0.tgz] +`) + second := []byte(`apiVersion: v1 +entries: + app: + - version: "1.0" + maintainers: &team + - name: ops + urls: [app-1.0.tgz] + - version: "2.0" + maintainers: *team + urls: [app-2.0.tgz] +`) + result, err := m.MergeIndexes([]virtual.MemberIndex{ + {RemoteName: "a", RepoType: "local", Body: first}, + {RemoteName: "b", RepoType: "local", Body: second}, + }, "https://proxy.example.com") + if err != nil { + t.Fatal(err) + } + var idx struct { + Entries map[string][]struct { + Version string `yaml:"version"` + Maintainers []struct { + Name string `yaml:"name"` + } `yaml:"maintainers"` + } `yaml:"entries"` + } + if err := yaml.Unmarshal(result, &idx); err != nil { + t.Fatalf("merged index does not parse: %v\n%s", err, result) + } + app := idx.Entries["app"] + if len(app) != 2 || app[1].Version != "2.0" || len(app[1].Maintainers) != 1 || app[1].Maintainers[0].Name != "ops" { + t.Errorf("alias not expanded: %+v\n%s", app, result) + } +} + +func TestHelmMerger_DropsAliasBomb(t *testing.T) { + m := &virtual.HelmMerger{} + body := []byte(`apiVersion: v1 +entries: + app: + - version: "1.0" + a: &a [x, x, x, x, x, x, x, x, x, x] + b: &b [*a, *a, *a, *a, *a, *a, *a, *a, *a, *a] + c: &c [*b, *b, *b, *b, *b, *b, *b, *b, *b, *b] + d: &d [*c, *c, *c, *c, *c, *c, *c, *c, *c, *c] + e: &e [*d, *d, *d, *d, *d, *d, *d, *d, *d, *d] + urls: [app-1.0.tgz] +`) + result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "a", RepoType: "local", Body: body}}, "https://proxy.example.com") + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(result), "app-1.0.tgz") { + t.Errorf("alias bomb entry was kept") + } +} + +func TestHelmMerger_LargeIndexWithoutAliasesIsComplete(t *testing.T) { + var b strings.Builder + b.WriteString("apiVersion: v1\nentries:\n") + for c := 0; c < 100; c++ { + fmt.Fprintf(&b, " chart%d:\n", c) + for v := 0; v < 100; v++ { + fmt.Fprintf(&b, " - name: chart%d\n version: \"1.0.%d\"\n description: d\n maintainers: [{name: ops, email: ops@example.com}]\n urls: [chart%d-1.0.%d.tgz]\n", c, v, c, v) + } + } + result, err := (&virtual.HelmMerger{}).MergeIndexes([]virtual.MemberIndex{{RemoteName: "a", RepoType: "local", Body: []byte(b.String())}}, "https://proxy.example.com") + if err != nil { + t.Fatal(err) + } + var idx struct { + Entries map[string][]map[string]any `yaml:"entries"` + } + if err := yaml.Unmarshal(result, &idx); err != nil { + t.Fatal(err) + } + total := 0 + for _, vs := range idx.Entries { + total += len(vs) + } + if len(idx.Entries) != 100 || total != 10000 { + t.Errorf("got %d charts, %d versions; want 100, 10000", len(idx.Entries), total) + } +} diff --git a/migrations/0005_helm_metadata.sql b/migrations/0005_helm_metadata.sql new file mode 100644 index 0000000..cd20389 --- /dev/null +++ b/migrations/0005_helm_metadata.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS helm_metadata ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + chart_yaml TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_helm_metadata_repo ON helm_metadata(repo_name); diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index dd66d22..3d902b9 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -167,12 +167,23 @@ docker push ${host}/${name}/myapp:latest`, case 'helm': return [ { - title: 'Add the Helm repo', + title: isLocal ? 'Add the Helm repo (index.yaml auto-generated)' : 'Add the Helm repo', language: 'bash', - code: `helm repo add ${name} ${proxy}/ + code: `helm repo add ${name} ${isLocal ? `${url}/api/v1/local/${name}` : proxy}/ helm repo update helm install ${name}/`, }, + ...(isLocal + ? [ + { + title: 'Publish a chart', + language: 'bash', + code: `helm package ./mychart +curl -fsSL --upload-file mychart-0.1.0.tgz \\ + ${url}/api/v2/remotes/${name}/files/mychart-0.1.0.tgz`, + }, + ] + : []), ]; case 'alpine': From 6e6481e09a6f247d0dd6bcfb5b2fcb91c64c2647 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 15:54:47 +1100 Subject: [PATCH 4/4] Split per-type registration points into per-type files (#150) Parallel package-type PRs conflict because every type edits the same shared lines (README tables, provider.go, server.go imports, the package type list, UI maps, the e2e caching table). A new type now only adds files. - move per-type README sections into docs//{remote,local,virtual}.md - split provider.go capability and metadata types into cap_*.go and metadata_*.go - import providers via internal/provider/all, one file per type - declare each package type in package_type_.go, registered via init() - split the e2e caching cases and UI colours/snippets into per-type files Reviewed-on: https://git.unkin.net/unkin/artifactapi/pulls/150 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- PLAN.md | 2 +- README.md | 234 ++------------- docs/alpine/remote-github.md | 7 + docs/alpine/remote.md | 7 + docs/cargo/remote.md | 7 + docs/common/caching.md | 100 +++++++ docs/deb/remote-github.md | 7 + docs/deb/remote.md | 7 + docs/docker/local.md | 17 ++ docs/docker/remote.md | 26 ++ docs/generic/remote.md | 28 ++ docs/goproxy/remote.md | 7 + docs/helm/local.md | 15 + docs/helm/remote.md | 19 ++ docs/helm/virtual.md | 9 + docs/npm/remote.md | 7 + docs/nuget/remote.md | 7 + docs/puppet/local.md | 14 + docs/puppet/remote.md | 7 + docs/pypi/remote.md | 7 + docs/rpm/remote-github.md | 82 ++++++ docs/rpm/remote.md | 7 + docs/terraform/local.md | 29 ++ docs/terraform/remote.md | 7 + e2e-docker/README.md | 5 +- e2e-docker/caching_alpine_test.go | 7 + e2e-docker/caching_cargo_test.go | 7 + e2e-docker/caching_docker_test.go | 7 + e2e-docker/caching_generic_test.go | 7 + e2e-docker/caching_goproxy_test.go | 7 + e2e-docker/caching_helm_test.go | 7 + e2e-docker/caching_npm_test.go | 7 + e2e-docker/caching_nuget_test.go | 7 + e2e-docker/caching_puppet_test.go | 7 + e2e-docker/caching_pypi_test.go | 7 + e2e-docker/caching_rpm_test.go | 7 + e2e-docker/caching_terraform_test.go | 7 + e2e-docker/caching_test.go | 36 +-- internal/provider/all/alpine.go | 3 + internal/provider/all/cargo.go | 3 + internal/provider/all/deb.go | 3 + internal/provider/all/docker.go | 3 + internal/provider/all/generic.go | 3 + internal/provider/all/goproxy.go | 3 + internal/provider/all/helm.go | 3 + internal/provider/all/npm.go | 3 + internal/provider/all/nuget.go | 3 + internal/provider/all/puppet.go | 3 + internal/provider/all/pypi.go | 3 + internal/provider/all/rpm.go | 3 + internal/provider/all/terraform.go | 3 + internal/provider/cap_file_store.go | 15 + internal/provider/cap_index_merger.go | 10 + internal/provider/cap_local_indexer.go | 11 + internal/provider/cap_local_uploader.go | 6 + internal/provider/cap_post_delete_hook.go | 11 + internal/provider/cap_post_upload_hook.go | 9 + internal/provider/cap_remote_server.go | 25 ++ internal/provider/cap_serve_rewriter.go | 12 + internal/provider/metadata_alpine.go | 49 +++ internal/provider/metadata_deb.go | 34 +++ internal/provider/metadata_helm.go | 30 ++ internal/provider/metadata_puppet.go | 32 ++ internal/provider/metadata_rpm.go | 60 ++++ internal/provider/metadata_store.go | 15 + internal/provider/provider.go | 251 ---------------- internal/server/server.go | 11 +- pkg/models/package_type.go | 38 +-- pkg/models/package_type_alpine.go | 5 + pkg/models/package_type_cargo.go | 5 + pkg/models/package_type_deb.go | 5 + pkg/models/package_type_docker.go | 5 + pkg/models/package_type_generic.go | 5 + pkg/models/package_type_github_alpine.go | 5 + pkg/models/package_type_github_deb.go | 5 + pkg/models/package_type_github_rpm.go | 5 + pkg/models/package_type_goproxy.go | 5 + pkg/models/package_type_helm.go | 5 + pkg/models/package_type_npm.go | 5 + pkg/models/package_type_nuget.go | 5 + pkg/models/package_type_puppet.go | 5 + pkg/models/package_type_pypi.go | 5 + pkg/models/package_type_rpm.go | 5 + pkg/models/package_type_terraform.go | 5 + pkg/models/package_type_test.go | 59 ++-- ui/src/components/UsageInstructions.tsx | 344 +--------------------- ui/src/packagetypes/defs/alpine.ts | 37 +++ ui/src/packagetypes/defs/cargo.ts | 22 ++ ui/src/packagetypes/defs/deb.ts | 35 +++ ui/src/packagetypes/defs/docker.ts | 28 ++ ui/src/packagetypes/defs/generic.ts | 30 ++ ui/src/packagetypes/defs/github_alpine.ts | 18 ++ ui/src/packagetypes/defs/github_deb.ts | 18 ++ ui/src/packagetypes/defs/goproxy.ts | 18 ++ ui/src/packagetypes/defs/helm.ts | 29 ++ ui/src/packagetypes/defs/npm.ts | 23 ++ ui/src/packagetypes/defs/nuget.ts | 23 ++ ui/src/packagetypes/defs/puppet.ts | 27 ++ ui/src/packagetypes/defs/pypi.ts | 27 ++ ui/src/packagetypes/defs/rpm.ts | 40 +++ ui/src/packagetypes/defs/terraform.ts | 34 +++ ui/src/packagetypes/index.ts | 11 + ui/src/packagetypes/types.ts | 27 ++ ui/src/pages/Locals.tsx | 18 +- ui/src/pages/Remotes.tsx | 18 +- 105 files changed, 1481 insertions(+), 927 deletions(-) create mode 100644 docs/alpine/remote-github.md create mode 100644 docs/alpine/remote.md create mode 100644 docs/cargo/remote.md create mode 100644 docs/common/caching.md create mode 100644 docs/deb/remote-github.md create mode 100644 docs/deb/remote.md create mode 100644 docs/docker/local.md create mode 100644 docs/docker/remote.md create mode 100644 docs/generic/remote.md create mode 100644 docs/goproxy/remote.md create mode 100644 docs/helm/local.md create mode 100644 docs/helm/remote.md create mode 100644 docs/helm/virtual.md create mode 100644 docs/npm/remote.md create mode 100644 docs/nuget/remote.md create mode 100644 docs/puppet/local.md create mode 100644 docs/puppet/remote.md create mode 100644 docs/pypi/remote.md create mode 100644 docs/rpm/remote-github.md create mode 100644 docs/rpm/remote.md create mode 100644 docs/terraform/local.md create mode 100644 docs/terraform/remote.md create mode 100644 e2e-docker/caching_alpine_test.go create mode 100644 e2e-docker/caching_cargo_test.go create mode 100644 e2e-docker/caching_docker_test.go create mode 100644 e2e-docker/caching_generic_test.go create mode 100644 e2e-docker/caching_goproxy_test.go create mode 100644 e2e-docker/caching_helm_test.go create mode 100644 e2e-docker/caching_npm_test.go create mode 100644 e2e-docker/caching_nuget_test.go create mode 100644 e2e-docker/caching_puppet_test.go create mode 100644 e2e-docker/caching_pypi_test.go create mode 100644 e2e-docker/caching_rpm_test.go create mode 100644 e2e-docker/caching_terraform_test.go create mode 100644 internal/provider/all/alpine.go create mode 100644 internal/provider/all/cargo.go create mode 100644 internal/provider/all/deb.go create mode 100644 internal/provider/all/docker.go create mode 100644 internal/provider/all/generic.go create mode 100644 internal/provider/all/goproxy.go create mode 100644 internal/provider/all/helm.go create mode 100644 internal/provider/all/npm.go create mode 100644 internal/provider/all/nuget.go create mode 100644 internal/provider/all/puppet.go create mode 100644 internal/provider/all/pypi.go create mode 100644 internal/provider/all/rpm.go create mode 100644 internal/provider/all/terraform.go create mode 100644 internal/provider/cap_file_store.go create mode 100644 internal/provider/cap_index_merger.go create mode 100644 internal/provider/cap_local_indexer.go create mode 100644 internal/provider/cap_local_uploader.go create mode 100644 internal/provider/cap_post_delete_hook.go create mode 100644 internal/provider/cap_post_upload_hook.go create mode 100644 internal/provider/cap_remote_server.go create mode 100644 internal/provider/cap_serve_rewriter.go create mode 100644 internal/provider/metadata_alpine.go create mode 100644 internal/provider/metadata_deb.go create mode 100644 internal/provider/metadata_helm.go create mode 100644 internal/provider/metadata_puppet.go create mode 100644 internal/provider/metadata_rpm.go create mode 100644 internal/provider/metadata_store.go create mode 100644 pkg/models/package_type_alpine.go create mode 100644 pkg/models/package_type_cargo.go create mode 100644 pkg/models/package_type_deb.go create mode 100644 pkg/models/package_type_docker.go create mode 100644 pkg/models/package_type_generic.go create mode 100644 pkg/models/package_type_github_alpine.go create mode 100644 pkg/models/package_type_github_deb.go create mode 100644 pkg/models/package_type_github_rpm.go create mode 100644 pkg/models/package_type_goproxy.go create mode 100644 pkg/models/package_type_helm.go create mode 100644 pkg/models/package_type_npm.go create mode 100644 pkg/models/package_type_nuget.go create mode 100644 pkg/models/package_type_puppet.go create mode 100644 pkg/models/package_type_pypi.go create mode 100644 pkg/models/package_type_rpm.go create mode 100644 pkg/models/package_type_terraform.go create mode 100644 ui/src/packagetypes/defs/alpine.ts create mode 100644 ui/src/packagetypes/defs/cargo.ts create mode 100644 ui/src/packagetypes/defs/deb.ts create mode 100644 ui/src/packagetypes/defs/docker.ts create mode 100644 ui/src/packagetypes/defs/generic.ts create mode 100644 ui/src/packagetypes/defs/github_alpine.ts create mode 100644 ui/src/packagetypes/defs/github_deb.ts create mode 100644 ui/src/packagetypes/defs/goproxy.ts create mode 100644 ui/src/packagetypes/defs/helm.ts create mode 100644 ui/src/packagetypes/defs/npm.ts create mode 100644 ui/src/packagetypes/defs/nuget.ts create mode 100644 ui/src/packagetypes/defs/puppet.ts create mode 100644 ui/src/packagetypes/defs/pypi.ts create mode 100644 ui/src/packagetypes/defs/rpm.ts create mode 100644 ui/src/packagetypes/defs/terraform.ts create mode 100644 ui/src/packagetypes/index.ts create mode 100644 ui/src/packagetypes/types.ts diff --git a/PLAN.md b/PLAN.md index f7bd97c..cd490a2 100644 --- a/PLAN.md +++ b/PLAN.md @@ -672,7 +672,7 @@ func TestMain(m *testing.M) { ```hcl provider "artifactapi" { - endpoint = "https://artifactapi.k8s.syd1.au.unkin.net" + endpoint = "https://artifactapi.example.com" } resource "artifactapi_remote" "terraform_registry" { diff --git a/README.md b/README.md index 48e3545..4f0a78c 100644 --- a/README.md +++ b/README.md @@ -20,98 +20,25 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173` ## Package Types -| Type | Mutable (auto-detected) | Immutable (auto-detected) | -|---|---|---| -| `generic` | nothing | everything | -| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | -| `helm` | `index.yaml` | `.tgz` charts | -| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | -| `npm` | package metadata | `.tgz` tarballs | -| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | -| `alpine` | `APKINDEX.tar.gz` | `.apk` | -| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | -| `terraform` | `*/versions` | `*/download/*/*` | -| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | -| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | -| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | -| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | - Providers classify paths automatically. Users only configure what to proxy and TTLs. -### `github_rpm` — GitHub releases as a yum repo (metadata-only, no precache) +Classification precedence, TTLs and eviction for every remote: [docs/common/caching.md](docs/common/caching.md). -A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` -repository without ever caching the packages. It scans releases for `.rpm` -assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ -obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package -metadata comes from a **ranged GET of just the RPM header** (the header sits at -the front of the file, so the whole package is never downloaded); the sha256 -checksum comes from the GitHub asset `digest` when present, else a one-time -lazy stream. Derived metadata is cached (keyed by asset) so repodata generation -is served from primed DB rows, never a cold on-demand derive. +Each type's paths, Terraform examples and local/virtual behaviour are in `docs//`: -Each package's `` points back at the remote, which **302-redirects** -the download to the `releases_remote` — an existing generic `github.com` remote -that streams the actual bytes. `dnf` follows the redirect transparently. - -#### Background syncer - -A single process-wide **background syncer** keeps every `github_rpm` remote's -derived metadata current off the client request path: - -- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime - scan, so its metadata is derived right away without blocking the create call. - The first `dnf` request is served from cache. If a request arrives before the - prime lands, it returns a retryable `503` (with `Retry-After`) rather than - serving an empty repo or blocking on a multi-minute derive. -- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for - new or changed releases no more often than its `mutable_ttl`. New/changed - assets are derived incrementally; assets already cached are never re-fetched, - and assets that disappear upstream are pruned. -- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per - remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed - and the syncer derives nothing. GitHub does not count `304` conditional - responses against the rate limit, so an unchanged repo is nearly free — this is - the main lever keeping GitHub traffic low. -- **Global rate limit.** Every GitHub call (releases list + each ranged asset - header GET) passes through a single token-bucket limiter **shared across all - remotes**, so GitHub is never hammered. Configure a token (`password`) on the - remote for the higher authenticated rate limit (~5000/hr vs ~60/hr - unauthenticated). -- **Multi-replica coordination.** State is shared through the database. Before a - periodic scan a replica must atomically claim a per-remote lease - (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, - `sync_lease_expires`); only the winner scans. This bounds total GitHub load to - ~once per `mutable_ttl` regardless of replica count, and the shared `etag` - lets any replica issue the conditional request. - -```hcl -# Backend that serves the actual .rpm bytes from github.com. -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - patterns = [ - "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets - ] -} - -resource "artifactapi_remote_github_rpm" "acme-tools" { - name = "acme-tools" - base_url = "https://api.github.com/repos/acme/tools" # the releases API root - releases_remote = "github" # backend for downloads - mutable_ttl = 3600 # release re-scan interval - - # Optional: restrict which release assets become packages (regex on filename). - patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] - - # Optional: a token for private repos / higher API rate limits. - # password = "ghp_..." -} -``` - -`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. -The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages -from the synthesized metadata. +- `alpine`: [remote](docs/alpine/remote.md), [remote-github](docs/alpine/remote-github.md) +- `cargo`: [remote](docs/cargo/remote.md) +- `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) +- `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) +- `generic`: [remote](docs/generic/remote.md) +- `goproxy`: [remote](docs/goproxy/remote.md) +- `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) +- `npm`: [remote](docs/npm/remote.md) +- `nuget`: [remote](docs/nuget/remote.md) +- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md) +- `pypi`: [remote](docs/pypi/remote.md) +- `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md) +- `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md) ### GitHub authentication @@ -180,136 +107,9 @@ GITHUB_APP_PRIVATE_KEY_PATH=/etc/artifactapi/github-app.pem ## Terraform -Remotes and virtuals are managed by Terraform. Each package type has its own resource: +Remotes and virtuals are managed by Terraform. Each package type has its own resource; see `docs//` for examples. -```hcl -resource "artifactapi_remote_generic" "github" { - name = "github" - base_url = "https://github.com" - - immutable_ttl = 0 - mutable_ttl = 7200 - - patterns = [ - "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", - "mikefarah/yq/.*/yq_linux_amd64$", - ] - - mutable_patterns = [ - ".*/archive/refs/heads/.*\\.tar\\.gz$", - ] -} - -resource "artifactapi_remote_docker" "dockerhub" { - name = "dockerhub" - base_url = "https://registry-1.docker.io" - - immutable_ttl = 0 - mutable_ttl = 300 - ban_tags_enabled = true - ban_tags = ["latest"] - - patterns = [ - "^library/postgres", - "^library/redis", - ] -} - -resource "artifactapi_remote_helm" "jetstack" { - name = "jetstack" - base_url = "https://charts.jetstack.io" - - immutable_ttl = 0 - mutable_ttl = 3600 -} - -resource "artifactapi_virtual" "helm" { - name = "helm" - package_type = "helm" - members = [artifactapi_remote_helm.jetstack.name] -} -``` - -Provider: [terraform-provider-artifactapi](../terraform-provider-artifactapi) - -### Serving providers as a registry - -A local `terraform` repo is a real provider registry: upload -`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under -`{namespace}/{type}/`, and Terraform installs them from a bare source address — -no `.terraformrc` mirror config: - -```hcl -terraform { - required_providers { - artifactapi = { - source = "artifactapi.k8s.syd1.au.unkin.net//" - version = "0.1.2" - } - } -} -``` - -The Terraform *namespace* segment is the artifactapi repo name; the provider is -matched by *type*. The registry serves service discovery -(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, -and a GPG-signed `SHA256SUMS` per the provider registry protocol. - -Signing needs a GPG key. By default artifactapi generates one on first start and -stores it in the database (`signing_keys` table), so every replica shares it and -there's nothing to provision. To bring your own key instead, point -`TF_SIGNING_KEY_PATH` at an armored private key (optionally -`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. -`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. - -### Local helm repository - -A local `helm` repo is a real chart repository: `PUT` a packaged chart to -`/api/v2/remotes//files/-.tgz` and `index.yaml` is -generated from each chart's `Chart.yaml` (with sha256 digests) at -`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` -name and version is stored but left out of the index. Add the local as a -member of a helm virtual to serve it alongside remotes. - -```bash -helm package ./mychart -curl -fsSL --upload-file mychart-0.1.0.tgz \ - https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/mychart-0.1.0.tgz -helm repo add https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ -``` - -### Local puppet forge - -A local `puppet` repo is a Forge: `PUT` a module tarball to -`/api/v2/remotes//files/--.tar.gz` and the Forge -v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each -release's `metadata.json` at `/api/v1/local/`. Versions are immutable. - -```bash -puppet module build ./hello -curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ - https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz -puppet module install unkin-hello \ - --module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ -``` - -### Local docker registry - -A local `docker` repo is a real container registry, not a mirror: it serves the -Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, -`podman`, `skopeo`, `buildah`) can use it directly. - -```sh -docker tag myapp:latest artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker push artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -docker pull artifactapi.k8s.syd1.au.unkin.net/docker-internal/myapp:latest -``` - -The first path segment after `/v2/` is the artifactapi repo name; the remainder -is the image name. Blobs and manifests are stored through the shared -content-addressable store (deduplicated by digest, reaped by GC once -unreferenced); tags are mutable references and re-pushing a tag moves it. Blob -uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. +Provider: [terraform-provider-artifactapi](https://git.unkin.net/unkin/terraform-provider-artifactapi) ## Access Control diff --git a/docs/alpine/remote-github.md b/docs/alpine/remote-github.md new file mode 100644 index 0000000..e9e473b --- /dev/null +++ b/docs/alpine/remote-github.md @@ -0,0 +1,7 @@ +# `github_alpine` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_alpine` | `APKINDEX.tar.gz` (synthesized) | `.apk` (redirected) | diff --git a/docs/alpine/remote.md b/docs/alpine/remote.md new file mode 100644 index 0000000..f415b81 --- /dev/null +++ b/docs/alpine/remote.md @@ -0,0 +1,7 @@ +# `alpine` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `alpine` | `APKINDEX.tar.gz` | `.apk` | diff --git a/docs/cargo/remote.md b/docs/cargo/remote.md new file mode 100644 index 0000000..d7bec9e --- /dev/null +++ b/docs/cargo/remote.md @@ -0,0 +1,7 @@ +# `cargo` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` | diff --git a/docs/common/caching.md b/docs/common/caching.md new file mode 100644 index 0000000..40d70e3 --- /dev/null +++ b/docs/common/caching.md @@ -0,0 +1,100 @@ +# Caching and classification (all remotes) + +Every proxied request on a remote is classified as **denied**, **immutable** or +**mutable**. The class picks the storage path and the freshness TTL. + +## Fields + +| Field | Meaning | +|---|---| +| `patterns` | Allowlist of regexes. Empty allows every path; otherwise a non-matching path is denied (403). | +| `blocklist` | Regexes; a match is denied (403). | +| `immutable_patterns` | Regexes forcing a path immutable. | +| `mutable_patterns` | Regexes forcing a path mutable. | +| `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. | +| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires, and the API stores `0` when the field is omitted. | +| `check_mutable` | Stores the upstream ETag and tries a conditional `HEAD` (`If-None-Match`) when a mutable path is fetched. The ETag expires with the `mutable_ttl` freshness key, so an expired mutable path is refetched in full, not revalidated. | +| `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. | +| `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. | +| `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. | + +## Precedence + +The first rule that matches wins: + +1. `blocklist` matches: **denied** +2. `patterns` is non-empty and nothing matches: **denied** +3. `immutable_patterns` matches: **immutable** +4. `mutable_patterns` matches: **mutable** +5. the provider's built-in `Classify` says mutable: **mutable** +6. anything else: **immutable** + +So `blocklist` beats the allowlist, and `immutable_patterns` beats +`mutable_patterns`. Either user list overrides the built-in rules. `GET` and +`HEAD` both apply this order. + +## Matching rules + +- Regexes are Go `regexp`, matched **unanchored** (`MatchString`). Add `^` or `$` yourself when you need them. +- They match the path after `/api/v1/remote//`, without a leading slash, e.g. `9/BaseOS/x86_64/os/repodata/repomd.xml`. +- The query string is not part of the path. It is dropped and not forwarded upstream. +- `create` and `update` reject an invalid regex in `patterns`, `blocklist`, `mutable_patterns`, `immutable_patterns` or `ban_tags` with a 400. Only invalid regexes already stored before that check are skipped at match time. + +## Lifecycles + +| | Immutable | Mutable | +|---|---|---| +| Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider | +| Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` | +| After expiry | refetched from upstream | refetched from upstream (the ETag expires with the TTL, so `check_mutable` does not revalidate it) | +| Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same | + +Concurrent misses on one path share a fetch lock. Waiters poll the store for +up to 5s instead of all going upstream. + +## Eviction + +```sh +curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects/ +curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects//* +``` + +- Evict drops the artifact row, the index object and the Redis TTL and ETag keys, so the next request goes upstream whatever the class or TTL. +- A single-path evict takes the fetch lock first, so an in-flight fetch can't re-cache the path. It returns 503 if the lock stays busy. +- A wildcard must end in `/*` and takes no lock. +- Returns 204, or 404 for an unknown remote. + +## Examples + +Bodies for `POST https://artifactapi.example.com/api/v2/remotes`. + +GitHub release assets through a `generic` remote, allowlisted: + +```json +{"name": "github", "package_type": "generic", "base_url": "https://github.com", + "immutable_ttl": 0, "mutable_ttl": 7200, + "patterns": ["^mikefarah/yq/releases/download/.*/yq_linux_amd64$"]} +``` + +rpm remote: `repodata/*` and `repomd.xml` are mutable by default, so only the TTLs are needed: + +```json +{"name": "almalinux", "package_type": "rpm", "base_url": "https://repo.almalinux.org/almalinux", + "immutable_ttl": 0, "mutable_ttl": 3600, "check_mutable": true, "stale_on_error": true} +``` + +Force a moving path mutable, e.g. branch tarballs on `generic`: + +```json +{"mutable_patterns": ["/archive/refs/heads/.*\\.tar\\.gz$"], "mutable_ttl": 600} +``` + +## Built-in classification per type + +Each type's page lists its auto-detected mutable and immutable paths: +[alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), +[cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md), +[docker](../docker/remote.md), [generic](../generic/remote.md), [goproxy](../goproxy/remote.md), +[helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md), +[puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md), +[rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md). diff --git a/docs/deb/remote-github.md b/docs/deb/remote-github.md new file mode 100644 index 0000000..99d9f11 --- /dev/null +++ b/docs/deb/remote-github.md @@ -0,0 +1,7 @@ +# `github_deb` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_deb` | `Packages*`, `Release*` (synthesized) | `.deb` (redirected) | diff --git a/docs/deb/remote.md b/docs/deb/remote.md new file mode 100644 index 0000000..865d949 --- /dev/null +++ b/docs/deb/remote.md @@ -0,0 +1,7 @@ +# `deb` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `deb` | `Packages*`, `Sources*`, `Release`, `InRelease`, `Release.gpg`, `Contents-*` | `.deb` | diff --git a/docs/docker/local.md b/docs/docker/local.md new file mode 100644 index 0000000..54d7a55 --- /dev/null +++ b/docs/docker/local.md @@ -0,0 +1,17 @@ +# Local docker registry + +A local `docker` repo is a real container registry, not a mirror: it serves the +Docker Registry HTTP API V2 for both push and pull, so any client (`docker`, +`podman`, `skopeo`, `buildah`) can use it directly. + +```sh +docker tag myapp:latest artifactapi.example.com/docker-internal/myapp:latest +docker push artifactapi.example.com/docker-internal/myapp:latest +docker pull artifactapi.example.com/docker-internal/myapp:latest +``` + +The first path segment after `/v2/` is the artifactapi repo name; the remainder +is the image name. Blobs and manifests are stored through the shared +content-addressable store (deduplicated by digest, reaped by GC once +unreferenced); tags are mutable references and re-pushing a tag moves it. Blob +uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. diff --git a/docs/docker/remote.md b/docs/docker/remote.md new file mode 100644 index 0000000..c8a79f1 --- /dev/null +++ b/docs/docker/remote.md @@ -0,0 +1,26 @@ +# `docker` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `docker` | tag manifests, `/tags/list` | blobs, digest manifests | + +## Terraform + +```hcl +resource "artifactapi_remote_docker" "dockerhub" { + name = "dockerhub" + base_url = "https://registry-1.docker.io" + + immutable_ttl = 0 + mutable_ttl = 300 + ban_tags_enabled = true + ban_tags = ["latest"] + + patterns = [ + "^library/postgres", + "^library/redis", + ] +} +``` diff --git a/docs/generic/remote.md b/docs/generic/remote.md new file mode 100644 index 0000000..7fac792 --- /dev/null +++ b/docs/generic/remote.md @@ -0,0 +1,28 @@ +# `generic` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `generic` | nothing | everything | + +## Terraform + +```hcl +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + + immutable_ttl = 0 + mutable_ttl = 7200 + + patterns = [ + "ducaale/xh/.*/xh-.*-x86_64-unknown-linux-musl.tar.gz$", + "mikefarah/yq/.*/yq_linux_amd64$", + ] + + mutable_patterns = [ + ".*/archive/refs/heads/.*\\.tar\\.gz$", + ] +} +``` diff --git a/docs/goproxy/remote.md b/docs/goproxy/remote.md new file mode 100644 index 0000000..52d1c49 --- /dev/null +++ b/docs/goproxy/remote.md @@ -0,0 +1,7 @@ +# `goproxy` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` | diff --git a/docs/helm/local.md b/docs/helm/local.md new file mode 100644 index 0000000..898ff88 --- /dev/null +++ b/docs/helm/local.md @@ -0,0 +1,15 @@ +# Local helm repository + +A local `helm` repo is a real chart repository: `PUT` a packaged chart to +`/api/v2/remotes//files/-.tgz` and `index.yaml` is +generated from each chart's `Chart.yaml` (with sha256 digests) at +`/api/v1/local//index.yaml`; a file not named after its `Chart.yaml` +name and version is stored but left out of the index. Add the local as a +member of a helm virtual to serve it alongside remotes. + +```bash +helm package ./mychart +curl -fsSL --upload-file mychart-0.1.0.tgz \ + https://artifactapi.example.com/api/v2/remotes//files/mychart-0.1.0.tgz +helm repo add https://artifactapi.example.com/api/v1/local/ +``` diff --git a/docs/helm/remote.md b/docs/helm/remote.md new file mode 100644 index 0000000..b51e028 --- /dev/null +++ b/docs/helm/remote.md @@ -0,0 +1,19 @@ +# `helm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `helm` | `index.yaml` | `.tgz` charts | + +## Terraform + +```hcl +resource "artifactapi_remote_helm" "jetstack" { + name = "jetstack" + base_url = "https://charts.jetstack.io" + + immutable_ttl = 0 + mutable_ttl = 3600 +} +``` diff --git a/docs/helm/virtual.md b/docs/helm/virtual.md new file mode 100644 index 0000000..91a56b0 --- /dev/null +++ b/docs/helm/virtual.md @@ -0,0 +1,9 @@ +# `helm` virtual + +```hcl +resource "artifactapi_virtual" "helm" { + name = "helm" + package_type = "helm" + members = [artifactapi_remote_helm.jetstack.name] +} +``` diff --git a/docs/npm/remote.md b/docs/npm/remote.md new file mode 100644 index 0000000..8559315 --- /dev/null +++ b/docs/npm/remote.md @@ -0,0 +1,7 @@ +# `npm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `npm` | package metadata | `.tgz` tarballs | diff --git a/docs/nuget/remote.md b/docs/nuget/remote.md new file mode 100644 index 0000000..ba0109a --- /dev/null +++ b/docs/nuget/remote.md @@ -0,0 +1,7 @@ +# `nuget` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` | diff --git a/docs/puppet/local.md b/docs/puppet/local.md new file mode 100644 index 0000000..c5258d0 --- /dev/null +++ b/docs/puppet/local.md @@ -0,0 +1,14 @@ +# Local puppet forge + +A local `puppet` repo is a Forge: `PUT` a module tarball to +`/api/v2/remotes//files/--.tar.gz` and the Forge +v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each +release's `metadata.json` at `/api/v1/local/`. Versions are immutable. + +```bash +puppet module build ./hello +curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ + https://artifactapi.example.com/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz +puppet module install unkin-hello \ + --module_repository https://artifactapi.example.com/api/v1/local/ +``` diff --git a/docs/puppet/remote.md b/docs/puppet/remote.md new file mode 100644 index 0000000..8c7435e --- /dev/null +++ b/docs/puppet/remote.md @@ -0,0 +1,7 @@ +# `puppet` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `puppet` | `v3/modules/*`, `v3/releases*` | `.tar.gz` | diff --git a/docs/pypi/remote.md b/docs/pypi/remote.md new file mode 100644 index 0000000..0f5fe32 --- /dev/null +++ b/docs/pypi/remote.md @@ -0,0 +1,7 @@ +# `pypi` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `pypi` | `simple/*` index pages | `.whl`, `.tar.gz` | diff --git a/docs/rpm/remote-github.md b/docs/rpm/remote-github.md new file mode 100644 index 0000000..f079b02 --- /dev/null +++ b/docs/rpm/remote-github.md @@ -0,0 +1,82 @@ +# `github_rpm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) | + +## GitHub releases as a yum repo (metadata-only, no precache) + +A `github_rpm` remote turns a GitHub repo's **releases** into a real `dnf`/`yum` +repository without ever caching the packages. It scans releases for `.rpm` +assets, derives each package's metadata (NEVRA, requires/provides/conflicts/ +obsoletes, files, checksum) and **synthesizes `repodata/` on the fly**. Package +metadata comes from a **ranged GET of just the RPM header** (the header sits at +the front of the file, so the whole package is never downloaded); the sha256 +checksum comes from the GitHub asset `digest` when present, else a one-time +lazy stream. Derived metadata is cached (keyed by asset) so repodata generation +is served from primed DB rows, never a cold on-demand derive. + +Each package's `` points back at the remote, which **302-redirects** +the download to the `releases_remote` — an existing generic `github.com` remote +that streams the actual bytes. `dnf` follows the redirect transparently. + +### Background syncer + +A single process-wide **background syncer** keeps every `github_rpm` remote's +derived metadata current off the client request path: + +- **Prime on create.** Creating a `github_rpm` remote enqueues a background prime + scan, so its metadata is derived right away without blocking the create call. + The first `dnf` request is served from cache. If a request arrives before the + prime lands, it returns a retryable `503` (with `Retry-After`) rather than + serving an empty repo or blocking on a multi-minute derive. +- **Periodic re-check, driven by `mutable_ttl`.** Each remote is re-checked for + new or changed releases no more often than its `mutable_ttl`. New/changed + assets are derived incrementally; assets already cached are never re-fetched, + and assets that disappear upstream are pruned. +- **ETag / 304 conditional requests.** The releases-list `ETag` is stored per + remote and sent as `If-None-Match`; a `304 Not Modified` means nothing changed + and the syncer derives nothing. GitHub does not count `304` conditional + responses against the rate limit, so an unchanged repo is nearly free — this is + the main lever keeping GitHub traffic low. +- **Global rate limit.** Every GitHub call (releases list + each ranged asset + header GET) passes through a single token-bucket limiter **shared across all + remotes**, so GitHub is never hammered. Configure a token (`password`) on the + remote for the higher authenticated rate limit (~5000/hr vs ~60/hr + unauthenticated). +- **Multi-replica coordination.** State is shared through the database. Before a + periodic scan a replica must atomically claim a per-remote lease + (`github_rpm_sync_state`: `last_synced_at`, `etag`, `sync_lease_owner`, + `sync_lease_expires`); only the winner scans. This bounds total GitHub load to + ~once per `mutable_ttl` regardless of replica count, and the shared `etag` + lets any replica issue the conditional request. + +```hcl +# Backend that serves the actual .rpm bytes from github.com. +resource "artifactapi_remote_generic" "github" { + name = "github" + base_url = "https://github.com" + patterns = [ + "acme/tools/releases/download/.*\\.rpm$", # allowlist the repo's release assets + ] +} + +resource "artifactapi_remote_github_rpm" "acme-tools" { + name = "acme-tools" + base_url = "https://api.github.com/repos/acme/tools" # the releases API root + releases_remote = "github" # backend for downloads + mutable_ttl = 3600 # release re-scan interval + + # Optional: restrict which release assets become packages (regex on filename). + patterns = [".*\\.x86_64\\.rpm$", ".*\\.noarch\\.rpm$"] + + # Optional: a token for private repos / higher API rate limits. + # password = "ghp_..." +} +``` + +`dnf` config: `baseurl=https://artifactapi.example/api/v1/remote/acme-tools`. +The repo is multi-arch (no `$basearch` needed) — `dnf` selects matching packages +from the synthesized metadata. diff --git a/docs/rpm/remote.md b/docs/rpm/remote.md new file mode 100644 index 0000000..f4a93b2 --- /dev/null +++ b/docs/rpm/remote.md @@ -0,0 +1,7 @@ +# `rpm` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `rpm` | `repomd.xml`, `repodata/*` | `.rpm` | diff --git a/docs/terraform/local.md b/docs/terraform/local.md new file mode 100644 index 0000000..2b3e0d4 --- /dev/null +++ b/docs/terraform/local.md @@ -0,0 +1,29 @@ +# Serving providers as a registry + +A local `terraform` repo is a real provider registry: upload +`terraform-provider-{type}_{version}_{os}_{arch}.zip` files under +`{namespace}/{type}/`, and Terraform installs them from a bare source address — +no `.terraformrc` mirror config: + +```hcl +terraform { + required_providers { + artifactapi = { + source = "artifactapi.example.com//" + version = "0.1.2" + } + } +} +``` + +The Terraform *namespace* segment is the artifactapi repo name; the provider is +matched by *type*. The registry serves service discovery +(`/.well-known/terraform.json`), the `providers.v1` version/download endpoints, +and a GPG-signed `SHA256SUMS` per the provider registry protocol. + +Signing needs a GPG key. By default artifactapi generates one on first start and +stores it in the database (`signing_keys` table), so every replica shares it and +there's nothing to provision. To bring your own key instead, point +`TF_SIGNING_KEY_PATH` at an armored private key (optionally +`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. +`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. diff --git a/docs/terraform/remote.md b/docs/terraform/remote.md new file mode 100644 index 0000000..d4fe58b --- /dev/null +++ b/docs/terraform/remote.md @@ -0,0 +1,7 @@ +# `terraform` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `terraform` | `*/versions` | `*/download/*/*` | diff --git a/e2e-docker/README.md b/e2e-docker/README.md index facd807..70825fc 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -22,9 +22,8 @@ already-running stack. ## Coverage - **Repository lifecycle** — add / change / delete for remote, local and virtual repos. -- **Caching** — one immutable artifact per remote package type (generic, docker, - helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the - mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes +- **Caching** — one immutable artifact per remote package type (one + `caching__test.go` case each) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`), generic (upload/download), pypi (wheel + generated `simple/` diff --git a/e2e-docker/caching_alpine_test.go b/e2e-docker/caching_alpine_test.go new file mode 100644 index 0000000..d970f44 --- /dev/null +++ b/e2e-docker/caching_alpine_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}) +} diff --git a/e2e-docker/caching_cargo_test.go b/e2e-docker/caching_cargo_test.go new file mode 100644 index 0000000..e5d6e0d --- /dev/null +++ b/e2e-docker/caching_cargo_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}) +} diff --git a/e2e-docker/caching_docker_test.go b/e2e-docker/caching_docker_test.go new file mode 100644 index 0000000..87f980c --- /dev/null +++ b/e2e-docker/caching_docker_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}) +} diff --git a/e2e-docker/caching_generic_test.go b/e2e-docker/caching_generic_test.go new file mode 100644 index 0000000..ff4dc48 --- /dev/null +++ b/e2e-docker/caching_generic_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"generic", "blobs/hello.bin", "blobs/hello.bin"}) +} diff --git a/e2e-docker/caching_goproxy_test.go b/e2e-docker/caching_goproxy_test.go new file mode 100644 index 0000000..3482bae --- /dev/null +++ b/e2e-docker/caching_goproxy_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}) +} diff --git a/e2e-docker/caching_helm_test.go b/e2e-docker/caching_helm_test.go new file mode 100644 index 0000000..548f1f2 --- /dev/null +++ b/e2e-docker/caching_helm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_npm_test.go b/e2e-docker/caching_npm_test.go new file mode 100644 index 0000000..912fa85 --- /dev/null +++ b/e2e-docker/caching_npm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}) +} diff --git a/e2e-docker/caching_nuget_test.go b/e2e-docker/caching_nuget_test.go new file mode 100644 index 0000000..be8ee69 --- /dev/null +++ b/e2e-docker/caching_nuget_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}) +} diff --git a/e2e-docker/caching_puppet_test.go b/e2e-docker/caching_puppet_test.go new file mode 100644 index 0000000..74d2024 --- /dev/null +++ b/e2e-docker/caching_puppet_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}) +} diff --git a/e2e-docker/caching_pypi_test.go b/e2e-docker/caching_pypi_test.go new file mode 100644 index 0000000..3948f8e --- /dev/null +++ b/e2e-docker/caching_pypi_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}) +} diff --git a/e2e-docker/caching_rpm_test.go b/e2e-docker/caching_rpm_test.go new file mode 100644 index 0000000..d4ff226 --- /dev/null +++ b/e2e-docker/caching_rpm_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}) +} diff --git a/e2e-docker/caching_terraform_test.go b/e2e-docker/caching_terraform_test.go new file mode 100644 index 0000000..f280d76 --- /dev/null +++ b/e2e-docker/caching_terraform_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}) +} diff --git a/e2e-docker/caching_test.go b/e2e-docker/caching_test.go index 420d7f1..603ecd1 100644 --- a/e2e-docker/caching_test.go +++ b/e2e-docker/caching_test.go @@ -9,33 +9,23 @@ import ( "testing" ) +type cachingCase struct { + pkgType string + // path is the request path under /api/v1/remote//. The provider + // derives the upstream URL from it (docker prepends /v2/, terraform + // prepends /v1/providers/), and the fixture lives at that resolved path. + path string + fixture string +} + +// cachingCases is filled by one caching__test.go file per package type. +var cachingCases []cachingCase + // TestCachingPerProvider proxies one immutable artifact for every remote // package type through the mock upstream and asserts: first fetch is served // from the remote, the second from cache, and the bytes match the origin. func TestCachingPerProvider(t *testing.T) { - cases := []struct { - pkgType string - // path is the request path under /api/v1/remote//. The provider - // derives the upstream URL from it (docker prepends /v2/, terraform - // prepends /v1/providers/), and the fixture lives at that resolved path. - path string - fixture string - }{ - {"generic", "blobs/hello.bin", "blobs/hello.bin"}, - {"npm", "mypkg/-/mypkg-1.0.0.tgz", "mypkg/-/mypkg-1.0.0.tgz"}, - {"helm", "charts/mychart-1.0.0.tgz", "charts/mychart-1.0.0.tgz"}, - {"pypi", "packages/foo-1.0-py3-none-any.whl", "packages/foo-1.0-py3-none-any.whl"}, - {"rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm", "rpmrepo/Packages/e2e-testpkg-1.0-1.noarch.rpm"}, - {"alpine", "alpine/x86_64/testpkg-1.0-r0.apk", "alpine/x86_64/testpkg-1.0-r0.apk"}, - {"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"}, - {"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"}, - {"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"}, - {"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"}, - {"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"}, - {"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"}, - } - - for _, tc := range cases { + for _, tc := range cachingCases { t.Run(tc.pkgType, func(t *testing.T) { name := "cache-" + tc.pkgType createRepo(t, fmt.Sprintf(`{ diff --git a/internal/provider/all/alpine.go b/internal/provider/all/alpine.go new file mode 100644 index 0000000..dd4d0fd --- /dev/null +++ b/internal/provider/all/alpine.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/alpine" diff --git a/internal/provider/all/cargo.go b/internal/provider/all/cargo.go new file mode 100644 index 0000000..328b155 --- /dev/null +++ b/internal/provider/all/cargo.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/cargo" diff --git a/internal/provider/all/deb.go b/internal/provider/all/deb.go new file mode 100644 index 0000000..86b87c2 --- /dev/null +++ b/internal/provider/all/deb.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/deb" diff --git a/internal/provider/all/docker.go b/internal/provider/all/docker.go new file mode 100644 index 0000000..00ea7d9 --- /dev/null +++ b/internal/provider/all/docker.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/docker" diff --git a/internal/provider/all/generic.go b/internal/provider/all/generic.go new file mode 100644 index 0000000..a7c1a1c --- /dev/null +++ b/internal/provider/all/generic.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/generic" diff --git a/internal/provider/all/goproxy.go b/internal/provider/all/goproxy.go new file mode 100644 index 0000000..c8fb8d8 --- /dev/null +++ b/internal/provider/all/goproxy.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy" diff --git a/internal/provider/all/helm.go b/internal/provider/all/helm.go new file mode 100644 index 0000000..b42b787 --- /dev/null +++ b/internal/provider/all/helm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/helm" diff --git a/internal/provider/all/npm.go b/internal/provider/all/npm.go new file mode 100644 index 0000000..f71d089 --- /dev/null +++ b/internal/provider/all/npm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/npm" diff --git a/internal/provider/all/nuget.go b/internal/provider/all/nuget.go new file mode 100644 index 0000000..bdc0922 --- /dev/null +++ b/internal/provider/all/nuget.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/nuget" diff --git a/internal/provider/all/puppet.go b/internal/provider/all/puppet.go new file mode 100644 index 0000000..266e990 --- /dev/null +++ b/internal/provider/all/puppet.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/puppet" diff --git a/internal/provider/all/pypi.go b/internal/provider/all/pypi.go new file mode 100644 index 0000000..27a48a9 --- /dev/null +++ b/internal/provider/all/pypi.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/pypi" diff --git a/internal/provider/all/rpm.go b/internal/provider/all/rpm.go new file mode 100644 index 0000000..8ec6c17 --- /dev/null +++ b/internal/provider/all/rpm.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/rpm" diff --git a/internal/provider/all/terraform.go b/internal/provider/all/terraform.go new file mode 100644 index 0000000..39e7f75 --- /dev/null +++ b/internal/provider/all/terraform.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/terraform" diff --git a/internal/provider/cap_file_store.go b/internal/provider/cap_file_store.go new file mode 100644 index 0000000..a41540f --- /dev/null +++ b/internal/provider/cap_file_store.go @@ -0,0 +1,15 @@ +package provider + +import ( + "context" + "io" +) + +type FileStore interface { + ListFilesByPrefix(ctx context.Context, repoName, prefix string) ([]FileEntry, error) + ListPackages(ctx context.Context, repoName string) ([]string, error) +} + +type BlobReader interface { + Download(ctx context.Context, key string) (io.ReadCloser, int64, error) +} diff --git a/internal/provider/cap_index_merger.go b/internal/provider/cap_index_merger.go new file mode 100644 index 0000000..17eb6fc --- /dev/null +++ b/internal/provider/cap_index_merger.go @@ -0,0 +1,10 @@ +package provider + +type IndexMerger interface { + MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) +} + +type MemberIndex struct { + RemoteName string + Body []byte +} diff --git a/internal/provider/cap_local_indexer.go b/internal/provider/cap_local_indexer.go new file mode 100644 index 0000000..371681c --- /dev/null +++ b/internal/provider/cap_local_indexer.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" + "net/http" +) + +type LocalIndexer interface { + ServeLocalIndex(w http.ResponseWriter, r *http.Request, files FileStore, repoName, path string) bool + GenerateLocalIndex(ctx context.Context, files FileStore, repoName, path string) ([]byte, error) +} diff --git a/internal/provider/cap_local_uploader.go b/internal/provider/cap_local_uploader.go new file mode 100644 index 0000000..be5d624 --- /dev/null +++ b/internal/provider/cap_local_uploader.go @@ -0,0 +1,6 @@ +package provider + +type LocalUploader interface { + ValidateUpload(filePath string) (storagePath, contentType string, err error) + UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any +} diff --git a/internal/provider/cap_post_delete_hook.go b/internal/provider/cap_post_delete_hook.go new file mode 100644 index 0000000..ac6cf56 --- /dev/null +++ b/internal/provider/cap_post_delete_hook.go @@ -0,0 +1,11 @@ +package provider + +import ( + "context" +) + +// PostDeleteHook lets a provider clean up derived state (e.g. RPM metadata that +// feeds generated repodata) after a local file is removed. +type PostDeleteHook interface { + AfterDelete(ctx context.Context, repoName, storagePath string, db MetadataDeleter) error +} diff --git a/internal/provider/cap_post_upload_hook.go b/internal/provider/cap_post_upload_hook.go new file mode 100644 index 0000000..a8f69a0 --- /dev/null +++ b/internal/provider/cap_post_upload_hook.go @@ -0,0 +1,9 @@ +package provider + +import ( + "context" +) + +type PostUploadHook interface { + AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs BlobReader, db MetadataStore) +} diff --git a/internal/provider/cap_remote_server.go b/internal/provider/cap_remote_server.go new file mode 100644 index 0000000..f4f3fd2 --- /dev/null +++ b/internal/provider/cap_remote_server.go @@ -0,0 +1,25 @@ +package provider + +import ( + "net/http" + + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// RemoteServer lets a remote provider fully answer a request itself instead of +// going through the byte-proxy engine. It is the remote-side analog of +// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize +// repodata from derived per-asset metadata and to redirect package downloads to +// a backend remote, without ever precaching the packages. Returning false lets +// the normal proxy path take over. +type RemoteServer interface { + ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool +} + +// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache +// and read the metadata it derives per upstream asset. *database.DB satisfies it. +type RemoteMetadataStore interface { + RPMMetadataReader + MetadataStore + MetadataDeleter +} diff --git a/internal/provider/cap_serve_rewriter.go b/internal/provider/cap_serve_rewriter.go new file mode 100644 index 0000000..07fc262 --- /dev/null +++ b/internal/provider/cap_serve_rewriter.go @@ -0,0 +1,12 @@ +package provider + +import ( + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// ServeRewriter rewrites a cached mutable remote response per request, for +// indexes whose URLs must name the requesting host and so cannot be rewritten +// before caching. +type ServeRewriter interface { + RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte +} diff --git a/internal/provider/metadata_alpine.go b/internal/provider/metadata_alpine.go new file mode 100644 index 0000000..19aaf65 --- /dev/null +++ b/internal/provider/metadata_alpine.go @@ -0,0 +1,49 @@ +package provider + +import ( + "context" +) + +// AlpineMetadataStore / AlpineMetadataDeleter / AlpineMetadataReader are the +// Alpine-specific persistence surfaces. They are kept separate from the shared +// RPM/Deb metadata interfaces so the apk provider can type-assert the generic +// MetadataStore/MetadataDeleter/FileStore it is handed without widening (and +// thus perturbing the test doubles of) the rpm and deb providers. *database.DB +// satisfies all three. +type AlpineMetadataStore interface { + InsertAlpineMetadata(ctx context.Context, meta *AlpineMetadata) error +} + +type AlpineMetadataDeleter interface { + DeleteAlpineMetadata(ctx context.Context, repoName, filePath string) error +} + +type AlpineMetadataReader interface { + ListAlpineMetadataEntries(ctx context.Context, repoName string) ([]AlpineMetadata, error) +} + +// AlpineMetadata is the derived per-package metadata for an Alpine .apk, holding +// the fields an APKINDEX record carries plus the apk pull checksum (Q1…, the +// sha1 of the control gzip stream) and the download/installed sizes. +type AlpineMetadata struct { + RepoName string + FilePath string + ContentHash string + Checksum string // C: "Q1" + base64(sha1(control gzip stream)) + Name string // P: + Version string // V: + Arch string // A: + DownloadSize int64 // S: on-disk .apk size + InstalledSize int64 // I: unpacked size from .PKGINFO + Description string // T: + URL string // U: + License string // L: + Origin string // o: + Maintainer string // m: + BuildTime int64 // t: + Commit string // c: + ProviderPriority string // k: + Depends []string // D: + Provides []string // p: + InstallIf []string // i: +} diff --git a/internal/provider/metadata_deb.go b/internal/provider/metadata_deb.go new file mode 100644 index 0000000..4fcd2a3 --- /dev/null +++ b/internal/provider/metadata_deb.go @@ -0,0 +1,34 @@ +package provider + +import ( + "context" + "time" +) + +// DebMetadataReader is the read surface the deb LocalIndexer needs to +// regenerate a flat apt repository (Packages/Release) from stored rows. +// *database.DB satisfies it; ServeLocalIndex type-asserts the FileStore to it, +// mirroring how the rpm provider reaches its RPMMetadataReader. +type DebMetadataReader interface { + ListDebMetadataEntries(ctx context.Context, repoName string) ([]DebMetadata, error) +} + +// DebMetadata is the derived per-package metadata for a Debian .deb, carrying +// the full raw control stanza so the Packages index can be regenerated +// faithfully alongside the computed size/md5/sha256 apt requires. +type DebMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + Architecture string + Control string + Size int64 + MD5 string + SHA256 string + // CreatedAt is the persisted insert time; the Release Date: is derived from + // the newest value so the index is byte-identical across replicas and + // regenerations (issue #117) rather than stamped from wall clock. + CreatedAt time.Time +} diff --git a/internal/provider/metadata_helm.go b/internal/provider/metadata_helm.go new file mode 100644 index 0000000..a69ed1f --- /dev/null +++ b/internal/provider/metadata_helm.go @@ -0,0 +1,30 @@ +package provider + +import ( + "context" + "time" +) + +type HelmMetadataStore interface { + InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error +} + +type HelmMetadataDeleter interface { + DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error +} + +type HelmMetadataReader interface { + ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error) +} + +// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml +// entries carry every field helm reads. +type HelmMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Version string + ChartYAML string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_puppet.go b/internal/provider/metadata_puppet.go new file mode 100644 index 0000000..a99bd0c --- /dev/null +++ b/internal/provider/metadata_puppet.go @@ -0,0 +1,32 @@ +package provider + +import ( + "context" + "time" +) + +type PuppetMetadataStore interface { + InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error +} + +type PuppetMetadataDeleter interface { + DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error +} + +type PuppetMetadataReader interface { + ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error) +} + +// PuppetMetadata is a local module release; MetadataJSON is the tarball's +// metadata.json verbatim so Forge release objects carry every field. +type PuppetMetadata struct { + RepoName string + FilePath string + ContentHash string + Slug string + Version string + MetadataJSON string + Size int64 + MD5 string + CreatedAt time.Time +} diff --git a/internal/provider/metadata_rpm.go b/internal/provider/metadata_rpm.go new file mode 100644 index 0000000..4633cf2 --- /dev/null +++ b/internal/provider/metadata_rpm.go @@ -0,0 +1,60 @@ +package provider + +import ( + "context" + "time" +) + +type RPMMetadataReader interface { + ListRPMMetadataEntries(ctx context.Context, repoName string) ([]RPMMetadata, error) +} + +type RPMMetadata struct { + RepoName string + FilePath string + ContentHash string + Name string + Epoch int + Version string + Release string + Arch string + Summary string + Description string + RPMSize int64 + InstalledSize int64 + License string + Vendor string + Group string + BuildHost string + SourceRPM string + URL string + Packager string + Requires []RPMDep + Provides []RPMDep + Conflicts []RPMDep + Obsoletes []RPMDep + Files []RPMFile + Changelogs []RPMChangelog + // CreatedAt is the persisted upload timestamp; used as a stable, replica-independent + // value for the repodata