diff --git a/README.md b/README.md index 5b4a4e9..2d76346 100644 --- a/README.md +++ b/README.md @@ -262,6 +262,21 @@ there's nothing to provision. To bring your own key instead, point `TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one. `TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols. +### Local puppet forge + +A local `puppet` repo is a Forge: `PUT` a module tarball to +`/api/v2/remotes//files/--.tar.gz` and the Forge +v3 API (`/v3/modules`, `/v3/releases`, `/v3/files`) is served from each +release's `metadata.json` at `/api/v1/local/`. Versions are immutable. + +```bash +puppet module build ./hello +curl -fsSL --upload-file pkg/unkin-hello-0.1.0.tar.gz \ + https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes//files/unkin-hello-0.1.0.tar.gz +puppet module install unkin-hello \ + --module_repository https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/ +``` + ### Local docker registry A local `docker` repo is a real container registry, not a mirror: it serves the diff --git a/e2e-docker/README.md b/e2e-docker/README.md index ec2ee0c..facd807 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -26,7 +26,8 @@ already-running stack. helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. -- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` +- **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`), + generic (upload/download), pypi (wheel + generated `simple/` index), rpm (real package + **automatic repodata** generation). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members. diff --git a/e2e-docker/puppet_local_test.go b/e2e-docker/puppet_local_test.go new file mode 100644 index 0000000..9c5acaa --- /dev/null +++ b/e2e-docker/puppet_local_test.go @@ -0,0 +1,88 @@ +//go:build dockere2e + +package e2edocker + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}` + +func puppetModuleTarball(t *testing.T) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range map[string]string{ + "unkin-hello-0.1.0/metadata.json": puppetHelloMetadata, + "unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n", + } { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(body)) + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +// TestLocalPuppetRealClient publishes a module to a puppet local and installs +// it with the real r10k and puppet module tool. The stdlib dependency is not on +// a local-only forge, so puppet runs with --ignore-dependencies. +func TestLocalPuppetRealClient(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + createRepo(t, `{"name":"local-puppet","package_type":"puppet","repo_type":"local"}`) + defer deleteRepo(t, "local-puppet") + + upload := api("/api/v2/remotes/local-puppet/files/unkin-hello-0.1.0.tar.gz") + defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/local-puppet/files/v3/files/unkin-hello-0.1.0.tar.gz"), nil, "") + tarball := puppetModuleTarball(t) + if resp, body := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusCreated { + t.Fatalf("upload: %d %s", resp.StatusCode, body) + } + if resp, _ := doRequest(t, http.MethodPut, upload, tarball, "application/gzip"); resp.StatusCode != http.StatusConflict { + t.Fatalf("re-upload of an existing version: %d, want 409", resp.StatusCode) + } + if resp, body := getEventually(t, api("/api/v1/local/local-puppet/v3/releases/unkin-hello-0.1.0"), 15*time.Second); resp.StatusCode != http.StatusOK || !strings.Contains(string(body), `"file_md5"`) { + t.Fatalf("release never indexed: %d %s", resp.StatusCode, body) + } + + forge := internal + "/api/v1/local/local-puppet" + run := func(image, script string) string { + t.Helper() + out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput() + if err != nil { + t.Fatalf("%s failed: %v\n%s", image, err, out) + } + return string(out) + } + + out := run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+ + "printf \"forge '"+forge+"'\\nmod 'unkin-hello', '0.1.0'\\n\" > Puppetfile && "+ + "r10k puppetfile install -c r10k.yaml --verbose && cat modules/hello/metadata.json") + if !strings.Contains(out, `"unkin-hello"`) { + t.Fatalf("r10k did not install unkin-hello:\n%s", out) + } + + out = run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --ignore-dependencies --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods") + if !strings.Contains(out, "unkin-hello (v0.1.0)") { + t.Fatalf("puppet module install did not install unkin-hello:\n%s", out) + } +} diff --git a/internal/database/puppet_metadata.go b/internal/database/puppet_metadata.go new file mode 100644 index 0000000..036c35d --- /dev/null +++ b/internal/database/puppet_metadata.go @@ -0,0 +1,46 @@ +package database + +import ( + "context" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func (db *DB) InsertPuppetMetadata(ctx context.Context, meta *provider.PuppetMetadata) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO puppet_metadata (repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) + ON CONFLICT (repo_name, file_path) DO NOTHING + `, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Slug, meta.Version, meta.MetadataJSON, meta.Size, meta.MD5) + return err +} + +func (db *DB) DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM puppet_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} + +// ListPuppetMetadataEntries returns a repo's releases, optionally narrowed to +// one module slug (empty slug lists every module). +func (db *DB) ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]provider.PuppetMetadata, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, content_hash, slug, version, metadata_json, file_size, file_md5, created_at + FROM puppet_metadata + WHERE repo_name = $1 AND ($2 = '' OR slug = $2) + ORDER BY slug, file_path + `, repoName, slug) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.PuppetMetadata + for rows.Next() { + var m provider.PuppetMetadata + if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Slug, &m.Version, &m.MetadataJSON, &m.Size, &m.MD5, &m.CreatedAt); err != nil { + return nil, err + } + result = append(result, m) + } + return result, rows.Err() +} diff --git a/internal/provider/provider.go b/internal/provider/provider.go index 4c005f3..aeecde8 100644 --- a/internal/provider/provider.go +++ b/internal/provider/provider.go @@ -247,3 +247,29 @@ func Get(t models.PackageType) (Provider, error) { func All() map[models.PackageType]Provider { return registry } + +type PuppetMetadataStore interface { + InsertPuppetMetadata(ctx context.Context, meta *PuppetMetadata) error +} + +type PuppetMetadataDeleter interface { + DeletePuppetMetadata(ctx context.Context, repoName, filePath string) error +} + +type PuppetMetadataReader interface { + ListPuppetMetadataEntries(ctx context.Context, repoName, slug string) ([]PuppetMetadata, error) +} + +// PuppetMetadata is a local module release; MetadataJSON is the tarball's +// metadata.json verbatim so Forge release objects carry every field. +type PuppetMetadata struct { + RepoName string + FilePath string + ContentHash string + Slug string + Version string + MetadataJSON string + Size int64 + MD5 string + CreatedAt time.Time +} diff --git a/internal/provider/puppet/local.go b/internal/provider/puppet/local.go new file mode 100644 index 0000000..b9a3a4c --- /dev/null +++ b/internal/provider/puppet/local.go @@ -0,0 +1,393 @@ +package puppet + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "path" + "regexp" + "slices" + "strconv" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/storage" +) + +const filesPrefix = "v3/files/" + +// releaseRe matches --, the Forge release slug and the +// tarball basename that `puppet module build` produces. +var releaseRe = regexp.MustCompile(`^([a-z0-9]+)-([a-z][a-z0-9_]*)-(\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?)$`) + +func parseRelease(s string) (slug, version string, ok bool) { + m := releaseRe.FindStringSubmatch(strings.ToLower(s)) + if m == nil { + return "", "", false + } + return m[1] + "-" + m[2], m[3], true +} + +func normalizeSlug(s string) string { + return strings.ToLower(strings.ReplaceAll(s, "/", "-")) +} + +// ValidateUpload stores tarballs under v3/files/ so Forge file_uri downloads +// resolve straight to the stored file. +func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { + filename := strings.ToLower(path.Base(path.Clean("/" + filePath))) + if _, _, ok := parseRelease(strings.TrimSuffix(filename, ".tar.gz")); !ok || !strings.HasSuffix(filename, ".tar.gz") { + return "", "", fmt.Errorf("file must be named --.tar.gz") + } + return filesPrefix + filename, "application/gzip", nil +} + +func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any { + return map[string]any{ + "filename": path.Base(storagePath), + "content_hash": contentHash, + "size_bytes": sizeBytes, + } +} + +func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) { + reader, size, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))) + if err != nil { + slog.Error("puppet metadata: download failed", "repo", repoName, "path", storagePath, "error", err) + return + } + defer func() { _ = reader.Close() }() + if size > maxModuleTarball { + slog.Error("puppet metadata: tarball too large", "repo", repoName, "path", storagePath, "size", size) + return + } + + hash := md5.New() + src := io.TeeReader(io.LimitReader(reader, maxModuleTarball), hash) + meta, err := parseModule(src) + if err != nil { + slog.Error("puppet metadata: parse failed", "repo", repoName, "path", storagePath, "error", err) + return + } + if _, err := io.Copy(io.Discard, src); err != nil { + slog.Error("puppet metadata: read failed", "repo", repoName, "path", storagePath, "error", err) + return + } + slug, version, _ := parseRelease(strings.TrimSuffix(path.Base(storagePath), ".tar.gz")) + if meta.Slug != slug || meta.Version != version { + slog.Error("puppet metadata: metadata.json does not match filename", "repo", repoName, "path", storagePath, "name", meta.Slug, "version", meta.Version) + return + } + meta.RepoName = repoName + meta.FilePath = storagePath + meta.ContentHash = contentHash + meta.Size = size + meta.MD5 = hex.EncodeToString(hash.Sum(nil)) + + store, ok := db.(provider.PuppetMetadataStore) + if !ok { + slog.Error("puppet metadata: store does not support puppet metadata", "repo", repoName) + return + } + if err := store.InsertPuppetMetadata(ctx, meta); err != nil { + slog.Error("puppet metadata: insert failed", "repo", repoName, "path", storagePath, "error", err) + return + } + slog.Info("puppet metadata: parsed", "repo", repoName, "module", meta.Slug, "version", meta.Version) +} + +func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error { + deleter, ok := db.(provider.PuppetMetadataDeleter) + if !ok { + return nil + } + return deleter.DeletePuppetMetadata(ctx, repoName, storagePath) +} + +const maxMetadataJSON = 1 << 20 + +var ( + maxModuleTarball int64 = 100 << 20 + maxModuleScan int64 = 200 << 20 +) + +// parseModule reads the top-level /metadata.json from a module tarball, +// scanning at most maxModuleScan decompressed bytes. +func parseModule(r io.Reader) (*provider.PuppetMetadata, error) { + zr, err := gzip.NewReader(r) + if err != nil { + return nil, err + } + scan := &io.LimitedReader{R: zr, N: maxModuleScan} + tr := tar.NewReader(scan) + for { + hdr, err := tr.Next() + if scan.N <= 0 { + return nil, errors.New("module tarball exceeds decompressed scan limit before metadata.json") + } + if err == io.EOF { + return nil, errors.New("no metadata.json in module tarball") + } + if err != nil { + return nil, err + } + dir, file := path.Split(strings.TrimPrefix(path.Clean(hdr.Name), "./")) + if hdr.Typeflag != tar.TypeReg || file != "metadata.json" || strings.Count(dir, "/") != 1 { + continue + } + body, err := io.ReadAll(io.LimitReader(tr, maxMetadataJSON)) + if err != nil { + return nil, err + } + var md struct { + Name string `json:"name"` + Version string `json:"version"` + } + if err := json.Unmarshal(body, &md); err != nil { + return nil, fmt.Errorf("parse metadata.json: %w", err) + } + if md.Name == "" || md.Version == "" { + return nil, errors.New("metadata.json missing name or version") + } + return &provider.PuppetMetadata{Slug: normalizeSlug(md.Name), Version: md.Version, MetadataJSON: string(body)}, nil + } +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool { + clean := strings.TrimPrefix(path.Clean("/"+reqPath), "/") + var list func(slug string) ([]provider.PuppetMetadata, error) + if reader, ok := files.(provider.PuppetMetadataReader); ok { + list = func(slug string) ([]provider.PuppetMetadata, error) { + metas, err := reader.ListPuppetMetadataEntries(r.Context(), repoName, slug) + sortReleases(metas) + return metas, err + } + } else { + list = func(string) ([]provider.PuppetMetadata, error) { + return nil, errors.New("puppet metadata not available") + } + } + + switch { + case clean == "v3/releases": + serveReleaseList(w, r, list) + case strings.HasPrefix(clean, "v3/releases/"): + slug, version, ok := parseRelease(strings.TrimPrefix(clean, "v3/releases/")) + if !ok { + notFound(w) + return true + } + metas, err := list(slug) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + for _, m := range metas { + if m.Version == version { + writeJSON(w, releaseObject(m)) + return true + } + } + notFound(w) + case strings.HasPrefix(clean, "v3/modules/"): + slug := normalizeSlug(strings.TrimPrefix(clean, "v3/modules/")) + metas, err := list(slug) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + if len(metas) == 0 { + notFound(w) + return true + } + writeJSON(w, moduleObject(metas)) + default: + return false + } + return true +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) { + return nil, fmt.Errorf("puppet local index generation for virtual repos not supported") +} + +func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) ([]provider.PuppetMetadata, error)) { + q := r.URL.Query() + slug := normalizeSlug(q.Get("module")) + limit, err := strconv.Atoi(q.Get("limit")) + if err != nil || limit <= 0 { + limit = 20 + } + limit = min(limit, 100) + offset, err := strconv.Atoi(q.Get("offset")) + if err != nil || offset < 0 { + offset = 0 + } + + metas, err := list(slug) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + offset = min(offset, len(metas)) + results := []map[string]any{} + for _, m := range metas[offset : offset+min(limit, len(metas)-offset)] { + results = append(results, releaseObject(m)) + } + + page := func(off int) any { + if off < 0 || off >= len(metas) && off != 0 { + return nil + } + v := url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}} + if slug != "" { + v.Set("module", slug) + } + return "/v3/releases?" + v.Encode() + } + var previous any + if offset > 0 { + previous = page(max(offset-limit, 0)) + } + writeJSON(w, map[string]any{ + "pagination": map[string]any{ + "limit": limit, + "offset": offset, + "first": page(0), + "previous": previous, + "current": page(offset), + "next": page(offset + limit), + "total": len(metas), + }, + "results": results, + }) +} + +func owner(slug string) map[string]any { + name, _, _ := strings.Cut(slug, "-") + return map[string]any{"uri": "/v3/users/" + name, "slug": name, "username": name} +} + +func moduleRef(slug string) map[string]any { + _, name, _ := strings.Cut(slug, "-") + return map[string]any{"uri": "/v3/modules/" + slug, "slug": slug, "name": name, "deprecated_at": nil, "owner": owner(slug)} +} + +const forgeTime = "2006-01-02 15:04:05 -0700" + +func releaseObject(m provider.PuppetMetadata) map[string]any { + slug := m.Slug + "-" + m.Version + return map[string]any{ + "uri": "/v3/releases/" + slug, + "slug": slug, + "module": moduleRef(m.Slug), + "version": m.Version, + "metadata": json.RawMessage(m.MetadataJSON), + "tags": []string{}, + "supported": false, + "pdk": false, + "file_uri": "/" + m.FilePath, + "file_size": m.Size, + "file_md5": m.MD5, + "file_sha256": strings.TrimPrefix(m.ContentHash, "sha256:"), + "downloads": 0, + "readme": nil, + "changelog": nil, + "license": nil, + "created_at": m.CreatedAt.Format(forgeTime), + "updated_at": m.CreatedAt.Format(forgeTime), + "deleted_at": nil, + } +} + +// moduleObject expects metas sorted newest version first. +func moduleObject(metas []provider.PuppetMetadata) map[string]any { + obj := moduleRef(metas[0].Slug) + releases := make([]map[string]any, 0, len(metas)) + for _, m := range metas { + slug := m.Slug + "-" + m.Version + releases = append(releases, map[string]any{ + "uri": "/v3/releases/" + slug, "slug": slug, "version": m.Version, + "file_uri": "/" + m.FilePath, "file_size": m.Size, "supported": false, + "created_at": m.CreatedAt.Format(forgeTime), "deleted_at": nil, + }) + } + obj["current_release"] = releaseObject(metas[0]) + obj["releases"] = releases + obj["downloads"] = 0 + obj["created_at"] = metas[len(metas)-1].CreatedAt.Format(forgeTime) + obj["updated_at"] = metas[0].CreatedAt.Format(forgeTime) + return obj +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func notFound(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"message":"404 Not Found","errors":["The requested resource could not be found"]}`) +} + +// sortReleases orders by module slug, then newest semver first. +func sortReleases(metas []provider.PuppetMetadata) { + slices.SortStableFunc(metas, func(a, b provider.PuppetMetadata) int { + if c := strings.Compare(a.Slug, b.Slug); c != 0 { + return c + } + return compareSemver(b.Version, a.Version) + }) +} + +func compareSemver(a, b string) int { + a, _, _ = strings.Cut(a, "+") + b, _, _ = strings.Cut(b, "+") + aCore, aPre, aHasPre := strings.Cut(a, "-") + bCore, bPre, bHasPre := strings.Cut(b, "-") + if c := compareIdents(strings.Split(aCore, "."), strings.Split(bCore, ".")); c != 0 { + return c + } + switch { + case aHasPre && !bHasPre: + return -1 + case !aHasPre && bHasPre: + return 1 + } + return compareIdents(strings.Split(aPre, "."), strings.Split(bPre, ".")) +} + +func compareIdents(a, b []string) int { + for i := 0; i < len(a) && i < len(b); i++ { + an, aErr := strconv.Atoi(a[i]) + bn, bErr := strconv.Atoi(b[i]) + var c int + switch { + case aErr == nil && bErr == nil: + c = an - bn + case aErr == nil: + c = -1 + case bErr == nil: + c = 1 + default: + c = strings.Compare(a[i], b[i]) + } + if c != 0 { + return c + } + } + return len(a) - len(b) +} diff --git a/internal/provider/puppet/local_test.go b/internal/provider/puppet/local_test.go new file mode 100644 index 0000000..57927d3 --- /dev/null +++ b/internal/provider/puppet/local_test.go @@ -0,0 +1,301 @@ +package puppet + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func moduleTarball(t *testing.T, dir, metadata string) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for name, body := range map[string]string{dir + "/metadata.json": metadata, dir + "/manifests/init.pp": "class hello {}\n"} { + if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + _, _ = tw.Write([]byte(body)) + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +const helloMetadata = `{"name":"unkin-hello","version":"0.1.0","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0"}]}` + +type fakeBlobs struct{ data []byte } + +func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) { + return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil +} + +type fakeStore struct{ inserted *provider.PuppetMetadata } + +func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil } +func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil } +func (f *fakeStore) InsertPuppetMetadata(_ context.Context, m *provider.PuppetMetadata) error { + f.inserted = m + return nil +} + +type fakeReader struct{ metas []provider.PuppetMetadata } + +func (f fakeReader) ListPuppetMetadataEntries(_ context.Context, _, slug string) ([]provider.PuppetMetadata, error) { + var out []provider.PuppetMetadata + for _, m := range f.metas { + if slug == "" || m.Slug == slug { + out = append(out, m) + } + } + return out, nil +} +func (fakeReader) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (fakeReader) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestValidateUpload(t *testing.T) { + p := &Provider{} + sp, ct, err := p.ValidateUpload("some/dir/Unkin-hello-0.1.0.tar.gz") + if err != nil || sp != "v3/files/unkin-hello-0.1.0.tar.gz" || ct != "application/gzip" { + t.Errorf("sp=%q ct=%q err=%v", sp, ct, err) + } + for _, bad := range []string{"hello-0.1.0.tar.gz", "unkin-hello-0.1.tar.gz", "unkin-hello-0.1.0.zip", "unkin-Hello_x-0.1.0.tgz"} { + if _, _, err := p.ValidateUpload(bad); err == nil { + t.Errorf("%q accepted", bad) + } + } +} + +func TestAfterUploadRecordsMetadata(t *testing.T) { + raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata) + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + m := store.inserted + if m == nil { + t.Fatal("nothing inserted") + } + sum := md5.Sum(raw) + if m.Slug != "unkin-hello" || m.Version != "0.1.0" || m.MD5 != hex.EncodeToString(sum[:]) || m.Size != int64(len(raw)) || m.MetadataJSON != helloMetadata { + t.Errorf("got %+v", m) + } +} + +func TestAfterUploadRejectsMismatchedFilename(t *testing.T) { + raw := moduleTarball(t, "unkin-hello-0.2.0", `{"name":"unkin-hello","version":"0.2.0"}`) + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + if store.inserted != nil { + t.Error("mismatched version indexed") + } +} + +func TestParseModuleRequiresTopLevelMetadata(t *testing.T) { + if _, err := parseModule(bytes.NewReader(moduleTarball(t, "a/b", helloMetadata))); err == nil { + t.Error("nested metadata.json accepted") + } + if _, err := parseModule(bytes.NewReader(moduleTarball(t, "x", `{"name":"unkin-hello"}`))); err == nil { + t.Error("missing version accepted") + } +} + +func tarballOf(t *testing.T, entries ...*tar.Header) []byte { + t.Helper() + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + for _, h := range entries { + if err := tw.WriteHeader(h); err != nil { + t.Fatal(err) + } + if h.Typeflag == tar.TypeReg { + body := helloMetadata + if !strings.HasSuffix(h.Name, "metadata.json") { + body = strings.Repeat("x", int(h.Size)) + } + _, _ = tw.Write([]byte(body)) + } + } + _ = tw.Close() + _ = zw.Close() + return buf.Bytes() +} + +func TestParseModuleSkipsNonRegularMetadata(t *testing.T) { + raw := tarballOf(t, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeSymlink, Linkname: "/etc/passwd"}, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json/", Typeflag: tar.TypeDir, Mode: 0o755}, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))}, + ) + meta, err := parseModule(bytes.NewReader(raw)) + if err != nil || meta.Slug != "unkin-hello" || meta.Version != "0.1.0" { + t.Fatalf("got %v, %v", meta, err) + } +} + +func TestParseModuleCapsDecompressedScan(t *testing.T) { + defer func(n int64) { maxModuleScan = n }(maxModuleScan) + maxModuleScan = 64 << 10 + raw := tarballOf(t, + &tar.Header{Name: "unkin-hello-0.1.0/big", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1 << 20}, + &tar.Header{Name: "unkin-hello-0.1.0/metadata.json", Typeflag: tar.TypeReg, Mode: 0o644, Size: int64(len(helloMetadata))}, + ) + if _, err := parseModule(bytes.NewReader(raw)); err == nil || !strings.Contains(err.Error(), "scan limit") { + t.Fatalf("expected scan limit error, got %v", err) + } + maxModuleScan = 2 << 20 + if _, err := parseModule(bytes.NewReader(raw)); err != nil { + t.Fatalf("under cap: %v", err) + } +} + +func TestAfterUploadCapsTarballSize(t *testing.T) { + defer func(n int64) { maxModuleTarball = n }(maxModuleTarball) + raw := moduleTarball(t, "unkin-hello-0.1.0", helloMetadata) + maxModuleTarball = int64(len(raw)) - 1 + store := &fakeStore{} + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + if store.inserted != nil { + t.Error("oversized tarball indexed") + } + maxModuleTarball = int64(len(raw)) + (&Provider{}).AfterUpload(context.Background(), "forge", "v3/files/unkin-hello-0.1.0.tar.gz", "sha256:abc", fakeBlobs{raw}, store) + if store.inserted == nil { + t.Error("tarball at cap not indexed") + } +} + +func TestReleaseListClampsPagination(t *testing.T) { + files := releases("0.1.0", "0.2.0", "0.3.0") + for _, q := range []string{"offset=9223372036854775807", "offset=9223372036854775807&limit=9223372036854775807", "offset=-5&limit=-5", "limit=9223372036854775807"} { + code, body := get(t, files, "v3/releases?module=unkin-hello&"+q) + if code != http.StatusOK { + t.Fatalf("%s: code %d", q, code) + } + pg := body["pagination"].(map[string]any) + if l := pg["limit"].(float64); l < 1 || l > 100 { + t.Errorf("%s: limit %v", q, l) + } + if o := pg["offset"].(float64); o < 0 || o > 3 { + t.Errorf("%s: offset %v", q, o) + } + } + _, body := get(t, files, "v3/releases?module=unkin-hello&offset=9223372036854775807") + if len(body["results"].([]any)) != 0 || body["pagination"].(map[string]any)["next"] != nil { + t.Errorf("past-end page: %v", body) + } +} + +func get(t *testing.T, files provider.FileStore, target string) (int, map[string]any) { + t.Helper() + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/x/"+target, nil) + path, _, _ := strings.Cut(target, "?") + if !(&Provider{}).ServeLocalIndex(rec, req, files, "forge", path) { + return 0, nil + } + var body map[string]any + _ = json.Unmarshal(rec.Body.Bytes(), &body) + return rec.Code, body +} + +func releases(versions ...string) fakeReader { + var f fakeReader + for i, v := range versions { + f.metas = append(f.metas, provider.PuppetMetadata{ + FilePath: "v3/files/unkin-hello-" + v + ".tar.gz", ContentHash: "sha256:feed", Slug: "unkin-hello", + Version: v, MetadataJSON: `{"name":"unkin-hello","version":"` + v + `"}`, MD5: "m" + v, + CreatedAt: time.Unix(int64(i), 0), + }) + } + f.metas = append(f.metas, provider.PuppetMetadata{Slug: "other-mod", Version: "9.9.9", MetadataJSON: `{}`}) + return f +} + +func TestReleaseListPaginatesNewestFirst(t *testing.T) { + files := releases("0.1.0", "0.10.0", "0.2.0", "1.0.0-rc.1", "1.0.0") + code, body := get(t, files, "v3/releases?module=unkin/hello&limit=2") + if code != http.StatusOK { + t.Fatalf("code %d", code) + } + results := body["results"].([]any) + pg := body["pagination"].(map[string]any) + if len(results) != 2 || results[0].(map[string]any)["version"] != "1.0.0" || results[1].(map[string]any)["version"] != "1.0.0-rc.1" { + t.Fatalf("page 1: %v", results) + } + if pg["total"] != float64(5) || pg["next"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" || pg["previous"] != nil { + t.Errorf("pagination: %v", pg) + } + + _, body = get(t, files, "v3/releases?module=unkin-hello&limit=2&offset=4") + results = body["results"].([]any) + pg = body["pagination"].(map[string]any) + if len(results) != 1 || results[0].(map[string]any)["version"] != "0.1.0" || pg["next"] != nil || pg["previous"] != "/v3/releases?limit=2&module=unkin-hello&offset=2" { + t.Errorf("last page: %v %v", results, pg) + } + + _, body = get(t, files, "v3/releases?module=nobody-none") + if len(body["results"].([]any)) != 0 { + t.Error("unknown module returned results") + } +} + +func TestReleaseObject(t *testing.T) { + code, body := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.1.0") + if code != http.StatusOK { + t.Fatalf("code %d", code) + } + if body["file_uri"] != "/v3/files/unkin-hello-0.1.0.tar.gz" || body["file_md5"] != "m0.1.0" || body["file_sha256"] != "feed" { + t.Errorf("release: %v", body) + } + if body["metadata"].(map[string]any)["name"] != "unkin-hello" || body["module"].(map[string]any)["owner"].(map[string]any)["username"] != "unkin" { + t.Errorf("release refs: %v", body) + } + if code, _ := get(t, releases("0.1.0"), "v3/releases/unkin-hello-0.2.0"); code != http.StatusNotFound { + t.Errorf("missing version: %d", code) + } +} + +func TestModuleObject(t *testing.T) { + code, body := get(t, releases("0.9.0", "0.10.0"), "v3/modules/unkin-hello") + if code != http.StatusOK { + t.Fatalf("code %d", code) + } + if body["slug"] != "unkin-hello" || body["name"] != "hello" || body["current_release"].(map[string]any)["version"] != "0.10.0" || len(body["releases"].([]any)) != 2 { + t.Errorf("module: %v", body) + } + if code, _ := get(t, releases(), "v3/modules/unkin-missing"); code != http.StatusNotFound { + t.Errorf("missing module: %d", code) + } +} + +func TestFilesFallThrough(t *testing.T) { + if code, _ := get(t, releases(), "v3/files/unkin-hello-0.1.0.tar.gz"); code != 0 { + t.Error("file downloads must fall through to the stored file") + } +} + +func TestCompareSemver(t *testing.T) { + ordered := []string{"0.1.0", "0.2.0", "0.10.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0", "1.0.1+build"} + for i := 1; i < len(ordered); i++ { + if compareSemver(ordered[i-1], ordered[i]) >= 0 || compareSemver(ordered[i], ordered[i-1]) <= 0 { + t.Errorf("%s should sort before %s", ordered[i-1], ordered[i]) + } + } + if compareSemver("1.0.0+a", "1.0.0+b") != 0 { + t.Error("build metadata must not affect precedence") + } +} diff --git a/migrations/0004_puppet_metadata.sql b/migrations/0004_puppet_metadata.sql new file mode 100644 index 0000000..a3bd85e --- /dev/null +++ b/migrations/0004_puppet_metadata.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS puppet_metadata ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + content_hash TEXT NOT NULL, + slug TEXT NOT NULL, + version TEXT NOT NULL, + metadata_json TEXT NOT NULL, + file_size BIGINT NOT NULL DEFAULT 0, + file_md5 TEXT NOT NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_puppet_metadata_repo_slug ON puppet_metadata(repo_name, slug); diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index 92c1e57..dd66d22 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -262,11 +262,21 @@ index = "sparse+${proxy}/"`, case 'puppet': return [ { - title: 'Install a module from the Forge proxy', + title: isLocal ? 'Install a module from this forge' : 'Install a module from the Forge proxy', language: 'bash', code: `puppet module install - \\ - --module_repository ${proxy}`, + --module_repository ${isLocal ? `${url}/api/v1/local/${name}` : proxy}`, }, + ...(isLocal + ? [ + { + title: 'Publish a module (versions are immutable)', + language: 'bash', + code: `puppet module build && curl -fsSL --upload-file pkg/--.tar.gz \\ + ${url}/api/v2/remotes/${name}/files/--.tar.gz`, + }, + ] + : []), ]; case 'deb':