diff --git a/README.md b/README.md index fa720b8..47c2bfe 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/< - `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) - `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md), [virtual](docs/npm/virtual.md) - `nuget`: [remote](docs/nuget/remote.md) -- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md) +- `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md), [virtual](docs/puppet/virtual.md) - `pypi`: [remote](docs/pypi/remote.md) - `rpm`: [remote](docs/rpm/remote.md), [remote-github](docs/rpm/remote-github.md) - `terraform`: [remote](docs/terraform/remote.md), [local](docs/terraform/local.md) diff --git a/docs/puppet/virtual.md b/docs/puppet/virtual.md new file mode 100644 index 0000000..11266a9 --- /dev/null +++ b/docs/puppet/virtual.md @@ -0,0 +1,37 @@ +# `puppet` virtual + +A puppet virtual serves one Forge v3 API over local and remote puppet members: +`puppet module install` and r10k work against `/api/v1/virtual/`, with +dependencies resolved across members. + +- `/v3/releases?module=` merges member releases; on a duplicate version + the earlier member wins. Releases are newest first, paginated with + `limit` (up to 100) and `offset`; pagination links and `file_uri` are + relative to the Forge root, so downloads route via the virtual. +- `/v3/modules/` lists the merged releases with the highest version as + `current_release`. +- `/v3/releases/-` and `/v3/files/` 302 to the member + owning that release. +- A member error fails the request with 502; a member without the module is + skipped. Merges, and modules no member has, are cached for 60s. + +```hcl +resource "artifactapi_virtual" "puppet" { + name = "puppet" + package_type = "puppet" + members = [artifactapi_local_puppet.internal.name, artifactapi_remote_puppet.forge.name] +} +``` + +```bash +puppet module install unkin-hello \ + --module_repository https://artifactapi.example.com/api/v1/virtual/puppet +``` + +```yaml +# r10k.yaml +forge: + baseurl: https://artifactapi.example.com/api/v1/virtual/puppet +``` + +Upload to the local member directly; the virtual is read-only. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index c7d2342..b5dbd00 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -40,7 +40,9 @@ already-running stack. downloads 302 to the owning member); ansible over two locals via stock `ansible-galaxy` (install through the virtual, download 302 to the owner); goproxy over two locals via a stock `golang:1.25` client (merged versions, - member order wins, version files 302 to the owning member). + member order wins, version files 302 to the owning member); puppet over two + locals via real `puppet module install` (dependency resolved across members) + and `r10k`, downloads 302 to the owning member. - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors (pool = `base_url` + `mirrorlist`): round-robin distribution across both mirrors (constant-body `mockupstreama` / `mockupstreamb`), failover past a dead primary, diff --git a/e2e-docker/puppet_local_test.go b/e2e-docker/puppet_local_test.go index 9c5acaa..b7d5a55 100644 --- a/e2e-docker/puppet_local_test.go +++ b/e2e-docker/puppet_local_test.go @@ -17,13 +17,18 @@ import ( const puppetHelloMetadata = `{"name":"unkin-hello","version":"0.1.0","author":"unkin","summary":"e2e","license":"Apache-2.0","source":"https://example.invalid","dependencies":[{"name":"puppetlabs/stdlib","version_requirement":">= 4.0.0 < 10.0.0"}]}` func puppetModuleTarball(t *testing.T) []byte { + t.Helper() + return puppetTarball(t, "unkin-hello-0.1.0", puppetHelloMetadata) +} + +func puppetTarball(t *testing.T, dir, metadata string) []byte { t.Helper() var buf bytes.Buffer zw := gzip.NewWriter(&buf) tw := tar.NewWriter(zw) for name, body := range map[string]string{ - "unkin-hello-0.1.0/metadata.json": puppetHelloMetadata, - "unkin-hello-0.1.0/manifests/init.pp": "class hello {}\n", + dir + "/metadata.json": metadata, + dir + "/manifests/init.pp": "class stub {}\n", } { if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body)), ModTime: time.Now()}); err != nil { t.Fatal(err) diff --git a/e2e-docker/puppet_virtual_test.go b/e2e-docker/puppet_virtual_test.go new file mode 100644 index 0000000..e38a3dd --- /dev/null +++ b/e2e-docker/puppet_virtual_test.go @@ -0,0 +1,83 @@ +//go:build dockere2e + +package e2edocker + +import ( + "fmt" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// TestVirtualPuppetInstall puts unkin-hello and its stdlib dependency on two +// puppet locals and installs through a virtual over both with the real puppet +// module tool (dependencies resolved) and r10k, downloads 302ing to each owner. +func TestVirtualPuppetInstall(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + id := time.Now().UnixNano() + a, b, virt := fmt.Sprintf("e2e-puppet-a-%d", id), fmt.Sprintf("e2e-puppet-b-%d", id), fmt.Sprintf("e2e-puppet-v-%d", id) + modules := map[string]struct{ file, dir, meta string }{ + a: {"unkin-hello-0.1.0.tar.gz", "unkin-hello-0.1.0", puppetHelloMetadata}, + b: {"puppetlabs-stdlib-9.6.0.tar.gz", "puppetlabs-stdlib-9.6.0", `{"name":"puppetlabs-stdlib","version":"9.6.0","author":"puppetlabs","summary":"stub","license":"Apache-2.0","source":"https://example.invalid","dependencies":[]}`}, + } + for repo, m := range modules { + createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"puppet","repo_type":"local"}`, repo)) + defer deleteRepo(t, repo) + if resp, body := doRequest(t, http.MethodPut, api("/api/v2/remotes/"+repo+"/files/"+m.file), puppetTarball(t, m.dir, m.meta), "application/gzip"); resp.StatusCode != http.StatusCreated { + t.Fatalf("upload %s: %d %s", m.file, resp.StatusCode, body) + } + defer doRequest(t, http.MethodDelete, api("/api/v2/remotes/"+repo+"/files/v3/files/"+m.file), nil, "") + if resp, body := getEventually(t, api("/api/v1/local/"+repo+"/v3/releases/"+strings.TrimSuffix(m.file, ".tar.gz")), 15*time.Second); resp.StatusCode != http.StatusOK { + t.Fatalf("%s never indexed: %d %s", m.file, resp.StatusCode, body) + } + } + createVirtual(t, fmt.Sprintf(`{"name":%q,"package_type":"puppet","members":[%q,%q]}`, virt, a, b)) + defer deleteVirtual(t, virt) + + forge := internal + "/api/v1/virtual/" + virt + run := func(image, script string) string { + t.Helper() + out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh", image, "-c", "set -eu; "+script).CombinedOutput() + if err != nil { + t.Fatalf("%s failed: %v\n%s", image, err, out) + } + return string(out) + } + + out := run("ghcr.io/openvoxproject/openvoxagent:latest", "export PATH=/opt/puppetlabs/bin:$PATH; puppet module install unkin-hello --module_repository "+forge+" --target-dir /tmp/mods && puppet module list --color=false --modulepath /tmp/mods") + for _, want := range []string{"unkin-hello (v0.1.0)", "puppetlabs-stdlib (v9.6.0)"} { + if !strings.Contains(out, want) { + t.Fatalf("puppet module install missing %q:\n%s", want, out) + } + } + + out = run("ghcr.io/voxpupuli/r10k:latest", "cd /tmp && printf 'forge:\\n baseurl: "+forge+"\\n' > r10k.yaml && "+ + "printf \"mod 'unkin-hello', '0.1.0'\\nmod 'puppetlabs-stdlib', '9.6.0'\\n\" > Puppetfile && "+ + "r10k puppetfile install -c r10k.yaml && cat modules/hello/metadata.json modules/stdlib/metadata.json") + if !strings.Contains(out, `"unkin-hello"`) || !strings.Contains(out, `"puppetlabs-stdlib"`) { + t.Fatalf("r10k did not install both modules:\n%s", out) + } + + noRedirect := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + for repo, m := range modules { + resp, err := noRedirect.Get(api("/api/v1/virtual/" + virt + "/v3/files/" + m.file)) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if loc := resp.Header.Get("Location"); resp.StatusCode != http.StatusFound || !strings.HasSuffix(loc, "/api/v1/local/"+repo+"/v3/files/"+m.file) { + t.Errorf("%s = %d %q, want 302 to %s", m.file, resp.StatusCode, loc, repo) + } + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 4f1d145..9c6ad33 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -215,6 +215,10 @@ func (h *ProxyHandler) handleVirtual(w http.ResponseWriter, r *http.Request) { h.serveVirtualAnsible(w, r, *virt, path, proxyBaseURL) return } + if virt.PackageType == models.PackagePuppet { + h.serveVirtualPuppet(w, r, *virt, path, proxyBaseURL) + return + } if virt.PackageType == models.PackageGoProxy { h.serveVirtualGoProxy(w, r, *virt, path) return diff --git a/internal/api/v1/puppet_virtual.go b/internal/api/v1/puppet_virtual.go new file mode 100644 index 0000000..dfb76f0 --- /dev/null +++ b/internal/api/v1/puppet_virtual.go @@ -0,0 +1,77 @@ +package v1 + +import ( + "encoding/json" + "errors" + "log/slog" + "net/http" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider/puppet" + "git.unkin.net/unkin/artifactapi/internal/virtual" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// serveVirtualPuppet serves the merged Forge v3 module and release listings +// and 302s release details and downloads to the member owning that release. +func (h *ProxyHandler) serveVirtualPuppet(w http.ResponseWriter, r *http.Request, virt models.Virtual, path, proxyBaseURL string) { + var doc any + var err error + switch { + case path == "v3/releases": + q := r.URL.Query() + slug := puppet.NormalizeSlug(q.Get("module")) + if !puppet.ValidSlug(slug) { + forgeError(w, http.StatusBadRequest, "module is required") + return + } + var d *virtual.PuppetDoc + d, err = h.virtualEngine.PuppetReleases(r.Context(), virt, slug) + releases := []any{} + if errors.Is(err, virtual.ErrNotFound) { + err = nil + } else if err == nil { + for _, rel := range d.Releases { + releases = append(releases, rel) + } + } + doc = puppet.ReleasesPage(slug, releases, q) + case strings.HasPrefix(path, "v3/modules/"): + slug := puppet.NormalizeSlug(strings.TrimPrefix(path, "v3/modules/")) + if !puppet.ValidSlug(slug) { + puppet.NotFound(w) + return + } + var d *virtual.PuppetDoc + if d, err = h.virtualEngine.PuppetReleases(r.Context(), virt, slug); err == nil { + doc, err = virtual.PuppetModule(d) + } + default: + var loc string + loc, err = h.virtualEngine.PuppetRedirect(r.Context(), virt, path, proxyBaseURL) + if err == nil { + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("Vary", "Host, X-Forwarded-Proto") + http.Redirect(w, r, loc, http.StatusFound) + return + } + } + switch { + case errors.Is(err, virtual.ErrNotFound): + puppet.NotFound(w) + case errors.Is(err, virtual.ErrBadPath): + forgeError(w, http.StatusBadRequest, err.Error()) + case err != nil: + slog.Error("virtual puppet fetch failed", "virtual", virt.Name, "path", path, "error", err) + forgeError(w, http.StatusBadGateway, "bad gateway") + default: + w.Header().Set("X-Artifact-Source", "virtual") + puppet.WriteJSON(w, doc) + } +} + +func forgeError(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]any{"message": http.StatusText(code), "errors": []string{msg}}) +} diff --git a/internal/api/v1/puppet_virtual_test.go b/internal/api/v1/puppet_virtual_test.go new file mode 100644 index 0000000..ca8774b --- /dev/null +++ b/internal/api/v1/puppet_virtual_test.go @@ -0,0 +1,90 @@ +package v1 + +import ( + "context" + "net/http" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/virtual" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestPuppetVirtualOverLocals(t *testing.T) { + h, srv := npmTestServer(t) + h.virtualEngine = virtual.NewEngine(h.db, nil) + a, b := npmRepo(t, h, models.PackagePuppet), npmRepo(t, h, models.PackagePuppet) + for _, p := range []struct{ repo, version string }{{a, "0.1.0"}, {b, "0.1.0"}, {b, "0.2.0"}} { + file := "v3/files/unkin-hello-" + p.version + ".tar.gz" + meta := &provider.PuppetMetadata{RepoName: p.repo, FilePath: file, ContentHash: "sha256:00", Slug: "unkin-hello", Version: p.version, MetadataJSON: `{"name":"unkin-hello","version":"` + p.version + `"}`} + if err := h.db.InsertPuppetMetadata(context.Background(), meta); err != nil { + t.Fatal(err) + } + } + virt := "v" + a + if err := h.db.CreateVirtual(context.Background(), &models.Virtual{Name: virt, PackageType: models.PackagePuppet, Members: []string{a, b}}); err != nil { + t.Fatal(err) + } + base := srv.URL + "/api/v1/virtual/" + virt + + var page struct { + Pagination struct { + Total int + Next *string + } + Results []struct { + Version string + FileURI string `json:"file_uri"` + } + } + getJSON(t, base+"/v3/releases?module=unkin/hello&limit=1", &page) + if page.Pagination.Total != 2 || len(page.Results) != 1 || page.Results[0].Version != "0.2.0" || page.Pagination.Next == nil { + t.Fatalf("page 1 = %+v", page) + } + if *page.Pagination.Next != "/v3/releases?limit=1&module=unkin-hello&offset=1" || page.Results[0].FileURI != "/v3/files/unkin-hello-0.2.0.tar.gz" { + t.Fatalf("links not relative to the forge root: next %q file_uri %q", *page.Pagination.Next, page.Results[0].FileURI) + } + getJSON(t, base+*page.Pagination.Next, &page) + if len(page.Results) != 1 || page.Results[0].Version != "0.1.0" || page.Pagination.Next != nil { + t.Fatalf("page 2 = %+v", page) + } + + var mod struct { + CurrentRelease struct{ Version string } `json:"current_release"` + Releases []struct{ Version string } + } + getJSON(t, base+"/v3/modules/unkin-hello", &mod) + if mod.CurrentRelease.Version != "0.2.0" || len(mod.Releases) != 2 { + t.Fatalf("module = %+v", mod) + } + + noRedirect := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + for p, want := range map[string]string{ + "/v3/files/unkin-hello-0.1.0.tar.gz": "/api/v1/local/" + a + "/v3/files/unkin-hello-0.1.0.tar.gz", + "/v3/releases/unkin-hello-0.2.0": "/api/v1/local/" + b + "/v3/releases/unkin-hello-0.2.0", + } { + resp, err := noRedirect.Get(base + p) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusFound || resp.Header.Get("Location") != srv.URL+want || resp.Header.Get("Cache-Control") != "no-store" { + t.Errorf("%s = %d %q %q, want 302 to %s", p, resp.StatusCode, resp.Header.Get("Location"), resp.Header.Get("Cache-Control"), want) + } + } + + getJSON(t, base+"/v3/releases?module=unkin-absent", &page) + if page.Pagination.Total != 0 || len(page.Results) != 0 { + t.Fatalf("absent module = %+v", page) + } + for p, want := range map[string]int{ + "/v3/releases": http.StatusBadRequest, + "/v3/modules/unkin-absent": http.StatusNotFound, + "/v3/files/unkin-hello-0.9.0.tar.gz": http.StatusNotFound, + } { + if code, body := npmDo(t, http.MethodGet, base+p, ""); code != want || !strings.Contains(body, `"errors"`) { + t.Errorf("%s = %d %s, want %d", p, code, body, want) + } + } +} diff --git a/internal/provider/puppet/local.go b/internal/provider/puppet/local.go index b9a3a4c..f6eb8d1 100644 --- a/internal/provider/puppet/local.go +++ b/internal/provider/puppet/local.go @@ -29,7 +29,7 @@ const filesPrefix = "v3/files/" // tarball basename that `puppet module build` produces. var releaseRe = regexp.MustCompile(`^([a-z0-9]+)-([a-z][a-z0-9_]*)-(\d+\.\d+\.\d+(?:-[0-9a-z.-]+)?(?:\+[0-9a-z.-]+)?)$`) -func parseRelease(s string) (slug, version string, ok bool) { +func ParseRelease(s string) (slug, version string, ok bool) { m := releaseRe.FindStringSubmatch(strings.ToLower(s)) if m == nil { return "", "", false @@ -37,7 +37,12 @@ func parseRelease(s string) (slug, version string, ok bool) { return m[1] + "-" + m[2], m[3], true } -func normalizeSlug(s string) string { +var slugRe = regexp.MustCompile(`^[a-z0-9]+-[a-z][a-z0-9_]*$`) + +// ValidSlug reports whether s is a normalized - module slug. +func ValidSlug(s string) bool { return slugRe.MatchString(s) } + +func NormalizeSlug(s string) string { return strings.ToLower(strings.ReplaceAll(s, "/", "-")) } @@ -45,7 +50,7 @@ func normalizeSlug(s string) string { // resolve straight to the stored file. func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) { filename := strings.ToLower(path.Base(path.Clean("/" + filePath))) - if _, _, ok := parseRelease(strings.TrimSuffix(filename, ".tar.gz")); !ok || !strings.HasSuffix(filename, ".tar.gz") { + if _, _, ok := ParseRelease(strings.TrimSuffix(filename, ".tar.gz")); !ok || !strings.HasSuffix(filename, ".tar.gz") { return "", "", fmt.Errorf("file must be named --.tar.gz") } return filesPrefix + filename, "application/gzip", nil @@ -82,7 +87,7 @@ func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, conte slog.Error("puppet metadata: read failed", "repo", repoName, "path", storagePath, "error", err) return } - slug, version, _ := parseRelease(strings.TrimSuffix(path.Base(storagePath), ".tar.gz")) + slug, version, _ := ParseRelease(strings.TrimSuffix(path.Base(storagePath), ".tar.gz")) if meta.Slug != slug || meta.Version != version { slog.Error("puppet metadata: metadata.json does not match filename", "repo", repoName, "path", storagePath, "name", meta.Slug, "version", meta.Version) return @@ -158,7 +163,7 @@ func parseModule(r io.Reader) (*provider.PuppetMetadata, error) { if md.Name == "" || md.Version == "" { return nil, errors.New("metadata.json missing name or version") } - return &provider.PuppetMetadata{Slug: normalizeSlug(md.Name), Version: md.Version, MetadataJSON: string(body)}, nil + return &provider.PuppetMetadata{Slug: NormalizeSlug(md.Name), Version: md.Version, MetadataJSON: string(body)}, nil } } @@ -181,9 +186,9 @@ func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files case clean == "v3/releases": serveReleaseList(w, r, list) case strings.HasPrefix(clean, "v3/releases/"): - slug, version, ok := parseRelease(strings.TrimPrefix(clean, "v3/releases/")) + slug, version, ok := ParseRelease(strings.TrimPrefix(clean, "v3/releases/")) if !ok { - notFound(w) + NotFound(w) return true } metas, err := list(slug) @@ -193,23 +198,23 @@ func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files } for _, m := range metas { if m.Version == version { - writeJSON(w, releaseObject(m)) + WriteJSON(w, releaseObject(m)) return true } } - notFound(w) + NotFound(w) case strings.HasPrefix(clean, "v3/modules/"): - slug := normalizeSlug(strings.TrimPrefix(clean, "v3/modules/")) + slug := NormalizeSlug(strings.TrimPrefix(clean, "v3/modules/")) metas, err := list(slug) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return true } if len(metas) == 0 { - notFound(w) + NotFound(w) return true } - writeJSON(w, moduleObject(metas)) + WriteJSON(w, moduleObject(metas)) default: return false } @@ -222,7 +227,22 @@ func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileSt func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) ([]provider.PuppetMetadata, error)) { q := r.URL.Query() - slug := normalizeSlug(q.Get("module")) + slug := NormalizeSlug(q.Get("module")) + metas, err := list(slug) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + releases := make([]any, len(metas)) + for i, m := range metas { + releases[i] = releaseObject(m) + } + WriteJSON(w, ReleasesPage(slug, releases, q)) +} + +// ReleasesPage is one limit/offset page of a /v3/releases listing, with +// pagination links relative to the Forge root. +func ReleasesPage(slug string, releases []any, q url.Values) map[string]any { limit, err := strconv.Atoi(q.Get("limit")) if err != nil || limit <= 0 { limit = 20 @@ -232,20 +252,10 @@ func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) if err != nil || offset < 0 { offset = 0 } - - metas, err := list(slug) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - offset = min(offset, len(metas)) - results := []map[string]any{} - for _, m := range metas[offset : offset+min(limit, len(metas)-offset)] { - results = append(results, releaseObject(m)) - } + offset = min(offset, len(releases)) page := func(off int) any { - if off < 0 || off >= len(metas) && off != 0 { + if off < 0 || off >= len(releases) && off != 0 { return nil } v := url.Values{"limit": {strconv.Itoa(limit)}, "offset": {strconv.Itoa(off)}} @@ -258,7 +268,7 @@ func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) if offset > 0 { previous = page(max(offset-limit, 0)) } - writeJSON(w, map[string]any{ + return map[string]any{ "pagination": map[string]any{ "limit": limit, "offset": offset, @@ -266,10 +276,10 @@ func serveReleaseList(w http.ResponseWriter, r *http.Request, list func(string) "previous": previous, "current": page(offset), "next": page(offset + limit), - "total": len(metas), + "total": len(releases), }, - "results": results, - }) + "results": releases[offset : offset+min(limit, len(releases)-offset)], + } } func owner(slug string) map[string]any { @@ -329,7 +339,7 @@ func moduleObject(metas []provider.PuppetMetadata) map[string]any { return obj } -func writeJSON(w http.ResponseWriter, v any) { +func WriteJSON(w http.ResponseWriter, v any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) enc := json.NewEncoder(w) @@ -337,7 +347,7 @@ func writeJSON(w http.ResponseWriter, v any) { _ = enc.Encode(v) } -func notFound(w http.ResponseWriter) { +func NotFound(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotFound) _, _ = io.WriteString(w, `{"message":"404 Not Found","errors":["The requested resource could not be found"]}`) @@ -349,11 +359,11 @@ func sortReleases(metas []provider.PuppetMetadata) { if c := strings.Compare(a.Slug, b.Slug); c != 0 { return c } - return compareSemver(b.Version, a.Version) + return CompareSemver(b.Version, a.Version) }) } -func compareSemver(a, b string) int { +func CompareSemver(a, b string) int { a, _, _ = strings.Cut(a, "+") b, _, _ = strings.Cut(b, "+") aCore, aPre, aHasPre := strings.Cut(a, "-") diff --git a/internal/provider/puppet/local_test.go b/internal/provider/puppet/local_test.go index 57927d3..210c2a8 100644 --- a/internal/provider/puppet/local_test.go +++ b/internal/provider/puppet/local_test.go @@ -291,11 +291,11 @@ func TestFilesFallThrough(t *testing.T) { func TestCompareSemver(t *testing.T) { ordered := []string{"0.1.0", "0.2.0", "0.10.0", "1.0.0-alpha", "1.0.0-alpha.1", "1.0.0-beta", "1.0.0-beta.2", "1.0.0-beta.11", "1.0.0", "1.0.1+build"} for i := 1; i < len(ordered); i++ { - if compareSemver(ordered[i-1], ordered[i]) >= 0 || compareSemver(ordered[i], ordered[i-1]) <= 0 { + if CompareSemver(ordered[i-1], ordered[i]) >= 0 || CompareSemver(ordered[i], ordered[i-1]) <= 0 { t.Errorf("%s should sort before %s", ordered[i-1], ordered[i]) } } - if compareSemver("1.0.0+a", "1.0.0+b") != 0 { + if CompareSemver("1.0.0+a", "1.0.0+b") != 0 { t.Error("build metadata must not affect precedence") } } diff --git a/internal/virtual/engine.go b/internal/virtual/engine.go index 800b2a9..4088194 100644 --- a/internal/virtual/engine.go +++ b/internal/virtual/engine.go @@ -45,6 +45,8 @@ type Engine struct { goMember func(ctx context.Context, name, path string) (*GoMember, error) + puppetDoc func(ctx context.Context, remote models.Remote, path string) ([]byte, error) + mergeTTL time.Duration sf singleflight.Group @@ -77,6 +79,7 @@ func NewEngine(db *database.DB, proxyEngine *proxy.Engine) *Engine { e.gemsMember = e.fetchGemsMember e.ansibleDoc = e.ansibleMemberDoc e.goMember = e.fetchGoMember + e.puppetDoc = e.fetchMemberPath return e } @@ -419,8 +422,9 @@ func (e *Engine) fetchMemberPath(ctx context.Context, remote models.Remote, path if !ok { return nil, fmt.Errorf("provider %q does not serve a local index", remote.PackageType) } + reqPath, _, _ := strings.Cut(path, "?") serve = func(w http.ResponseWriter, r *http.Request) bool { - return indexer.ServeLocalIndex(w, r, e.db, remote.Name, path) + return indexer.ServeLocalIndex(w, r, e.db, remote.Name, reqPath) } } else if rs, ok := prov.(provider.RemoteServer); ok { serve = func(w http.ResponseWriter, r *http.Request) bool { diff --git a/internal/virtual/puppet_merger.go b/internal/virtual/puppet_merger.go new file mode 100644 index 0000000..941eab9 --- /dev/null +++ b/internal/virtual/puppet_merger.go @@ -0,0 +1,210 @@ +package virtual + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "net/url" + "path" + "slices" + "strconv" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider/puppet" + "git.unkin.net/unkin/artifactapi/internal/proxy" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +const ( + maxPuppetModules = 1024 + puppetMemberPage = 100 + maxPuppetMemberPages = 100 +) + +// PuppetMember is one member's releases of a module; Found is false when the +// member does not have it. +type PuppetMember struct { + RemoteName string + Found bool + Releases []map[string]any +} + +// PuppetDoc is a module's merged releases, newest version first, with +// file_uri routed via the virtual; Owners maps each version to its member; +// missing caches that no member has the module. +type PuppetDoc struct { + Releases []map[string]any + Owners map[string]string + body []byte + missing bool +} + +func (d *PuppetDoc) index() []byte { return d.body } +func (d *PuppetDoc) files() map[string][]byte { return nil } + +// MergePuppetReleases merges member releases of slug; the earlier member wins +// a duplicate version. A release whose file is not -.tar.gz is +// dropped, since its download could not be routed. ErrNotFound means no +// member has the module. +func MergePuppetReleases(slug string, members []PuppetMember) (*PuppetDoc, error) { + d := &PuppetDoc{Releases: []map[string]any{}, Owners: map[string]string{}} + found := false + for _, m := range members { + found = found || m.Found + for _, r := range m.Releases { + v, _ := r["version"].(string) + file, _ := r["file_uri"].(string) + name := path.Base(file) + s, fv, ok := puppet.ParseRelease(strings.TrimSuffix(name, ".tar.gz")) + if !ok || s != slug || fv != v || !strings.HasSuffix(name, ".tar.gz") { + continue + } + if _, dup := d.Owners[v]; dup { + continue + } + d.Owners[v] = m.RemoteName + r["file_uri"] = "/v3/files/" + name + d.Releases = append(d.Releases, r) + } + } + if !found { + return nil, ErrNotFound + } + slices.SortStableFunc(d.Releases, func(a, b map[string]any) int { + return puppet.CompareSemver(b["version"].(string), a["version"].(string)) + }) + var err error + d.body, err = json.Marshal(d.Releases) + return d, err +} + +// PuppetModule is the /v3/modules/ document for d: the newest release +// as current_release and every release listed. +func PuppetModule(d *PuppetDoc) (map[string]any, error) { + if len(d.Releases) == 0 { + return nil, ErrNotFound + } + top := d.Releases[0] + ref, ok := top["module"].(map[string]any) + if !ok { + return nil, fmt.Errorf("release %v has no module", top["slug"]) + } + mod := maps.Clone(ref) + releases := make([]map[string]any, len(d.Releases)) + for i, r := range d.Releases { + releases[i] = map[string]any{} + for _, k := range []string{"uri", "slug", "version", "file_uri", "file_size", "supported", "created_at", "deleted_at"} { + releases[i][k] = r[k] + } + } + mod["current_release"] = top + mod["releases"] = releases + return mod, nil +} + +// PuppetReleases returns the merged releases of slug across members. +func (e *Engine) PuppetReleases(ctx context.Context, virt models.Virtual, slug string) (*PuppetDoc, error) { + prefix := "puppet\x00" + virt.Name + "\x00" + inserted := false + repo, err := e.merged(ctx, prefix+slug, func(ctx context.Context) (mergedRepo, error) { + inserted = true + members, err := fetchMembers(virt, func(m string) (*PuppetMember, error) { return e.puppetMember(ctx, m, slug) }) + if err != nil { + return nil, err + } + d, err := MergePuppetReleases(slug, members) + if errors.Is(err, ErrNotFound) { + return &PuppetDoc{missing: true}, nil + } + return d, err + }) + if err != nil { + return nil, err + } + if inserted { + e.evictOldest(prefix, maxPuppetModules) + } + if d := repo.(*PuppetDoc); !d.missing { + return d, nil + } + return nil, ErrNotFound +} + +// PuppetRedirect maps v3/releases/- or v3/files/ to the +// route of the member owning that release. +func (e *Engine) PuppetRedirect(ctx context.Context, virt models.Virtual, p, proxyBaseURL string) (string, error) { + rel, ok := strings.CutPrefix(p, "v3/releases/") + if !ok { + file, isFile := strings.CutPrefix(p, "v3/files/") + if rel, ok = strings.CutSuffix(file, ".tar.gz"); !isFile || !ok { + return "", ErrNotFound + } + } + slug, version, ok := puppet.ParseRelease(rel) + if !ok { + return "", ErrNotFound + } + d, err := e.PuppetReleases(ctx, virt, slug) + if err != nil { + return "", err + } + owner, ok := d.Owners[version] + if !ok { + return "", ErrNotFound + } + loc, ok, err := e.memberLocation(ctx, owner, p, "", proxyBaseURL) + if err != nil { + return "", err + } + if !ok { + return "", fmt.Errorf("member %q not found", owner) + } + return loc, nil +} + +// puppetMember reads every release a member lists for slug, following its +// pagination. +func (e *Engine) puppetMember(ctx context.Context, member, slug string) (*PuppetMember, error) { + remote, err := e.getRemote(ctx, member) + if err != nil { + return nil, fmt.Errorf("remote %q: %w", member, err) + } + m := &PuppetMember{RemoteName: member} + q := url.Values{"module": {slug}, "limit": {strconv.Itoa(puppetMemberPage)}, "exclude_fields": {"readme changelog license reference"}} + for page := 0; page < maxPuppetMemberPages; page++ { + if page > 0 { + q.Set("offset", strconv.Itoa(len(m.Releases))) + } + p := "v3/releases?" + q.Encode() + body, err := e.puppetDoc(ctx, *remote, p) + var pe *proxy.ProxyError + if errors.Is(err, ErrNotFound) || errors.As(err, &pe) && pe.Status == http.StatusNotFound { + return m, nil + } + if err != nil { + return nil, err + } + var doc struct { + Pagination struct{ Total int } + Results []map[string]any + } + dec := json.NewDecoder(bytes.NewReader(body)) + dec.UseNumber() + if err := dec.Decode(&doc); err != nil { + return nil, fmt.Errorf("%s/%s: %v", member, p, err) + } + m.Found = m.Found || doc.Pagination.Total > 0 + m.Releases = append(m.Releases, doc.Results...) + if len(m.Releases) >= doc.Pagination.Total { + return m, nil + } + if len(doc.Results) == 0 { + return nil, fmt.Errorf("%s/%s: empty page after %d of %d releases", member, p, len(m.Releases), doc.Pagination.Total) + } + } + return nil, fmt.Errorf("%s: more than %d pages of %s releases", member, maxPuppetMemberPages, slug) +} diff --git a/internal/virtual/puppet_merger_test.go b/internal/virtual/puppet_merger_test.go new file mode 100644 index 0000000..f6dc785 --- /dev/null +++ b/internal/virtual/puppet_merger_test.go @@ -0,0 +1,205 @@ +package virtual + +import ( + "context" + "encoding/json" + "errors" + "net/url" + "strconv" + "strings" + "sync" + "testing" + "time" + + "git.unkin.net/unkin/artifactapi/internal/proxy" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func puppetRel(slug, version string) map[string]any { + return map[string]any{ + "slug": slug + "-" + version, "version": version, + "file_uri": "/v3/files/" + slug + "-" + version + ".tar.gz", + "module": map[string]any{"slug": slug, "uri": "/v3/modules/" + slug}, + } +} + +func versionsOf(rs []map[string]any) string { + var out []string + for _, r := range rs { + out = append(out, r["version"].(string)) + } + return strings.Join(out, ",") +} + +func TestMergePuppetReleases(t *testing.T) { + absolute := puppetRel("unkin-hello", "0.3.0") + absolute["file_uri"] = "https://forge.example/api/v1/remote/forge/v3/files/unkin-hello-0.3.0.tar.gz" + internalDup := puppetRel("unkin-hello", "0.2.0") + internalDup["tag"] = "internal" + wrongFile := puppetRel("unkin-hello", "9.0.0") + wrongFile["file_uri"] = "/v3/files/unkin-other-9.0.0.tar.gz" + + d, err := MergePuppetReleases("unkin-hello", []PuppetMember{ + {RemoteName: "internal", Found: true, Releases: []map[string]any{puppetRel("unkin-hello", "0.1.0"), internalDup}}, + {RemoteName: "missing"}, + {RemoteName: "forge", Found: true, Releases: []map[string]any{absolute, puppetRel("unkin-hello", "0.2.0"), puppetRel("unkin-hello", "0.10.0"), wrongFile, puppetRel("unkin-other", "1.0.0")}}, + }) + if err != nil { + t.Fatal(err) + } + if got := versionsOf(d.Releases); got != "0.10.0,0.3.0,0.2.0,0.1.0" { + t.Fatalf("merged = %s, want newest semver first without foreign or misnamed releases", got) + } + for v, owner := range map[string]string{"0.1.0": "internal", "0.2.0": "internal", "0.3.0": "forge", "0.10.0": "forge"} { + if d.Owners[v] != owner { + t.Errorf("owner of %s = %q, want %q", v, d.Owners[v], owner) + } + } + if d.Releases[2]["tag"] != "internal" { + t.Errorf("duplicate 0.2.0 not taken from the first member: %v", d.Releases[2]) + } + if d.Releases[1]["file_uri"] != "/v3/files/unkin-hello-0.3.0.tar.gz" { + t.Errorf("file_uri = %v, want routed via the virtual", d.Releases[1]["file_uri"]) + } + + mod, err := PuppetModule(d) + if err != nil { + t.Fatal(err) + } + if mod["slug"] != "unkin-hello" || mod["current_release"].(map[string]any)["version"] != "0.10.0" || len(mod["releases"].([]map[string]any)) != 4 { + t.Fatalf("module = %v", mod) + } + + if _, err := MergePuppetReleases("unkin-hello", []PuppetMember{{RemoteName: "a"}, {RemoteName: "b"}}); !errors.Is(err, ErrNotFound) { + t.Fatalf("no member has it: want ErrNotFound, got %v", err) + } +} + +// forgePages answers a member's /v3/releases like the Forge, two per page. +func forgePages(slug string, versions []string, p string) ([]byte, error) { + _, raw, _ := strings.Cut(p, "?") + q, _ := url.ParseQuery(raw) + offset, _ := strconv.Atoi(q.Get("offset")) + end := min(offset+2, len(versions)) + var results []map[string]any + for _, v := range versions[min(offset, end):end] { + results = append(results, puppetRel(slug, v)) + } + return json.Marshal(map[string]any{"pagination": map[string]int{"total": len(versions)}, "results": results}) +} + +func puppetEngine(doc func(remote models.Remote, path string) ([]byte, error)) *Engine { + return &Engine{ + mergeTTL: time.Minute, + getRemote: func(_ context.Context, name string) (*models.Remote, error) { + rt := models.RepoTypeRemote + if name == "internal" { + rt = models.RepoTypeLocal + } + return &models.Remote{Name: name, RepoType: rt}, nil + }, + puppetDoc: func(_ context.Context, remote models.Remote, path string) ([]byte, error) { return doc(remote, path) }, + } +} + +func TestPuppetReleasesFollowsMemberPages(t *testing.T) { + var mu sync.Mutex + var asked []string + e := puppetEngine(func(remote models.Remote, p string) ([]byte, error) { + mu.Lock() + asked = append(asked, remote.Name+" "+p) + mu.Unlock() + switch remote.Name { + case "internal": + return forgePages("puppetlabs-stdlib", []string{"9.6.0"}, p) + case "gone": + return nil, &proxy.ProxyError{Status: 404} + } + return forgePages("puppetlabs-stdlib", []string{"9.6.0", "9.5.0", "9.4.0", "8.0.0", "7.0.0"}, p) + }) + virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "gone", "forge"}} + d, err := e.PuppetReleases(context.Background(), virt, "puppetlabs-stdlib") + if err != nil { + t.Fatal(err) + } + if versionsOf(d.Releases) != "9.6.0,9.5.0,9.4.0,8.0.0,7.0.0" || d.Owners["9.6.0"] != "internal" || d.Owners["7.0.0"] != "forge" { + t.Fatalf("merged = %s %v", versionsOf(d.Releases), d.Owners) + } + if len(asked) != 5 { + t.Fatalf("want 1 internal + 1 gone + 3 forge pages, asked %v", asked) + } + for _, a := range asked { + if strings.HasPrefix(a, "forge ") && !strings.Contains(a, "module=puppetlabs-stdlib") { + t.Errorf("member page without module filter: %q", a) + } + } + if _, err := e.PuppetReleases(context.Background(), virt, "puppetlabs-stdlib"); err != nil || len(asked) != 5 { + t.Fatalf("merge not cached: %v, %d fetches", err, len(asked)) + } +} + +func TestPuppetReleasesFailsClosed(t *testing.T) { + for name, doc := range map[string]func(models.Remote, string) ([]byte, error){ + "member error": func(r models.Remote, p string) ([]byte, error) { + if r.Name == "forge" { + return nil, &proxy.ProxyError{Status: 503} + } + return forgePages("unkin-hello", []string{"0.1.0"}, p) + }, + "short member list": func(models.Remote, string) ([]byte, error) { + return []byte(`{"pagination":{"total":3},"results":[]}`), nil + }, + } { + e := puppetEngine(doc) + virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "forge"}} + if _, err := e.PuppetReleases(context.Background(), virt, "unkin-hello"); err == nil || errors.Is(err, ErrNotFound) { + t.Errorf("%s: want a non-404 error, got %v", name, err) + } + } +} + +func TestPuppetRedirect(t *testing.T) { + e := puppetEngine(func(r models.Remote, p string) ([]byte, error) { + if r.Name == "internal" { + return forgePages("unkin-hello", []string{"0.1.0"}, p) + } + return forgePages("unkin-hello", []string{"0.2.0", "0.1.0"}, p) + }) + virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "forge"}} + for p, want := range map[string]string{ + "v3/files/unkin-hello-0.1.0.tar.gz": "https://a.example/api/v1/local/internal/v3/files/unkin-hello-0.1.0.tar.gz", + "v3/releases/unkin-hello-0.2.0": "https://a.example/api/v1/remote/forge/v3/releases/unkin-hello-0.2.0", + } { + if loc, err := e.PuppetRedirect(context.Background(), virt, p, "https://a.example"); err != nil || loc != want { + t.Errorf("%s -> %q %v, want %q", p, loc, err, want) + } + } + for _, p := range []string{"v3/files/unkin-hello-0.3.0.tar.gz", "v3/files/unkin-hello-0.1.0.zip", "v3/releases/nope", "v3/users/unkin"} { + if _, err := e.PuppetRedirect(context.Background(), virt, p, "https://a.example"); !errors.Is(err, ErrNotFound) { + t.Errorf("%s: want ErrNotFound, got %v", p, err) + } + } +} + +func TestPuppetReleasesCachesMissingModule(t *testing.T) { + var mu sync.Mutex + asked := map[string]int{} + e := puppetEngine(func(r models.Remote, p string) ([]byte, error) { + mu.Lock() + asked[r.Name]++ + mu.Unlock() + if r.Name == "internal" { + return nil, &proxy.ProxyError{Status: 404} + } + return forgePages("unkin-nope", nil, p) + }) + virt := models.Virtual{Name: "puppet", PackageType: models.PackagePuppet, Members: []string{"internal", "forge"}} + for range 2 { + if _, err := e.PuppetReleases(context.Background(), virt, "unkin-nope"); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + } + if asked["internal"] != 1 || asked["forge"] != 1 { + t.Fatalf("a module no member has must be cached, asked %v", asked) + } +}