Compare commits

..

1 Commits

Author SHA1 Message Date
unkin-agent 524c6db46f Add nuget v3 remote type
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
- add `nuget` package type proxying a NuGet v3 feed, with `index.json` mapped to the upstream service index
- rewrite upstream-origin URLs in served JSON per request so flat container and registration lookups go through the remote
- treat `.nupkg`/`.nuspec` as immutable and indexes as mutable
- add a `nuget.config` usage snippet to the UI and a nuget e2e caching case
2026-10-16 23:13:23 +11:00
21 changed files with 240 additions and 599 deletions
+2 -16
View File
@@ -1,6 +1,6 @@
# ArtifactAPI
Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
## Quick Start
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
| `terraform` | `*/versions` | `*/download/*/*` |
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
| `nuget` | `index.json` (service index), registration and flat container indexes | `.nupkg`, `.nuspec` |
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
Providers classify paths automatically. Users only configure what to proxy and TTLs.
@@ -261,21 +262,6 @@ there's nothing to provision. To bring your own key instead, point
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
### Local helm repository
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
generated from each chart's `Chart.yaml` (with sha256 digests) at
`/api/v1/local/<repo>/index.yaml`. Add the local as a member of a helm virtual
to serve it alongside remotes.
```bash
helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \
https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
helm repo add <repo> https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>
```
### Local docker registry
A local `docker` repo is a real container registry, not a mirror: it serves the
+1 -1
View File
@@ -23,7 +23,7 @@ already-running stack.
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, nuget) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
+1
View File
@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
{"nuget", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg", "nuget/e2epkg/1.0.0/e2epkg.1.0.0.nupkg"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
}
-71
View File
@@ -1,71 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"net/http"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it
// to a helm local, then drives helm repo add/search/pull/template against the
// local and against a helm virtual that has the local as a member.
func TestLocalHelmRealClient(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`)
defer deleteRepo(t, "local-helm")
createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`)
defer deleteVirtual(t, "vhelm")
helm := func(script string) string {
t.Helper()
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh",
"alpine/helm:3.16.4", "-c", "set -eu; "+script).Output()
if err != nil {
stderr := ""
if ee, ok := err.(*exec.ExitError); ok {
stderr = string(ee.Stderr)
}
t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr)
}
return string(out)
}
chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz")
uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip")
// Chart metadata is recorded asynchronously after upload; poll the index.
var body []byte
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) {
var resp *http.Response
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "")
if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") {
break
}
}
if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") {
t.Fatalf("index.yaml never listed the chart:\n%s", body)
}
for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} {
out := helm("helm repo add e2e " + repoURL + " >/dev/null && " +
"helm search repo e2e/e2echart && " +
"cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " +
"helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'")
if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") {
t.Fatalf("helm via %s:\n%s", repoURL, out)
}
}
}
+13 -1
View File
@@ -1,6 +1,7 @@
package v1
import (
"bytes"
"errors"
"fmt"
"io"
@@ -105,13 +106,24 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
}
defer result.Reader.Close()
var body io.Reader = result.Reader
if sr, ok := prov.(provider.ServeRewriter); ok && prov.Classify(path) == provider.Mutable {
raw, err := io.ReadAll(result.Reader)
if err != nil {
http.Error(w, "bad gateway", http.StatusBadGateway)
return
}
raw = sr.RewriteServed(raw, *remote, fmt.Sprintf("%s://%s", scheme(r), r.Host))
body, result.Size = bytes.NewReader(raw), int64(len(raw))
}
w.Header().Set("Content-Type", result.ContentType)
w.Header().Set("X-Artifact-Source", result.Source)
if result.Size > 0 {
w.Header().Set("X-Artifact-Size", fmt.Sprintf("%d", result.Size))
}
w.WriteHeader(http.StatusOK)
io.Copy(w, result.Reader)
io.Copy(w, body)
}
func (h *ProxyHandler) handleProxyHead(w http.ResponseWriter, r *http.Request) {
-44
View File
@@ -1,44 +0,0 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml)
VALUES ($1,$2,$3,$4,$5,$6)
ON CONFLICT (repo_name, file_path) DO NOTHING
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML)
return err
}
func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at
FROM helm_metadata
WHERE repo_name = $1
ORDER BY name, version, file_path
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []provider.HelmMetadata
for rows.Next() {
var m provider.HelmMetadata
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil {
return nil, err
}
result = append(result, m)
}
return result, rows.Err()
}
-175
View File
@@ -1,23 +1,12 @@
package helm
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"path"
"strings"
"time"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -67,167 +56,3 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
// ValidateUpload stores charts flat by filename so index.yaml can reference
// them with relative urls, as `helm repo index` does.
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := path.Base(path.Clean("/" + filePath))
if !strings.HasSuffix(filename, ".tgz") || filename == ".tgz" {
return "", "", fmt.Errorf("file must be a .tgz chart package")
}
return filename, "application/gzip", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
return map[string]any{
"filename": storagePath,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
reader, _, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
if err != nil {
slog.Error("helm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer func() { _ = reader.Close() }()
meta, err := parseChart(reader)
if err != nil {
slog.Error("helm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
meta.RepoName = repoName
meta.FilePath = storagePath
meta.ContentHash = contentHash
store, ok := db.(provider.HelmMetadataStore)
if !ok {
slog.Error("helm metadata: store does not support helm metadata", "repo", repoName)
return
}
if err := store.InsertHelmMetadata(ctx, meta); err != nil {
slog.Error("helm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("helm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version)
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
deleter, ok := db.(provider.HelmMetadataDeleter)
if !ok {
return nil
}
return deleter.DeleteHelmMetadata(ctx, repoName, storagePath)
}
func isIndexPath(p string) bool {
return strings.TrimPrefix(path.Clean("/"+p), "/") == "index.yaml"
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
if !isIndexPath(reqPath) {
return false
}
body, err := p.GenerateLocalIndex(r.Context(), files, repoName, reqPath)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
w.Header().Set("Content-Type", "text/yaml")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(body)
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, reqPath string) ([]byte, error) {
if !isIndexPath(reqPath) {
return nil, fmt.Errorf("unsupported index path: %q", reqPath)
}
reader, ok := files.(provider.HelmMetadataReader)
if !ok {
return nil, errors.New("helm metadata not available")
}
metas, err := reader.ListHelmMetadataEntries(ctx, repoName)
if err != nil {
return nil, err
}
return generateIndex(metas)
}
// generateIndex builds index.yaml from stored Chart.yaml documents; it is
// stamped from the newest upload so every replica serves identical bytes.
func generateIndex(metas []provider.HelmMetadata) ([]byte, error) {
entries := map[string][]map[string]yaml.Node{}
var newest time.Time
for _, m := range metas {
var chart map[string]yaml.Node
if err := yaml.Unmarshal([]byte(m.ChartYAML), &chart); err != nil {
return nil, fmt.Errorf("chart %s: %w", m.FilePath, err)
}
if chart == nil {
chart = map[string]yaml.Node{}
}
chart["name"] = node(m.Name)
chart["version"] = node(m.Version)
chart["urls"] = node([]string{m.FilePath})
chart["digest"] = node(strings.TrimPrefix(m.ContentHash, "sha256:"))
chart["created"] = node(m.CreatedAt.UTC().Format(time.RFC3339Nano))
entries[m.Name] = append(entries[m.Name], chart)
if m.CreatedAt.After(newest) {
newest = m.CreatedAt
}
}
return yaml.Marshal(map[string]any{
"apiVersion": "v1",
"entries": entries,
"generated": newest.UTC().Format(time.RFC3339Nano),
})
}
func node(v any) yaml.Node {
var n yaml.Node
_ = n.Encode(v)
return n
}
const maxChartYAML = 1 << 20
// parseChart reads the top-level <chart>/Chart.yaml from a packaged chart.
func parseChart(r io.Reader) (*provider.HelmMetadata, error) {
zr, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
tr := tar.NewReader(zr)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil, errors.New("no Chart.yaml in chart archive")
}
if err != nil {
return nil, err
}
dir, file := path.Split(path.Clean(hdr.Name))
if file != "Chart.yaml" || strings.Count(dir, "/") != 1 {
continue
}
raw, err := io.ReadAll(io.LimitReader(tr, maxChartYAML))
if err != nil {
return nil, err
}
var chart struct {
Name string `yaml:"name"`
Version string `yaml:"version"`
}
if err := yaml.Unmarshal(raw, &chart); err != nil {
return nil, fmt.Errorf("parse Chart.yaml: %w", err)
}
if chart.Name == "" || chart.Version == "" {
return nil, errors.New("chart metadata missing name or version")
}
return &provider.HelmMetadata{Name: chart.Name, Version: chart.Version, ChartYAML: string(raw)}, nil
}
}
-162
View File
@@ -1,162 +0,0 @@
package helm
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"io"
"net/http/httptest"
"testing"
"time"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func chartTgz(t *testing.T, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for name, body := range files {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
type fakeBlobs struct{ data []byte }
func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
}
type fakeStore struct {
inserted *provider.HelmMetadata
metas []provider.HelmMetadata
}
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
func (f *fakeStore) InsertHelmMetadata(_ context.Context, m *provider.HelmMetadata) error {
f.inserted = m
return nil
}
func (f *fakeStore) ListHelmMetadataEntries(context.Context, string) ([]provider.HelmMetadata, error) {
return f.metas, nil
}
func (f *fakeStore) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (f *fakeStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestValidateUpload(t *testing.T) {
p := &Provider{}
for in, want := range map[string]string{
"mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
"charts/mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
"../../mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
} {
sp, ct, err := p.ValidateUpload(in)
if err != nil || sp != want || ct != "application/gzip" {
t.Errorf("ValidateUpload(%q) = %q %q %v", in, sp, ct, err)
}
}
for _, bad := range []string{"index.yaml", "chart.tar.gz", ".tgz", ""} {
if _, _, err := p.ValidateUpload(bad); err == nil {
t.Errorf("ValidateUpload(%q) accepted", bad)
}
}
}
func TestAfterUploadParsesTopLevelChart(t *testing.T) {
data := chartTgz(t, map[string]string{
"mychart/charts/sub/Chart.yaml": "apiVersion: v2\nname: sub\nversion: 9.9.9\n",
"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0\ndescription: hi\n",
"mychart/values.yaml": "a: 1\n",
})
store := &fakeStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "mychart-1.0.tgz", "sha256:ab", fakeBlobs{data}, store)
m := store.inserted
if m == nil || m.Name != "mychart" || m.Version != "1.0" || m.FilePath != "mychart-1.0.tgz" || m.ContentHash != "sha256:ab" {
t.Fatalf("inserted %+v", m)
}
}
func TestAfterUploadRejectsChartWithoutMetadata(t *testing.T) {
store := &fakeStore{}
data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\n"})
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{data}, store)
if store.inserted != nil {
t.Fatal("chart without version was recorded")
}
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{[]byte("not gzip")}, store)
if store.inserted != nil {
t.Fatal("non-gzip upload was recorded")
}
}
func TestServeLocalIndex(t *testing.T) {
t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
t2 := t1.Add(time.Hour)
store := &fakeStore{metas: []provider.HelmMetadata{
{FilePath: "mychart-1.0.tgz", ContentHash: "sha256:aa", Name: "mychart", Version: "1.0", CreatedAt: t1,
ChartYAML: "apiVersion: v2\nname: mychart\nversion: 1.0\nappVersion: \"2\"\ndescription: hi\n"},
{FilePath: "other-0.2.0.tgz", ContentHash: "sha256:bb", Name: "other", Version: "0.2.0", CreatedAt: t2,
ChartYAML: "apiVersion: v2\nname: other\nversion: 0.2.0\n"},
}}
p := &Provider{}
if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), store, "r", "mychart-1.0.tgz") {
t.Fatal("chart path handled as index")
}
rec := httptest.NewRecorder()
if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), store, "r", "./index.yaml") {
t.Fatal("index.yaml not handled")
}
if rec.Code != 200 || rec.Header().Get("Content-Type") != "text/yaml" {
t.Fatalf("code=%d ct=%q", rec.Code, rec.Header().Get("Content-Type"))
}
var idx struct {
APIVersion string `yaml:"apiVersion"`
Generated string `yaml:"generated"`
Entries map[string][]map[string]any
}
if err := yaml.Unmarshal(rec.Body.Bytes(), &idx); err != nil {
t.Fatal(err)
}
if idx.APIVersion != "v1" || idx.Generated != t2.Format(time.RFC3339Nano) {
t.Errorf("header %+v", idx)
}
e := idx.Entries["mychart"][0]
if e["digest"] != "aa" || e["description"] != "hi" || e["appVersion"] != "2" || e["apiVersion"] != "v2" {
t.Errorf("entry %v", e)
}
if urls, _ := e["urls"].([]any); len(urls) != 1 || urls[0] != "mychart-1.0.tgz" {
t.Errorf("urls %v", e["urls"])
}
if e["version"] != "1.0" || e["name"] != "mychart" {
t.Errorf("version must stay a string: %#v", e["version"])
}
if len(idx.Entries["other"]) != 1 {
t.Errorf("other entries %v", idx.Entries["other"])
}
again, err := p.GenerateLocalIndex(context.Background(), store, "r", "index.yaml")
if err != nil || !bytes.Equal(again, rec.Body.Bytes()) {
t.Errorf("index not deterministic: %v", err)
}
}
+85
View File
@@ -0,0 +1,85 @@
// Package nuget proxies a NuGet v3 feed: the service index, registration pages
// and flat container indexes are mutable, .nupkg/.nuspec downloads immutable.
package nuget
import (
"context"
"encoding/json"
"net/http"
"net/url"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
// serviceIndex is the path clients use as their source URL; it maps to base_url.
const serviceIndex = "index.json"
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageNuGet }
func (p *Provider) Classify(path string) provider.Mutability {
if isPackage(path) {
return provider.Immutable
}
return provider.Mutable
}
func (p *Provider) ContentType(path string) string {
switch {
case strings.HasSuffix(path, ".nupkg"):
return "application/octet-stream"
case strings.HasSuffix(path, ".nuspec"):
return "application/xml"
}
return "application/json"
}
// UpstreamURL maps index.json to base_url (the upstream service index) and any
// other path to the same path on the base_url's origin, where nuget.org serves
// v3-flatcontainer/ and v3/registration*/.
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
path = strings.TrimLeft(path, "/")
if path == serviceIndex {
return remote.BaseURL
}
return origin(remote.BaseURL) + "/" + path
}
func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) {
return nil, nil
}
// RewriteServed points upstream-origin URLs in cached JSON back at this remote.
// It runs per request because the URLs must name the requesting host.
func (p *Provider) RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte {
from := origin(remote.BaseURL)
if from == "" || !json.Valid(body) {
return body
}
to := strings.TrimRight(proxyBaseURL, "/") + "/api/v1/remote/" + remote.Name
return []byte(strings.ReplaceAll(string(body), from, to))
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
func origin(baseURL string) string {
u, err := url.Parse(baseURL)
if err != nil || u.Host == "" {
return ""
}
return u.Scheme + "://" + u.Host
}
func isPackage(path string) bool {
return strings.HasSuffix(path, ".nupkg") || strings.HasSuffix(path, ".nuspec")
}
+68
View File
@@ -0,0 +1,68 @@
package nuget
import (
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
var _ provider.ServeRewriter = (*Provider)(nil)
var nugetOrg = models.Remote{Name: "nuget-org", BaseURL: "https://api.nuget.org/v3/index.json"}
func TestClassify(t *testing.T) {
p := &Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.13.0.3.nupkg", provider.Immutable},
{"v3-flatcontainer/newtonsoft.json/13.0.3/newtonsoft.json.nuspec", provider.Immutable},
{"v3-flatcontainer/newtonsoft.json/index.json", provider.Mutable},
{"v3/registration5-gz-semver2/newtonsoft.json/index.json", provider.Mutable},
{"index.json", provider.Mutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestUpstreamURL(t *testing.T) {
p := &Provider{}
tests := []struct {
path, want string
}{
{"index.json", "https://api.nuget.org/v3/index.json"},
{"/index.json", "https://api.nuget.org/v3/index.json"},
{"v3-flatcontainer/serilog/index.json", "https://api.nuget.org/v3-flatcontainer/serilog/index.json"},
{"v3/registration5-gz-semver2/serilog/index.json", "https://api.nuget.org/v3/registration5-gz-semver2/serilog/index.json"},
}
for _, tt := range tests {
if got := p.UpstreamURL(nugetOrg, tt.path); got != tt.want {
t.Errorf("UpstreamURL(%q) = %q, want %q", tt.path, got, tt.want)
}
}
}
func TestRewriteServed(t *testing.T) {
p := &Provider{}
body := `{"resources":[` +
`{"@id":"https://api.nuget.org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` +
`{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}`
want := `{"resources":[` +
`{"@id":"http://aa.example:8000/api/v1/remote/nuget-org/v3-flatcontainer/","@type":"PackageBaseAddress/3.0.0"},` +
`{"@id":"https://azuresearch-usnc.nuget.org/query","@type":"SearchQueryService"}]}`
if got := string(p.RewriteServed([]byte(body), nugetOrg, "http://aa.example:8000/")); got != want {
t.Errorf("RewriteServed =\n%s\nwant\n%s", got, want)
}
if got := string(p.RewriteServed([]byte(body), models.Remote{Name: "x"}, "http://aa.example")); got != body {
t.Error("remote without a parseable base_url must pass the body through")
}
cert := "\x30\x82https://api.nuget.org"
if got := string(p.RewriteServed([]byte(cert), nugetOrg, "http://aa.example")); got != cert {
t.Error("non-JSON bodies must pass through untouched")
}
}
+7 -24
View File
@@ -75,6 +75,13 @@ type RemoteServer interface {
ServeRemote(w http.ResponseWriter, r *http.Request, remote models.Remote, path, proxyBaseURL string, store RemoteMetadataStore) bool
}
// ServeRewriter rewrites a cached mutable remote response per request, for
// indexes whose URLs must name the requesting host and so cannot be rewritten
// before caching.
type ServeRewriter interface {
RewriteServed(body []byte, remote models.Remote, proxyBaseURL string) []byte
}
// RemoteMetadataStore is the persistence surface a RemoteServer needs to cache
// and read the metadata it derives per upstream asset. *database.DB satisfies it.
type RemoteMetadataStore interface {
@@ -240,27 +247,3 @@ func Get(t models.PackageType) (Provider, error) {
func All() map[models.PackageType]Provider {
return registry
}
type HelmMetadataStore interface {
InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error
}
type HelmMetadataDeleter interface {
DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error
}
type HelmMetadataReader interface {
ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error)
}
// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml
// entries carry every field helm reads.
type HelmMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
ChartYAML string
CreatedAt time.Time
}
+1
View File
@@ -29,6 +29,7 @@ import (
_ "git.unkin.net/unkin/artifactapi/internal/provider/goproxy"
_ "git.unkin.net/unkin/artifactapi/internal/provider/helm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/npm"
_ "git.unkin.net/unkin/artifactapi/internal/provider/nuget"
_ "git.unkin.net/unkin/artifactapi/internal/provider/puppet"
_ "git.unkin.net/unkin/artifactapi/internal/provider/pypi"
"git.unkin.net/unkin/artifactapi/internal/provider/rpm"
+39 -46
View File
@@ -16,17 +16,22 @@ func init() {
type HelmMerger struct{}
type helmIndex struct {
APIVersion string `yaml:"apiVersion"`
Entries map[string][]map[string]yaml.Node `yaml:"entries"`
Generated string `yaml:"generated,omitempty"`
APIVersion string `yaml:"apiVersion"`
Entries map[string][]helmChartVersion `yaml:"entries"`
Generated string `yaml:"generated,omitempty"`
}
type helmChartVersion struct {
Name string `yaml:"name"`
Version string `yaml:"version"`
URLs []string `yaml:"urls"`
rest map[string]any
}
// MergeIndexes keeps each chart version's fields as raw nodes so digest,
// apiVersion and the rest survive the merge untouched; only urls are rewritten.
func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
merged := &helmIndex{
APIVersion: "v1",
Entries: make(map[string][]map[string]yaml.Node),
Entries: make(map[string][]helmChartVersion),
}
seen := map[string]map[string]bool{}
@@ -42,26 +47,42 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
seen[chart] = map[string]bool{}
}
for _, ver := range versions {
version := ver["version"].Value
if seen[chart][version] {
key := chart + ":" + ver.Version
if seen[chart][ver.Version] {
continue
}
seen[chart][version] = true
seen[chart][ver.Version] = true
if urlsNode, ok := ver["urls"]; ok && proxyBaseURL != "" {
var urls []string
if err := urlsNode.Decode(&urls); err != nil {
continue
if proxyBaseURL != "" {
routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
}
rewriteHelmURLs(urls, member, proxyBaseURL)
var n yaml.Node
if err := n.Encode(urls); err != nil {
return nil, err
baseHost := extractHost(member.BaseURL)
for i, u := range ver.URLs {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
if baseHost != "" && extractHost(u) != baseHost {
continue
}
relPath := extractPathRelativeToBase(u, member.BaseURL)
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
relPath)
} else {
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
u)
}
}
ver["urls"] = n
}
merged.Entries[chart] = append(merged.Entries[chart], ver)
_ = key
}
}
}
@@ -69,34 +90,6 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
return yaml.Marshal(merged)
}
func rewriteHelmURLs(urls []string, member MemberIndex, proxyBaseURL string) {
routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
}
baseHost := extractHost(member.BaseURL)
for i, u := range urls {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
if baseHost != "" && extractHost(u) != baseHost {
continue
}
relPath := extractPathRelativeToBase(u, member.BaseURL)
urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
relPath)
} else {
urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
u)
}
}
}
func extractHost(rawURL string) string {
idx := strings.Index(rawURL, "://")
if idx == -1 {
-33
View File
@@ -122,36 +122,3 @@ entries:
t.Error("expected fallback repo to be excluded for duplicate")
}
}
func TestHelmMerger_PreservesChartFields(t *testing.T) {
m := &virtual.HelmMerger{}
body := []byte(`apiVersion: v1
entries:
app:
- apiVersion: v2
name: app
version: "1.0"
appVersion: "3.1"
description: in-house
digest: deadbeef
urls:
- app-1.0.tgz
`)
result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "charts", RepoType: "local", Body: body}}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
out := string(result)
for _, want := range []string{
"digest: deadbeef",
"apiVersion: v2",
`appVersion: "3.1"`,
"description: in-house",
`version: "1.0"`,
"https://proxy.example.com/api/v1/local/charts/app-1.0.tgz",
} {
if !strings.Contains(out, want) {
t.Errorf("merged index missing %q:\n%s", want, out)
}
}
}
-13
View File
@@ -1,13 +0,0 @@
CREATE TABLE IF NOT EXISTS helm_metadata (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
chart_yaml TEXT NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE INDEX IF NOT EXISTS idx_helm_metadata_repo ON helm_metadata(repo_name);
+2
View File
@@ -17,6 +17,7 @@ const (
PackageTerraform PackageType = "terraform"
PackageGoProxy PackageType = "goproxy"
PackageCargo PackageType = "cargo"
PackageNuGet PackageType = "nuget"
PackageGitHubRPM PackageType = "github_rpm"
PackageGitHubDeb PackageType = "github_deb"
PackageGitHubAlpine PackageType = "github_alpine"
@@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{
PackageTerraform: true,
PackageGoProxy: true,
PackageCargo: true,
PackageNuGet: true,
PackageGitHubRPM: true,
PackageGitHubDeb: true,
PackageGitHubAlpine: true,
+1
View File
@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
models.PackageTerraform,
models.PackageGoProxy,
models.PackageCargo,
models.PackageNuGet,
models.PackageGitHubRPM,
models.PackageGitHubDeb,
models.PackageGitHubAlpine,
+18 -13
View File
@@ -167,23 +167,12 @@ docker push ${host}/${name}/myapp:latest`,
case 'helm':
return [
{
title: isLocal ? 'Add the Helm repo (index.yaml auto-generated)' : 'Add the Helm repo',
title: 'Add the Helm repo',
language: 'bash',
code: `helm repo add ${name} ${isLocal ? `${url}/api/v1/local/${name}` : proxy}/
code: `helm repo add ${name} ${proxy}/
helm repo update
helm install <release> ${name}/<chart>`,
},
...(isLocal
? [
{
title: 'Publish a chart',
language: 'bash',
code: `helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \\
${url}/api/v2/remotes/${name}/files/mychart-0.1.0.tgz`,
},
]
: []),
];
case 'alpine':
@@ -254,6 +243,22 @@ index = "sparse+${proxy}/"`,
},
];
case 'nuget':
return [
{
title: 'Use this feed instead of nuget.org',
language: 'xml',
code: `<!-- nuget.config -->
<configuration>
<packageSources>
<clear />
<add key="${name}" value="${proxy}/index.json" protocolVersion="3" />
</packageSources>
</configuration>`,
note: 'Serve artifactapi over HTTPS: NuGet rejects the repository signatures resource over plain HTTP.',
},
];
case 'puppet':
return [
{
+1
View File
@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
terraform: 'blue',
goproxy: 'green',
cargo: 'yellow',
nuget: 'blue',
};
export function Locals() {
+1
View File
@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
terraform: 'blue',
goproxy: 'green',
cargo: 'yellow',
nuget: 'blue',
};
export function Remotes() {