diff --git a/README.md b/README.md index 4f0a78c..d89e501 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,10 @@ Each type's paths, Terraform examples and local/virtual behaviour are in `docs/< - `deb`: [remote](docs/deb/remote.md), [remote-github](docs/deb/remote-github.md) - `docker`: [remote](docs/docker/remote.md), [local](docs/docker/local.md) - `generic`: [remote](docs/generic/remote.md) +- `gems`: [remote](docs/gems/remote.md) - `goproxy`: [remote](docs/goproxy/remote.md) - `helm`: [remote](docs/helm/remote.md), [local](docs/helm/local.md), [virtual](docs/helm/virtual.md) -- `npm`: [remote](docs/npm/remote.md) +- `npm`: [remote](docs/npm/remote.md), [local](docs/npm/local.md) - `nuget`: [remote](docs/nuget/remote.md) - `puppet`: [remote](docs/puppet/remote.md), [local](docs/puppet/local.md) - `pypi`: [remote](docs/pypi/remote.md) @@ -129,6 +130,7 @@ No patterns + no blocklist = open proxy. Provider handles mutability classificat ``` GET /api/v1/remote/{name}/{path} Proxy/cache artifact GET /api/v1/virtual/{name}/{path} Virtual repo (merged index) +PUT /api/v1/local/{name}/{package} npm publish (local npm repos) GET /v2/{name}/{path} Docker Registry v2 ``` diff --git a/docs/common/caching.md b/docs/common/caching.md index 40d70e3..20fd733 100644 --- a/docs/common/caching.md +++ b/docs/common/caching.md @@ -12,8 +12,8 @@ Every proxied request on a remote is classified as **denied**, **immutable** or | `immutable_patterns` | Regexes forcing a path immutable. | | `mutable_patterns` | Regexes forcing a path mutable. | | `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. | -| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires, and the API stores `0` when the field is omitted. | -| `check_mutable` | Stores the upstream ETag and tries a conditional `HEAD` (`If-None-Match`) when a mutable path is fetched. The ETag expires with the `mutable_ttl` freshness key, so an expired mutable path is refetched in full, not revalidated. | +| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires. Omitted on create, it defaults to `3600`; omitted on update, the stored value is kept. | +| `check_mutable` | When a mutable path's `mutable_ttl` expires, sends a conditional `HEAD` (`If-None-Match`) with the stored upstream ETag. A `304` re-arms the TTL and serves the stored copy; anything else refetches in full. The ETag is kept for `max(10 × mutable_ttl, 7 days)`, re-armed on each `304`, so idle paths age out. It is only stored when `check_mutable` is set and `mutable_ttl` is non-zero, and never for immutable paths. | | `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. | | `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. | | `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. | @@ -46,7 +46,7 @@ So `blocklist` beats the allowlist, and `immutable_patterns` beats |---|---|---| | Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider | | Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` | -| After expiry | refetched from upstream | refetched from upstream (the ETag expires with the TTL, so `check_mutable` does not revalidate it) | +| After expiry | refetched from upstream | revalidated with `If-None-Match` when `check_mutable` is set and an ETag is stored, else refetched | | Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same | Concurrent misses on one path share a fetch lock. Waiters poll the store for @@ -60,6 +60,7 @@ curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects//*` and takes no lock. - Returns 204, or 404 for an unknown remote. @@ -94,7 +95,7 @@ Force a moving path mutable, e.g. branch tarballs on `generic`: Each type's page lists its auto-detected mutable and immutable paths: [alpine](../alpine/remote.md), [alpine github](../alpine/remote-github.md), [cargo](../cargo/remote.md), [deb](../deb/remote.md), [deb github](../deb/remote-github.md), -[docker](../docker/remote.md), [generic](../generic/remote.md), [goproxy](../goproxy/remote.md), +[docker](../docker/remote.md), [generic](../generic/remote.md), [gems](../gems/remote.md), [goproxy](../goproxy/remote.md), [helm](../helm/remote.md), [npm](../npm/remote.md), [nuget](../nuget/remote.md), [puppet](../puppet/remote.md), [pypi](../pypi/remote.md), [rpm](../rpm/remote.md), [rpm github](../rpm/remote-github.md), [terraform](../terraform/remote.md). diff --git a/docs/gems/remote.md b/docs/gems/remote.md new file mode 100644 index 0000000..9f4707b --- /dev/null +++ b/docs/gems/remote.md @@ -0,0 +1,7 @@ +# `gems` remote + +Shared classification, TTL and eviction rules: [caching](../common/caching.md). + +| Type | Mutable (auto-detected) | Immutable (auto-detected) | +|---|---|---| +| `gems` | compact index (`versions`, `info/*`, `names`), `*specs.4.8.gz` | `gems/*.gem`, `quick/Marshal.4.8/*.gemspec.rz` | diff --git a/docs/npm/local.md b/docs/npm/local.md new file mode 100644 index 0000000..798a48b --- /dev/null +++ b/docs/npm/local.md @@ -0,0 +1,14 @@ +# Local npm registry + +A local `npm` repo is a real npm registry: `npm publish` (scoped and unscoped) +and `npm install` work against `/api/v1/local//`. npm requires an +`_authToken` for the registry before it will publish; any value works. + +```ini +# .npmrc +registry=https://artifactapi.example.com/api/v1/local// +//artifactapi.example.com/api/v1/local//:_authToken=unused +``` + +Versions are immutable: republishing an existing version is rejected. The +dist-tags sent with a publish (`latest`, or `--tag`) move to that version. diff --git a/e2e-docker/README.md b/e2e-docker/README.md index 70825fc..85364c2 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -27,7 +27,9 @@ already-running stack. verified against the origin fixture. - **Local uploads** — puppet (Forge v3 API via real `r10k` and `puppet module install`), generic (upload/download), pypi (wheel + generated `simple/` - index), rpm (real package + **automatic repodata** generation). + index), rpm (real package + **automatic repodata** generation), npm (stock + `node:22` client: `npm publish` unscoped + scoped, `npm install` + `require()`, + second version, republish rejected). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members. - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors diff --git a/e2e-docker/caching_gems_test.go b/e2e-docker/caching_gems_test.go new file mode 100644 index 0000000..9ef2c8b --- /dev/null +++ b/e2e-docker/caching_gems_test.go @@ -0,0 +1,7 @@ +//go:build dockere2e + +package e2edocker + +func init() { + cachingCases = append(cachingCases, cachingCase{"gems", "gems/mygem-1.0.0.gem", "gems/mygem-1.0.0.gem"}) +} diff --git a/e2e-docker/fixtures/gems/mygem-1.0.0.gem b/e2e-docker/fixtures/gems/mygem-1.0.0.gem new file mode 100644 index 0000000..666781d --- /dev/null +++ b/e2e-docker/fixtures/gems/mygem-1.0.0.gem @@ -0,0 +1 @@ +mygem-1.0.0 e2e fixture diff --git a/e2e-docker/npm_local_test.go b/e2e-docker/npm_local_test.go new file mode 100644 index 0000000..9455611 --- /dev/null +++ b/e2e-docker/npm_local_test.go @@ -0,0 +1,62 @@ +//go:build dockere2e + +package e2edocker + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// TestLocalNPMPublishInstall drives a stock npm client against a local npm +// repo: publish unscoped and scoped packages, install and require() them in a +// fresh project, publish a second version, and reject a republish. +func TestLocalNPMPublishInstall(t *testing.T) { + network := os.Getenv("COMPOSE_NETWORK") + internal := os.Getenv("ARTIFACTAPI_INTERNAL") + if network == "" || internal == "" { + t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh") + } + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker not available on the test host") + } + + name := fmt.Sprintf("e2e-npm-local-%d", time.Now().UnixNano()) + createRepo(t, fmt.Sprintf(`{"name":%q,"package_type":"npm","repo_type":"local"}`, name)) + defer deleteRepo(t, name) + + reg := strings.TrimRight(internal, "/") + "/api/v1/local/" + name + "/" + script := `set -euo pipefail +printf 'registry=%s\n%s:_authToken=unused\n' "$REG" "${REG#http:}" > ~/.npmrc +mk() { d="/w/src/$1-$2"; mkdir -p "$d"; cd "$d" + printf '{"name":"%s","version":"%s","main":"index.js"}' "$1" "$2" > package.json + echo "module.exports = '$1@$2';" > index.js; } +mk e2e-npm-plain 1.0.0 && npm publish +mk @e2e/npm-scoped 1.0.0 && npm publish +mk e2e-npm-plain 1.0.0 +if npm publish 2>/dev/null; then echo "republish accepted"; exit 1; fi +mkdir -p /w/app && cd /w/app && npm init -y >/dev/null +npm install --no-audit --cache /tmp/fresh-cache e2e-npm-plain @e2e/npm-scoped +node -e "console.log('REQ', require('e2e-npm-plain'), require('@e2e/npm-scoped'))" +mk e2e-npm-plain 1.1.0 && npm publish +echo "VERSIONS $(npm view e2e-npm-plain versions --json | tr -d ' \n')" +echo "LATEST $(npm view e2e-npm-plain dist-tags.latest)"` + + out, err := exec.Command("docker", "run", "--rm", "--network", network, + "-e", "REG="+reg, "node:22", "bash", "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("npm client run failed: %v\n%s", err, out) + } + for _, want := range []string{ + "REQ e2e-npm-plain@1.0.0 @e2e/npm-scoped@1.0.0", + `VERSIONS ["1.0.0","1.1.0"]`, + "LATEST 1.1.0", + } { + if !strings.Contains(string(out), want) { + t.Fatalf("missing %q in npm output:\n%s", want, out) + } + } +} diff --git a/internal/api/v1/npm.go b/internal/api/v1/npm.go new file mode 100644 index 0000000..e5caf25 --- /dev/null +++ b/internal/api/v1/npm.go @@ -0,0 +1,322 @@ +package v1 + +import ( + "bytes" + "context" + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/storage" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// Local npm repos speak the npm registry publish/read protocol. Per package: +// /-/-.tgz tarball (immutable) +// /-/-.json version manifest (immutable) +// /-/dist-tags/ version string (mutable) + +var ( + npmNameRe = regexp.MustCompile(`^(@[a-z0-9][a-z0-9._~-]*/)?[a-z0-9][a-z0-9._~-]*$`) + npmRefRe = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z.+_-]*$`) + npmSemverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) + + npmMaxPublishBytes int64 = 256 << 20 +) + +func (h *ProxyHandler) localNPMRemote(r *http.Request, name string) (*models.Remote, bool) { + remote, err := h.db.GetRemote(r.Context(), name) + if err != nil { + return nil, false + } + return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageNPM +} + +// npmPackageName returns the package a packument/publish path addresses, or "" +// when the path is something else (e.g. a tarball under /-/). +func npmPackageName(raw string) string { + p, err := url.PathUnescape(raw) + if err != nil { + return "" + } + p = strings.TrimSuffix(p, "/") + if !npmNameRe.MatchString(p) { + return "" + } + return p +} + +type npmRelease struct { + version string + manifest []byte + tarball []byte + tags []string +} + +func parseNPMPublish(body io.Reader, pkg string) (*npmRelease, error) { + var doc struct { + Name string `json:"name"` + DistTags map[string]string `json:"dist-tags"` + Versions map[string]json.RawMessage `json:"versions"` + Attachments map[string]struct { + Data string `json:"data"` + } `json:"_attachments"` + } + if err := json.NewDecoder(body).Decode(&doc); err != nil { + return nil, fmt.Errorf("invalid publish body: %w", err) + } + if doc.Name != pkg { + return nil, fmt.Errorf("body name %q does not match %q", doc.Name, pkg) + } + if len(doc.Versions) != 1 || len(doc.Attachments) != 1 { + return nil, errors.New("publish must carry exactly one version and one tarball") + } + + rel := &npmRelease{} + for v, m := range doc.Versions { + rel.version, rel.manifest = v, m + } + if !npmSemverRe.MatchString(rel.version) { + return nil, fmt.Errorf("invalid version %q", rel.version) + } + var meta struct { + Name string `json:"name"` + Version string `json:"version"` + Dist struct { + Integrity string `json:"integrity"` + Shasum string `json:"shasum"` + } `json:"dist"` + } + if err := json.Unmarshal(rel.manifest, &meta); err != nil { + return nil, fmt.Errorf("invalid version manifest: %w", err) + } + if meta.Name != pkg || meta.Version != rel.version { + return nil, errors.New("version manifest name/version mismatch") + } + + for _, a := range doc.Attachments { + tgz, err := base64.StdEncoding.DecodeString(a.Data) + if err != nil { + return nil, fmt.Errorf("invalid tarball encoding: %w", err) + } + rel.tarball = tgz + } + if err := verifyNPMDist(rel.tarball, meta.Dist.Integrity, meta.Dist.Shasum); err != nil { + return nil, err + } + + for tag, v := range doc.DistTags { + if !npmRefRe.MatchString(tag) || v != rel.version { + return nil, fmt.Errorf("dist-tag %q must point at the published version", tag) + } + rel.tags = append(rel.tags, tag) + } + return rel, nil +} + +func verifyNPMDist(tgz []byte, integrity, shasum string) error { + if b64, ok := strings.CutPrefix(integrity, "sha512-"); ok { + sum := sha512.Sum512(tgz) + if base64.StdEncoding.EncodeToString(sum[:]) != b64 { + return errors.New("tarball does not match dist.integrity") + } + return nil + } + if shasum != "" { + sum := sha1.Sum(tgz) + if hex.EncodeToString(sum[:]) != shasum { + return errors.New("tarball does not match dist.shasum") + } + return nil + } + return errors.New("version manifest carries no sha512 integrity or shasum") +} + +func npmError(w http.ResponseWriter, status int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +func (h *ProxyHandler) handleLocalPut(w http.ResponseWriter, r *http.Request) { + remote, ok := h.localNPMRemote(r, chi.URLParam(r, "localName")) + if !ok { + npmError(w, http.StatusMethodNotAllowed, "publish is only supported on local npm repositories") + return + } + pkg := npmPackageName(chi.URLParam(r, "*")) + if pkg == "" { + npmError(w, http.StatusBadRequest, "invalid package name") + return + } + rel, err := parseNPMPublish(http.MaxBytesReader(w, r.Body, npmMaxPublishBytes), pkg) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + npmError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("publish body exceeds %d bytes", npmMaxPublishBytes)) + return + } + npmError(w, http.StatusBadRequest, err.Error()) + return + } + + base := pkg[strings.LastIndex(pkg, "/")+1:] + stem := pkg + "/-/" + base + "-" + rel.version + // The manifest gates visibility and tags may only name visible versions, so + // the order is tarball, manifest, tags. A tarball left by a failed publish is + // accepted again only with identical bytes; an existing manifest always conflicts. + conflict := func() { + npmError(w, http.StatusConflict, fmt.Sprintf("%s@%s already exists; overwrites are not allowed", pkg, rel.version)) + } + ok, err = h.npmStoreImmutable(r.Context(), remote.Name, stem+".tgz", rel.tarball, "application/octet-stream") + if err != nil { + npmError(w, http.StatusInternalServerError, err.Error()) + return + } + if !ok { + conflict() + return + } + if _, err := h.npmStore(r.Context(), remote.Name, stem+".json", rel.manifest, "application/json", false); err != nil { + if errors.Is(err, database.ErrAlreadyExists) { + conflict() + return + } + npmError(w, http.StatusInternalServerError, err.Error()) + return + } + for _, tag := range rel.tags { + if _, err := h.npmStore(r.Context(), remote.Name, pkg+"/-/dist-tags/"+tag, []byte(rel.version), "text/plain", true); err != nil { + npmError(w, http.StatusInternalServerError, err.Error()) + return + } + } + + slog.Info("local npm package published", "repo", remote.Name, "package", pkg, "version", rel.version) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "id": pkg}) +} + +// npmStoreImmutable reports false when path already holds different bytes. +func (h *ProxyHandler) npmStoreImmutable(ctx context.Context, repo, path string, data []byte, contentType string) (bool, error) { + hash, err := h.npmStore(ctx, repo, path, data, contentType, false) + if !errors.Is(err, database.ErrAlreadyExists) { + return err == nil, err + } + existing, err := h.db.GetLocalFile(ctx, repo, path) + if err != nil || existing == nil { + return false, fmt.Errorf("read existing %s: %v", path, err) + } + return existing.ContentHash == hash, nil +} + +func (h *ProxyHandler) npmStore(ctx context.Context, repo, path string, data []byte, contentType string, mutable bool) (string, error) { + result, err := h.cas.Store(ctx, bytes.NewReader(data), contentType) + if err != nil { + return "", err + } + if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, contentType); err != nil { + return "", err + } + if mutable { + return result.ContentHash, h.db.UpsertLocalFile(ctx, repo, path, result.ContentHash) + } + return result.ContentHash, h.db.CreateLocalFile(ctx, repo, path, result.ContentHash) +} + +func (h *ProxyHandler) readLocalBlob(ctx context.Context, contentHash string) ([]byte, error) { + reader, _, err := h.store.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:"))) + if err != nil { + return nil, err + } + defer reader.Close() + return io.ReadAll(reader) +} + +func (h *ProxyHandler) serveNPMPackument(w http.ResponseWriter, r *http.Request, remote *models.Remote, pkg string) { + files, err := h.db.ListLocalFilesByPrefix(r.Context(), remote.Name, pkg+"/-/") + if err != nil { + npmError(w, http.StatusInternalServerError, err.Error()) + return + } + read := func(hash string) ([]byte, error) { return h.readLocalBlob(r.Context(), hash) } + tarballBase := fmt.Sprintf("%s://%s/api/v1/local/%s/", Scheme(r), r.Host, remote.Name) + doc, err := buildNPMPackument(pkg, files, read, tarballBase) + if err != nil { + npmError(w, http.StatusInternalServerError, err.Error()) + return + } + if doc == nil { + npmError(w, http.StatusNotFound, "not found") + return + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Artifact-Source", "local") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(doc) +} + +// buildNPMPackument returns nil when pkg has no published versions. +func buildNPMPackument(pkg string, files []database.LocalFile, read func(hash string) ([]byte, error), tarballBase string) (map[string]any, error) { + prefix := pkg + "/-/" + versions := map[string]any{} + tags := map[string]string{} + for _, f := range files { + // LIKE treats '_' as a wildcard, so the prefix match can over-select. + rest, ok := strings.CutPrefix(f.FilePath, prefix) + if !ok { + continue + } + if tag, isTag := strings.CutPrefix(rest, "dist-tags/"); isTag { + v, err := read(f.ContentHash) + if err != nil { + return nil, err + } + tags[tag] = string(v) + continue + } + stem, isManifest := strings.CutSuffix(rest, ".json") + if !isManifest { + continue + } + raw, err := read(f.ContentHash) + if err != nil { + return nil, err + } + var m map[string]any + if err := json.Unmarshal(raw, &m); err != nil { + return nil, fmt.Errorf("manifest %s: %w", f.FilePath, err) + } + dist, _ := m["dist"].(map[string]any) + if dist == nil { + dist = map[string]any{} + m["dist"] = dist + } + dist["tarball"] = tarballBase + prefix + stem + ".tgz" + v, _ := m["version"].(string) + versions[v] = m + } + if len(versions) == 0 { + return nil, nil + } + for tag, v := range tags { + if _, ok := versions[v]; !ok { + delete(tags, tag) + } + } + return map[string]any{"_id": pkg, "name": pkg, "dist-tags": tags, "versions": versions}, nil +} diff --git a/internal/api/v1/npm_handler_test.go b/internal/api/v1/npm_handler_test.go new file mode 100644 index 0000000..8b21c09 --- /dev/null +++ b/internal/api/v1/npm_handler_test.go @@ -0,0 +1,293 @@ +package v1 + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/storage" + "git.unkin.net/unkin/artifactapi/internal/testsupport" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +var ( + testDSN string + testMinio testsupport.MinioConn +) + +func TestMain(m *testing.M) { + ctx := context.Background() + dsn, termPG, err := testsupport.StartPostgres(ctx) + if err != nil { + os.Exit(m.Run()) + } + conn, termMinio, err := testsupport.StartMinio(ctx) + if err != nil { + termPG() + os.Exit(m.Run()) + } + testDSN, testMinio = dsn, conn + code := m.Run() + termMinio() + termPG() + os.Exit(code) +} + +func npmTestServer(t *testing.T) (*ProxyHandler, *httptest.Server) { + t.Helper() + if testDSN == "" { + t.Skip("Docker unavailable") + } + db, err := database.New(testDSN) + if err != nil { + t.Fatal(err) + } + t.Cleanup(db.Close) + var store *storage.S3 + for i := 0; i < 20; i++ { + if store, err = storage.NewS3(testMinio.Endpoint, testMinio.AccessKey, testMinio.SecretKey, "npm", false, ""); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if err != nil { + t.Fatal(err) + } + h := NewProxyHandler(nil, nil, db, store, nil) + r := chi.NewRouter() + r.Mount("/api/v1", h.Routes()) + srv := httptest.NewServer(r) + t.Cleanup(srv.Close) + return h, srv +} + +func npmRepo(t *testing.T, h *ProxyHandler, pt models.PackageType) string { + t.Helper() + name := fmt.Sprintf("npm-%s-%d", pt, time.Now().UnixNano()) + if err := h.db.CreateRemote(context.Background(), &models.Remote{Name: name, PackageType: pt, RepoType: models.RepoTypeLocal}); err != nil { + t.Fatal(err) + } + return name +} + +func npmDo(t *testing.T, method, url, body string) (int, string) { + t.Helper() + req, _ := http.NewRequest(method, url, strings.NewReader(body)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + b, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(b) +} + +func packumentVersions(t *testing.T, url string) map[string]any { + t.Helper() + code, body := npmDo(t, http.MethodGet, url, "") + if code != http.StatusOK { + t.Fatalf("GET %s = %d: %s", url, code, body) + } + var doc struct { + Versions map[string]any `json:"versions"` + } + if err := json.Unmarshal([]byte(body), &doc); err != nil { + t.Fatal(err) + } + return doc.Versions +} + +func packumentTags(t *testing.T, url string) map[string]string { + t.Helper() + _, body := npmDo(t, http.MethodGet, url, "") + var doc struct { + Tags map[string]string `json:"dist-tags"` + } + if err := json.Unmarshal([]byte(body), &doc); err != nil { + t.Fatal(err) + } + return doc.Tags +} + +func concurrentPuts(t *testing.T, url string, bodies ...string) []int { + t.Helper() + var wg sync.WaitGroup + codes := make([]int, len(bodies)) + for i, b := range bodies { + wg.Add(1) + go func() { + defer wg.Done() + codes[i], _ = npmDo(t, http.MethodPut, url, b) + }() + } + wg.Wait() + return codes +} + +func TestNPMPublishHandler(t *testing.T) { + h, srv := npmTestServer(t) + repo := npmRepo(t, h, models.PackageNPM) + base := srv.URL + "/api/v1/local/" + repo + "/" + + t.Run("scoped name via %2f", func(t *testing.T) { + body := publishBody(t, "@acme/widget", []byte("scoped tgz"), publishOpts{}) + if code, b := npmDo(t, http.MethodPut, base+"@acme%2fwidget", body); code != http.StatusCreated { + t.Fatalf("publish = %d: %s", code, b) + } + v := packumentVersions(t, base+"@acme%2fwidget") + dist := v["1.0.0"].(map[string]any)["dist"].(map[string]any) + tarball := dist["tarball"].(string) + if !strings.HasPrefix(tarball, base+"@acme/widget/-/widget-1.0.0.tgz") { + t.Fatalf("tarball = %s", tarball) + } + if code, b := npmDo(t, http.MethodGet, tarball, ""); code != http.StatusOK || b != "scoped tgz" { + t.Fatalf("tarball fetch = %d %q", code, b) + } + }) + + t.Run("republish", func(t *testing.T) { + if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusCreated { + t.Fatalf("first publish = %d: %s", code, b) + } + if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("different"), publishOpts{})); code != http.StatusConflict { + t.Errorf("different bytes = %d, want 409", code) + } + if code, b := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v2"), publishOpts{version: "1.1.0"})); code != http.StatusCreated { + t.Fatalf("publish 1.1.0 = %d: %s", code, b) + } + if code, _ := npmDo(t, http.MethodPut, base+"repub", publishBody(t, "repub", []byte("v1"), publishOpts{})); code != http.StatusConflict { + t.Errorf("identical republish = %d, want 409", code) + } + if tags := packumentTags(t, base+"repub"); tags["latest"] != "1.1.0" { + t.Errorf("latest = %q, want 1.1.0", tags["latest"]) + } + }) + + t.Run("rejected republish leaves tags", func(t *testing.T) { + for _, v := range []string{"1.0.0", "1.1.0"} { + if code, b := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz "+v), publishOpts{version: v})); code != http.StatusCreated { + t.Fatalf("publish %s = %d: %s", v, code, b) + } + } + if code, _ := npmDo(t, http.MethodPut, base+"rej", publishBody(t, "rej", []byte("tgz 1.0.0"), publishOpts{shasum: "x"})); code != http.StatusConflict { + t.Fatalf("re-PUT 1.0.0 = %d, want 409", code) + } + if tags := packumentTags(t, base+"rej"); tags["latest"] != "1.1.0" { + t.Errorf("latest = %q, want 1.1.0", tags["latest"]) + } + }) + + t.Run("mid-publish window", func(t *testing.T) { + if code, b := npmDo(t, http.MethodPut, base+"mid", publishBody(t, "mid", []byte("v1"), publishOpts{})); code != http.StatusCreated { + t.Fatalf("publish = %d: %s", code, b) + } + ctx := context.Background() + if _, err := h.npmStore(ctx, repo, "mid/-/mid-1.0.1.tgz", []byte("v2"), "application/octet-stream", false); err != nil { + t.Fatal(err) + } + if tags := packumentTags(t, base+"mid"); tags["latest"] != "1.0.0" { + t.Errorf("tarball-only 1.0.1: latest = %q", tags["latest"]) + } + if _, err := h.npmStore(ctx, repo, "mid/-/dist-tags/latest", []byte("1.0.1"), "text/plain", true); err != nil { + t.Fatal(err) + } + if tags := packumentTags(t, base+"mid"); tags["latest"] == "1.0.1" { + t.Error("latest points at invisible 1.0.1") + } + }) + + t.Run("concurrent identical", func(t *testing.T) { + body := publishBody(t, "same", []byte("same tgz"), publishOpts{}) + codes := concurrentPuts(t, base+"same", body, body) + if codes[0]+codes[1] != http.StatusCreated+http.StatusConflict || codes[0] == codes[1] { + t.Fatalf("codes = %v, want one 201 and one 409", codes) + } + if tags := packumentTags(t, base+"same"); len(tags) != 1 || tags["latest"] != "1.0.0" { + t.Errorf("tags = %v", tags) + } + }) + + t.Run("concurrent different manifests", func(t *testing.T) { + tgz := []byte("race tgz") + codes := concurrentPuts(t, base+"race", + publishBody(t, "race", tgz, publishOpts{shasum: "a", tags: map[string]string{"a": "1.0.0"}}), + publishBody(t, "race", tgz, publishOpts{shasum: "b", tags: map[string]string{"b": "1.0.0"}})) + won := map[int]string{} + for i, c := range codes { + won[c] = []string{"a", "b"}[i] + } + if len(won) != 2 || won[http.StatusCreated] == "" || won[http.StatusConflict] == "" { + t.Fatalf("codes = %v, want one 201 and one 409", codes) + } + v := packumentVersions(t, base+"race")["1.0.0"].(map[string]any) + if got := v["dist"].(map[string]any)["shasum"]; got != won[http.StatusCreated] { + t.Errorf("manifest shasum = %v, winner %s", got, won[http.StatusCreated]) + } + tags := packumentTags(t, base+"race") + if len(tags) != 1 || tags[won[http.StatusCreated]] != "1.0.0" { + t.Errorf("tags = %v, winner %s", tags, won[http.StatusCreated]) + } + }) + + t.Run("retry after tarball-only write", func(t *testing.T) { + tgz := []byte("half published") + if _, err := h.npmStore(context.Background(), repo, "retry/-/retry-1.0.0.tgz", tgz, "application/octet-stream", false); err != nil { + t.Fatal(err) + } + if code, _ := npmDo(t, http.MethodGet, base+"retry", ""); code != http.StatusNotFound { + t.Fatalf("tarball-only version visible: %d", code) + } + if code, _ := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", []byte("other bytes"), publishOpts{})); code != http.StatusConflict { + t.Errorf("retry with different bytes = %d, want 409", code) + } + if code, b := npmDo(t, http.MethodPut, base+"retry", publishBody(t, "retry", tgz, publishOpts{})); code != http.StatusCreated { + t.Fatalf("retry with identical bytes = %d: %s", code, b) + } + if _, ok := packumentVersions(t, base+"retry")["1.0.0"]; !ok { + t.Error("retried version missing from packument") + } + }) + + t.Run("concurrent versions", func(t *testing.T) { + codes := concurrentPuts(t, base+"conc", + publishBody(t, "conc", []byte("tgz 1.0.0"), publishOpts{version: "1.0.0"}), + publishBody(t, "conc", []byte("tgz 2.0.0"), publishOpts{version: "2.0.0"})) + if codes[0] != http.StatusCreated || codes[1] != http.StatusCreated { + t.Fatalf("codes = %v", codes) + } + v := packumentVersions(t, base+"conc") + if _, ok := v["1.0.0"]; !ok { + t.Error("1.0.0 missing") + } + if _, ok := v["2.0.0"]; !ok { + t.Error("2.0.0 missing") + } + }) + + t.Run("body too large", func(t *testing.T) { + old := npmMaxPublishBytes + npmMaxPublishBytes = 64 + defer func() { npmMaxPublishBytes = old }() + if code, _ := npmDo(t, http.MethodPut, base+"big", publishBody(t, "big", []byte("x"), publishOpts{})); code != http.StatusRequestEntityTooLarge { + t.Errorf("oversized body = %d, want 413", code) + } + }) + + t.Run("non-npm local", func(t *testing.T) { + generic := npmRepo(t, h, models.PackageGeneric) + if code, _ := npmDo(t, http.MethodPut, srv.URL+"/api/v1/local/"+generic+"/pkg", publishBody(t, "pkg", []byte("x"), publishOpts{})); code != http.StatusMethodNotAllowed { + t.Errorf("publish to generic local = %d, want 405", code) + } + }) +} diff --git a/internal/api/v1/npm_test.go b/internal/api/v1/npm_test.go new file mode 100644 index 0000000..82cf676 --- /dev/null +++ b/internal/api/v1/npm_test.go @@ -0,0 +1,166 @@ +package v1 + +import ( + "crypto/sha1" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/database" +) + +func TestNPMPackageName(t *testing.T) { + tests := map[string]string{ + "left-pad": "left-pad", + "left-pad/": "left-pad", + "@acme%2fwidget": "@acme/widget", + "@acme%2Fwidget": "@acme/widget", + "@acme/widget": "@acme/widget", + "left-pad/-/left-pad.tgz": "", + "@acme/widget/-/w-1.tgz": "", + "a/b": "", + "../etc": "", + "Upper": "", + "": "", + } + for in, want := range tests { + if got := npmPackageName(in); got != want { + t.Errorf("npmPackageName(%q) = %q, want %q", in, got, want) + } + } +} + +type publishOpts struct { + bodyName, manifestName, version, integrity, shasum string + tags map[string]string + extraVersion bool +} + +func publishBody(t *testing.T, pkg string, tgz []byte, o publishOpts) string { + t.Helper() + sha := sha512.Sum512(tgz) + sh1 := sha1.Sum(tgz) + if o.bodyName == "" { + o.bodyName = pkg + } + if o.manifestName == "" { + o.manifestName = pkg + } + if o.version == "" { + o.version = "1.0.0" + } + if o.integrity == "" { + o.integrity = "sha512-" + base64.StdEncoding.EncodeToString(sha[:]) + } + if o.shasum == "" { + o.shasum = hex.EncodeToString(sh1[:]) + } + if o.tags == nil { + o.tags = map[string]string{"latest": o.version} + } + versions := map[string]any{o.version: map[string]any{ + "name": o.manifestName, "version": o.version, "main": "index.js", + "dist": map[string]any{"integrity": o.integrity, "shasum": o.shasum, "tarball": "http://elsewhere/x.tgz"}, + }} + if o.extraVersion { + versions["2.0.0"] = map[string]any{"name": pkg, "version": "2.0.0"} + } + b, _ := json.Marshal(map[string]any{ + "_id": pkg, "name": o.bodyName, "dist-tags": o.tags, "versions": versions, + "_attachments": map[string]any{pkg + "-" + o.version + ".tgz": map[string]any{ + "content_type": "application/octet-stream", "data": base64.StdEncoding.EncodeToString(tgz), "length": len(tgz), + }}, + }) + return string(b) +} + +func TestParseNPMPublish(t *testing.T) { + tgz := []byte("fake tarball bytes") + + rel, err := parseNPMPublish(strings.NewReader(publishBody(t, "@acme/widget", tgz, publishOpts{tags: map[string]string{"latest": "1.0.0", "beta": "1.0.0"}})), "@acme/widget") + if err != nil { + t.Fatalf("valid scoped publish rejected: %v", err) + } + if rel.version != "1.0.0" || string(rel.tarball) != string(tgz) || len(rel.tags) != 2 { + t.Fatalf("unexpected release: %+v", rel) + } + + if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none", shasum: hex.EncodeToString(make([]byte, 20))})), "left-pad"); err == nil { + t.Fatal("shasum fallback should reject a mismatched tarball") + } + if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, publishOpts{integrity: "none"})), "left-pad"); err != nil { + t.Fatalf("matching shasum fallback rejected: %v", err) + } + + bad := map[string]publishOpts{ + "body name mismatch": {bodyName: "other"}, + "manifest name mismatch": {manifestName: "other"}, + "integrity mismatch": {integrity: "sha512-AAAA"}, + "path-like version": {version: "../1.0.0"}, + "non-semver version": {version: "1..2"}, + "leading-zero version": {version: "01.0.0"}, + "tag at another version": {tags: map[string]string{"latest": "9.9.9"}}, + "path-like tag": {tags: map[string]string{"../x": "1.0.0"}}, + "multiple versions at once": {extraVersion: true}, + } + for name, o := range bad { + if _, err := parseNPMPublish(strings.NewReader(publishBody(t, "left-pad", tgz, o)), "left-pad"); err == nil { + t.Errorf("%s: expected rejection", name) + } + } + if _, err := parseNPMPublish(strings.NewReader(`{"name":"left-pad","versions":{}}`), "left-pad"); err == nil { + t.Error("deprecate-style body without attachments should be rejected") + } +} + +func TestBuildNPMPackument(t *testing.T) { + blobs := map[string]string{ + "h1": `{"name":"my_pkg","version":"1.0.0","dist":{"shasum":"aa","tarball":"http://elsewhere/x.tgz"}}`, + "h2": `{"name":"my_pkg","version":"1.1.0"}`, + "h3": "1.1.0", + "h4": "2.0.0", + "hx": `{"name":"myXpkg","version":"6.6.6"}`, + } + read := func(h string) ([]byte, error) { + b, ok := blobs[h] + if !ok { + return nil, errors.New("missing blob") + } + return []byte(b), nil + } + files := []database.LocalFile{ + {FilePath: "my_pkg/-/dist-tags/latest", ContentHash: "h3"}, + {FilePath: "my_pkg/-/dist-tags/next", ContentHash: "h4"}, + {FilePath: "my_pkg/-/my_pkg-1.0.0.json", ContentHash: "h1"}, + {FilePath: "my_pkg/-/my_pkg-1.0.0.tgz", ContentHash: "nope"}, + {FilePath: "my_pkg/-/my_pkg-1.1.0.json", ContentHash: "h2"}, + {FilePath: "myXpkg/-/myXpkg-6.6.6.json", ContentHash: "hx"}, + } + + doc, err := buildNPMPackument("my_pkg", files, read, "http://reg/api/v1/local/npm/") + if err != nil { + t.Fatal(err) + } + if tags := doc["dist-tags"].(map[string]string); tags["latest"] != "1.1.0" || len(tags) != 1 { + t.Errorf("dist-tags = %v", tags) + } + versions := doc["versions"].(map[string]any) + if len(versions) != 2 { + t.Fatalf("versions = %v (LIKE over-match must be dropped)", versions) + } + v1 := versions["1.0.0"].(map[string]any)["dist"].(map[string]any) + if v1["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.0.0.tgz" || v1["shasum"] != "aa" { + t.Errorf("1.0.0 dist = %v", v1) + } + if versions["1.1.0"].(map[string]any)["dist"].(map[string]any)["tarball"] != "http://reg/api/v1/local/npm/my_pkg/-/my_pkg-1.1.0.tgz" { + t.Error("1.1.0 missing rewritten tarball") + } + + if doc, err := buildNPMPackument("absent", files, read, "x/"); err != nil || doc != nil { + t.Errorf("absent package: doc=%v err=%v", doc, err) + } +} diff --git a/internal/api/v2/remotes.go b/internal/api/v2/remotes.go index a8f679e..b5c3a82 100644 --- a/internal/api/v2/remotes.go +++ b/internal/api/v2/remotes.go @@ -67,8 +67,12 @@ func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, remote) } +// defaultMutableTTL matches the remotes.mutable_ttl column default; a create +// that omits mutable_ttl gets it, an explicit 0 still means never expire. +const defaultMutableTTL = 3600 + func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) { - var remote models.Remote + remote := models.Remote{MutableTTL: defaultMutableTTL} if err := json.NewDecoder(r.Body).Decode(&remote); err != nil { http.Error(w, "invalid json", http.StatusBadRequest) return @@ -114,7 +118,15 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) { func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") + // Capture the current backend before the update so we can tell whether the + // remote's base_url (its upstream) changed. A read failure just means we + // skip the freshness flush; it must not block the update. + existing, err := h.db.GetRemote(r.Context(), name) + oldKnown := err == nil var remote models.Remote + if oldKnown { + remote.MutableTTL = existing.MutableTTL + } if err := json.NewDecoder(r.Body).Decode(&remote); err != nil { http.Error(w, "invalid json", http.StatusBadRequest) return @@ -132,13 +144,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - // Capture the current backend before the update so we can tell whether the - // remote's base_url (its upstream) changed. A read failure just means we - // skip the freshness flush; it must not block the update. - oldBaseURL, oldKnown := "", false - if existing, err := h.db.GetRemote(r.Context(), name); err == nil { - oldBaseURL, oldKnown = existing.BaseURL, true - } if err := h.db.UpdateRemote(r.Context(), &remote); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -147,13 +152,13 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) { // Release / APKINDEX): purge it so the next request re-fetches from the new // upstream instead of serving stale data until TTL expiry. A flush failure // is logged but does not fail the request — the DB update already landed. - if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil { + if oldKnown && existing.BaseURL != remote.BaseURL && h.cache != nil { if err := h.cache.FlushRemote(r.Context(), name); err != nil { slog.Warn("flush cached metadata after base_url change failed", "remote", name, "error", err) } else { slog.Info("flushed cached metadata after base_url change", - "remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL) + "remote", name, "old_base_url", existing.BaseURL, "new_base_url", remote.BaseURL) } } writeJSON(w, http.StatusOK, remote) @@ -165,6 +170,11 @@ func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + if h.cache != nil { + if err := h.cache.FlushRemote(r.Context(), name); err != nil { + slog.Warn("flush cached metadata after delete failed", "remote", name, "error", err) + } + } w.WriteHeader(http.StatusNoContent) } diff --git a/internal/api/v2/remotes_flush_test.go b/internal/api/v2/remotes_flush_test.go index 09a25b5..4f8036a 100644 --- a/internal/api/v2/remotes_flush_test.go +++ b/internal/api/v2/remotes_flush_test.go @@ -94,3 +94,28 @@ func TestUpdateFlushFailureStillSucceeds(t *testing.T) { t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls) } } + +// Deleting a remote flushes its cached state so a re-created remote of the +// same name starts cold; a flush error does not fail the delete. +func TestDeleteFlushesCache(t *testing.T) { + if testDSN == "" { + t.Skip("Docker unavailable") + } + db, err := database.New(testDSN) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + for _, ff := range []*fakeFlusher{{}, {err: errors.New("redis down")}} { + const name = "rpm-flush-delete" + seedRemote(t, db, name, "https://example.com/repo") + h := NewRemotesHandler(db, ff, nil).Routes() + if c := do(t, h, "DELETE", "/"+name, ""); c != 204 { + t.Fatalf("delete = %d, want 204", c) + } + if len(ff.calls) != 1 || ff.calls[0] != name { + t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name) + } + } +} diff --git a/internal/api/v2/remotes_ttl_test.go b/internal/api/v2/remotes_ttl_test.go new file mode 100644 index 0000000..7beb859 --- /dev/null +++ b/internal/api/v2/remotes_ttl_test.go @@ -0,0 +1,57 @@ +package v2 + +import ( + "context" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/database" +) + +// An omitted mutable_ttl defaults to 3600 on create and keeps the stored value +// on update; an explicit 0 is stored as 0 in both. +func TestMutableTTLOmittedVsZero(t *testing.T) { + if testDSN == "" { + t.Skip("Docker unavailable") + } + db, err := database.New(testDSN) + if err != nil { + t.Fatal(err) + } + defer db.Close() + h := NewRemotesHandler(db, nil, nil).Routes() + ttl := func(name string) int { + t.Helper() + r, err := db.GetRemote(context.Background(), name) + if err != nil { + t.Fatalf("get %s: %v", name, err) + } + return r.MutableTTL + } + const body = `"package_type":"rpm","base_url":"https://example.com/repo"` + + steps := []struct { + method, path, json string + want int + }{ + {"POST", "/", `{"name":"ttl-omit",` + body + `}`, 3600}, + {"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":120}`, 120}, + {"PUT", "/ttl-omit", `{` + body + `}`, 120}, + {"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":0}`, 0}, + {"PUT", "/ttl-omit", `{` + body + `}`, 0}, + } + for i, s := range steps { + if c := do(t, h, s.method, s.path, s.json); c/100 != 2 { + t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c) + } + if got := ttl("ttl-omit"); got != s.want { + t.Errorf("step %d: mutable_ttl = %d, want %d", i, got, s.want) + } + } + + if c := do(t, h, "POST", "/", `{"name":"ttl-zero",`+body+`,"mutable_ttl":0}`); c != 201 { + t.Fatalf("create explicit 0 = %d", c) + } + if got := ttl("ttl-zero"); got != 0 { + t.Errorf("explicit 0 stored as %d", got) + } +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 4093a60..8bcb380 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -86,6 +86,12 @@ func TestETagAndToken(t *testing.T) { if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` { t.Errorf("etag = %q", v) } + if err := testRedis.SetETag(ctx, "r", "p", "", 0); err != nil { + t.Fatal(err) + } + if v, _ := testRedis.GetETag(ctx, "r", "p"); v != "" { + t.Errorf("empty etag should drop the stored one, got %q", v) + } if v, _ := testRedis.GetToken(ctx, "missing"); v != "" { t.Error("missing token should be empty") diff --git a/internal/cache/redis.go b/internal/cache/redis.go index b33e0cc..eeca6c5 100644 --- a/internal/cache/redis.go +++ b/internal/cache/redis.go @@ -57,8 +57,12 @@ func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error { return r.client.Del(ctx, key).Err() } +// SetETag stores etag for path, or drops the stored one when etag is empty. func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error { key := fmt.Sprintf("etag:%s:%s", remote, path) + if etag == "" { + return r.client.Del(ctx, key).Err() + } return r.client.Set(ctx, key, etag, ttl).Err() } diff --git a/internal/provider/all/gems.go b/internal/provider/all/gems.go new file mode 100644 index 0000000..ebd6ccf --- /dev/null +++ b/internal/provider/all/gems.go @@ -0,0 +1,3 @@ +package all + +import _ "git.unkin.net/unkin/artifactapi/internal/provider/gems" diff --git a/internal/provider/gems/gems.go b/internal/provider/gems/gems.go new file mode 100644 index 0000000..39d324e --- /dev/null +++ b/internal/provider/gems/gems.go @@ -0,0 +1,62 @@ +// Package gems proxies a RubyGems source: mutable compact index (versions, +// info/*, names) and legacy specs indexes, plus immutable .gem and +// quick/Marshal.4.8 gemspec downloads. +package gems + +import ( + "context" + "net/http" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/auth" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func init() { + provider.Register(&Provider{}) +} + +type Provider struct{} + +func (p *Provider) Type() models.PackageType { return models.PackageGems } + +func (p *Provider) Classify(path string) provider.Mutability { + if isGem(path) || isQuickSpec(path) { + return provider.Immutable + } + return provider.Mutable +} + +func (p *Provider) ContentType(path string) string { + path = strings.TrimLeft(path, "/") + if path == "versions" || path == "names" || strings.HasPrefix(path, "info/") { + return "text/plain; charset=utf-8" + } + if strings.HasPrefix(path, "api/v1/") && strings.HasSuffix(path, ".json") { + return "application/json" + } + return "application/octet-stream" +} + +func (p *Provider) UpstreamURL(remote models.Remote, path string) string { + return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/") +} + +func (p *Provider) RewriteResponse(_ []byte, _ models.Remote, _ string) ([]byte, error) { + return nil, nil +} + +func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) { + return auth.BasicHeaders(remote), nil +} + +func isGem(path string) bool { + path = strings.TrimLeft(path, "/") + return strings.HasPrefix(path, "gems/") && strings.HasSuffix(path, ".gem") +} + +func isQuickSpec(path string) bool { + path = strings.TrimLeft(path, "/") + return strings.HasPrefix(path, "quick/Marshal.4.8/") && strings.HasSuffix(path, ".gemspec.rz") +} diff --git a/internal/provider/gems/gems_test.go b/internal/provider/gems/gems_test.go new file mode 100644 index 0000000..9707bb3 --- /dev/null +++ b/internal/provider/gems/gems_test.go @@ -0,0 +1,61 @@ +package gems + +import ( + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +func TestClassify(t *testing.T) { + p := &Provider{} + tests := []struct { + path string + want provider.Mutability + }{ + {"gems/rack-3.1.8.gem", provider.Immutable}, + {"/gems/nokogiri-1.16.7-x86_64-linux.gem", provider.Immutable}, + {"quick/Marshal.4.8/rack-3.1.8.gemspec.rz", provider.Immutable}, + {"versions", provider.Mutable}, + {"names", provider.Mutable}, + {"info/rack", provider.Mutable}, + {"info/gems", provider.Mutable}, + {"specs.4.8.gz", provider.Mutable}, + {"latest_specs.4.8.gz", provider.Mutable}, + {"prerelease_specs.4.8.gz", provider.Mutable}, + {"api/v1/dependencies", provider.Mutable}, + {"quick/Marshal.4.8/", provider.Mutable}, + } + for _, tt := range tests { + if got := p.Classify(tt.path); got != tt.want { + t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want) + } + } +} + +func TestContentType(t *testing.T) { + p := &Provider{} + tests := map[string]string{ + "versions": "text/plain; charset=utf-8", + "info/rack": "text/plain; charset=utf-8", + "names": "text/plain; charset=utf-8", + "gems/rack-3.1.8.gem": "application/octet-stream", + "specs.4.8.gz": "application/octet-stream", + "api/v1/gems/rack.json": "application/json", + "api/v1/dependencies.json": "application/json", + "api/v1/dependencies": "application/octet-stream", + } + for path, want := range tests { + if got := p.ContentType(path); got != want { + t.Errorf("ContentType(%q) = %q, want %q", path, got, want) + } + } +} + +func TestUpstreamURL(t *testing.T) { + p := &Provider{} + r := models.Remote{BaseURL: "https://rubygems.org/"} + if got := p.UpstreamURL(r, "/info/rack"); got != "https://rubygems.org/info/rack" { + t.Errorf("UpstreamURL = %q", got) + } +} diff --git a/internal/proxy/engine.go b/internal/proxy/engine.go index cc267cc..1a6ca2f 100644 --- a/internal/proxy/engine.go +++ b/internal/proxy/engine.go @@ -160,7 +160,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p notModified, err := e.checkUpstream(ctx, remote, path, etag, prov) if err == nil && notModified { _ = e.cache.SetTTL(ctx, remote.Name, path, ttl) - _ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl) + _ = e.cache.SetETag(ctx, remote.Name, path, etag, etagTTL(ttl)) result, err := e.serveFromStore(ctx, remote, path) if err == nil { result.Source = "cache" @@ -491,8 +491,9 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote } _ = e.cache.SetTTL(ctx, remote.Name, path, ttl) - if etag := resp.Header.Get("ETag"); etag != "" { - _ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl) + // A zero TTL never expires, so it never needs revalidating. + if remote.CheckMutable && ttl > 0 { + _ = e.cache.SetETag(ctx, remote.Name, path, resp.Header.Get("ETag"), etagTTL(ttl)) } return &FetchResult{ @@ -520,9 +521,6 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote } _ = e.cache.SetTTL(ctx, remote.Name, path, ttl) - if etag := resp.Header.Get("ETag"); etag != "" { - _ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl) - } reader, info, err := e.store.Download(ctx, casResult.S3Key) if err != nil { @@ -642,6 +640,12 @@ func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, pa return resp.StatusCode == http.StatusNotModified, nil } +// etagTTL keeps a mutable path's ETag well past its freshness key so an expired +// path revalidates with a conditional HEAD, while idle paths still age out. +func etagTTL(ttl time.Duration) time.Duration { + return max(10*ttl, 7*24*time.Hour) +} + func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration { switch class { case ClassImmutable: diff --git a/internal/proxy/engine_test.go b/internal/proxy/engine_test.go index 2374aec..447de34 100644 --- a/internal/proxy/engine_test.go +++ b/internal/proxy/engine_test.go @@ -17,12 +17,14 @@ import ( "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/testsupport" "git.unkin.net/unkin/artifactapi/pkg/models" + goredis "github.com/redis/go-redis/v9" ) var ( testEngine *Engine testCache *cache.Redis testDB *database.DB + testRedis string upstream *httptest.Server ) @@ -64,6 +66,7 @@ func TestMain(m *testing.M) { } testCache = redis + testRedis = redisURL testDB = db testEngine = NewEngine(db, redis, s3) upstream = httptest.NewServer(http.HandlerFunc(mockUpstream)) @@ -274,30 +277,169 @@ func TestCircuitOpenServesStale(t *testing.T) { res.Reader.Close() } +// expire drops path's freshness key so the next fetch revalidates. +func expire(t *testing.T, remote, path string) { + t.Helper() + if err := testCache.SetTTL(context.Background(), remote, path, time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(5 * time.Millisecond) +} + +func mutableFetch(t *testing.T, r models.Remote, path string) (string, string) { + t.Helper() + res, err := testEngine.Fetch(context.Background(), r, path, prov(t, models.PackageRPM)) + if err != nil { + t.Fatalf("fetch: %v", err) + } + return readAll(t, res), res.Source +} + +const repomd = "8/Everything/x86_64/repodata/repomd.xml" + func TestMutableRevalidation(t *testing.T) { + requireStack(t) + srv, rev, conditional := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-reval", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60}) + + if body, _ := mutableFetch(t, r, repomd); body != "rev1:/"+repomd { + t.Fatalf("initial fetch = %q", body) + } + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" { + t.Errorf("after expiry = %q from %s, want cached rev1", body, src) + } + if n := conditional.Load(); n != 1 { + t.Errorf("conditional requests = %d, want 1 revalidation", n) + } + if fresh, _ := testCache.CheckTTL(context.Background(), r.Name, repomd); !fresh { + t.Error("304 should re-arm the freshness key") + } + + rev.Store("rev2") + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev2:/"+repomd || src != "remote" { + t.Errorf("after upstream change = %q from %s, want remote rev2", body, src) + } +} + +func etagKeyTTL(t *testing.T, remote, path string) time.Duration { + t.Helper() + opts, err := goredis.ParseURL(testRedis) + if err != nil { + t.Fatal(err) + } + c := goredis.NewClient(opts) + defer func() { _ = c.Close() }() + d, err := c.PTTL(context.Background(), "etag:"+remote+":"+path).Result() + if err != nil { + t.Fatal(err) + } + return d +} + +// The ETag is only kept when it can be used, and never without an expiry. +func TestMutableETagStorage(t *testing.T) { + requireStack(t) + srv, _, _ := changingUpstream(t) + for _, tc := range []struct { + name string + check bool + ttl int + want time.Duration + }{ + {"eng-etag-off", false, 60, -2}, + {"eng-etag-never", true, 0, -2}, + {"eng-etag-short", true, 60, 7 * 24 * time.Hour}, + {"eng-etag-long", true, 86400, 10 * 24 * time.Hour}, + } { + r := seed(t, models.Remote{Name: tc.name, PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: tc.check, MutableTTL: tc.ttl}) + mutableFetch(t, r, repomd) + got := etagKeyTTL(t, tc.name, repomd) + if tc.want < 0 { + if got != tc.want { + t.Errorf("%s: etag key ttl = %v, want no key", tc.name, got) + } + continue + } + if got <= tc.want-time.Minute || got > tc.want { + t.Errorf("%s: etag key ttl = %v, want ~%v", tc.name, got, tc.want) + } + } +} + +// Immutable paths are never revalidated, so their upstream ETag is not kept. +func TestImmutableFetchStoresNoETag(t *testing.T) { + requireStack(t) + srv, _, _ := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-etag-imm", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60}) + const rpm = "Packages/e2e-1.0-1.noarch.rpm" + if body, src := mutableFetch(t, r, rpm); body != "rev1:/"+rpm || src != "remote" { + t.Fatalf("fetch = %q from %s", body, src) + } + if got := etagKeyTTL(t, r.Name, rpm); got != -2 { + t.Errorf("etag key ttl = %v, want no key", got) + } +} + +// A 304 for a path whose S3 body is gone falls back to a full refetch. +func TestNotModifiedWithEvictedBodyRefetches(t *testing.T) { + requireStack(t) + srv, _, conditional := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-304-gone", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60}) + + mutableFetch(t, r, repomd) + if err := testEngine.store.Delete(context.Background(), storage.IndexKey(r.Name, repomd)); err != nil { + t.Fatal(err) + } + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "remote" { + t.Errorf("got %q from %s, want refetched rev1 from remote", body, src) + } + if n := conditional.Load(); n != 1 { + t.Errorf("conditional requests = %d, want the 304 HEAD first", n) + } +} + +// A failing revalidation HEAD still serves the stored copy with stale_on_error. +func TestStaleOnErrorWhenRevalidationFails(t *testing.T) { + requireStack(t) + srv, _, _ := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-stale-head", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60, StaleOnError: true}) + + mutableFetch(t, r, repomd) + srv.Close() + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" { + t.Errorf("got %q from %s, want stale rev1 from cache", body, src) + } +} + +// After a remote is deleted and flushed, a re-created remote of the same name +// fetches from its own upstream instead of revalidating the old ETag. +func TestRecreatedRemoteHasNoStaleState(t *testing.T) { requireStack(t) ctx := context.Background() - r := seed(t, models.Remote{Name: "eng-npm", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true}) - p := prov(t, models.PackageNPM) + oldSrv, _, _ := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-recreate", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: oldSrv.URL, CheckMutable: true, MutableTTL: 60}) + mutableFetch(t, r, repomd) - res, err := testEngine.Fetch(ctx, r, "pkg", p) - if err != nil { - t.Fatalf("initial mutable fetch: %v", err) + if err := testDB.DeleteRemote(ctx, r.Name); err != nil { + t.Fatal(err) } - res.Reader.Close() - - // Expire only the freshness marker; the ETag persists, forcing a - // conditional revalidation that the upstream answers with 304. - testCache.SetTTL(ctx, "eng-npm", "pkg", time.Millisecond) - time.Sleep(10 * time.Millisecond) - res, err = testEngine.Fetch(ctx, r, "pkg", p) - if err != nil { - t.Fatalf("revalidation fetch: %v", err) + if err := testCache.FlushRemote(ctx, r.Name); err != nil { + t.Fatal(err) } - if res.Source != "cache" { - t.Errorf("revalidated response should come from cache, got %s", res.Source) + newSrv, rev, conditional := changingUpstream(t) + rev.Store("rev9") + r.BaseURL = newSrv.URL + seed(t, r) + if body, src := mutableFetch(t, r, repomd); body != "rev9:/"+repomd || src != "remote" { + t.Errorf("got %q from %s, want rev9 from the new upstream", body, src) + } + if n := conditional.Load(); n != 0 { + t.Errorf("conditional requests = %d, want none after flush", n) } - res.Reader.Close() } func TestBearerTokenFlow(t *testing.T) { diff --git a/internal/proxy/evict_test.go b/internal/proxy/evict_test.go index 15b8def..622cc35 100644 --- a/internal/proxy/evict_test.go +++ b/internal/proxy/evict_test.go @@ -59,7 +59,7 @@ type cached struct{ row, index, ttl, etag bool } // Mutable indexes live in the S3 index; immutable blobs get an artifact row. var ( indexCached = cached{index: true, ttl: true, etag: true} - blobCached = cached{row: true, ttl: true, etag: true} + blobCached = cached{row: true, ttl: true} ) func layers(t *testing.T, remote, path string) cached { diff --git a/pkg/models/package_type_gems.go b/pkg/models/package_type_gems.go new file mode 100644 index 0000000..58e0bd1 --- /dev/null +++ b/pkg/models/package_type_gems.go @@ -0,0 +1,5 @@ +package models + +const PackageGems PackageType = "gems" + +func init() { registerPackageType(PackageGems) } diff --git a/ui/src/packagetypes/defs/gems.ts b/ui/src/packagetypes/defs/gems.ts new file mode 100644 index 0000000..b1a8e9a --- /dev/null +++ b/ui/src/packagetypes/defs/gems.ts @@ -0,0 +1,17 @@ +import type { PackageTypeUI } from '../types'; + +const def: PackageTypeUI = { + color: 'red', + snippets: ({ proxy }) => { + return [ + { + title: 'Use this remote as the RubyGems source', + language: 'bash', + code: `gem sources --clear-all --add ${proxy}/ +bundle config mirror.https://rubygems.org ${proxy}`, + }, + ]; + }, +}; + +export default def; diff --git a/ui/src/packagetypes/defs/npm.ts b/ui/src/packagetypes/defs/npm.ts index 441c99f..a10af90 100644 --- a/ui/src/packagetypes/defs/npm.ts +++ b/ui/src/packagetypes/defs/npm.ts @@ -2,7 +2,28 @@ import type { PackageTypeUI } from '../types'; const def: PackageTypeUI = { color: 'red', - snippets: ({ proxy }) => { + snippets: ({ name, url, host, proxy, isLocal }) => { + if (isLocal) { + const reg = `${url}/api/v1/local/${name}/`; + return [ + { + title: 'Configure .npmrc (registry + publish token)', + language: 'bash', + code: `cat >> .npmrc <<'EOF' +registry=${reg} +//${host}/api/v1/local/${name}/:_authToken=unused +EOF`, + note: 'npm refuses to publish without an _authToken for the registry; any value works. For a scope only, use @:registry=… instead of registry=.', + }, + { + title: 'Publish and install', + language: 'bash', + code: `npm publish +npm install `, + note: 'Versions are immutable: republishing an existing version is rejected. The dist-tags sent with the publish (latest, or --tag) move to the new version.', + }, + ]; + } return [ { title: 'Point npm at this registry',