diff --git a/README.md b/README.md index 660a658..28cfafe 100644 --- a/README.md +++ b/README.md @@ -279,6 +279,20 @@ content-addressable store (deduplicated by digest, reaped by GC once unreferenced); tags are mutable references and re-pushing a tag moves it. Blob uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. +### Local cargo registry + +A local `cargo` repo is a private crate registry: it serves the sparse index and +the Web API publish, yank and unyank endpoints, so `cargo publish` works against +it directly. + +```toml +[registries.] +index = "sparse+https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local//" +``` + +cargo requires a token for publish/yank (`CARGO_REGISTRIES__TOKEN`); like +other local repos, artifactapi does not check it. + ## Access Control | Field | Default | Behaviour | diff --git a/e2e-docker/README.md b/e2e-docker/README.md index 3125d23..8a6338c 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -27,7 +27,8 @@ already-running stack. mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` - index), rpm (real package + **automatic repodata** generation). + index), rpm (real package + **automatic repodata** generation), cargo (publish, + sparse index, download, yank). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members. - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors diff --git a/e2e-docker/cargo_local_test.go b/e2e-docker/cargo_local_test.go new file mode 100644 index 0000000..476380d --- /dev/null +++ b/e2e-docker/cargo_local_test.go @@ -0,0 +1,56 @@ +//go:build dockere2e + +package e2edocker + +import ( + "bytes" + "encoding/binary" + "net/http" + "strings" + "testing" +) + +// TestLocalCargoRegistry publishes through the cargo Web API, then reads the +// sparse index, downloads the crate and yanks it. +func TestLocalCargoRegistry(t *testing.T) { + createRepo(t, `{"name":"local-cargo","package_type":"cargo","repo_type":"local"}`) + defer deleteRepo(t, "local-cargo") + base := api("/api/v1/local/local-cargo") + + meta := `{"name":"e2e-crate","vers":"0.1.0","deps":[],"features":{}}` + crate := []byte("e2e crate bytes") + var body []byte + body = binary.LittleEndian.AppendUint32(body, uint32(len(meta))) + body = append(body, meta...) + body = binary.LittleEndian.AppendUint32(body, uint32(len(crate))) + body = append(body, crate...) + + if resp, b := doRequest(t, http.MethodPut, base+"/api/v1/crates/new", body, "application/octet-stream"); resp.StatusCode != http.StatusOK { + t.Fatalf("publish: status %d: %s", resp.StatusCode, b) + } + if resp, _ := doRequest(t, http.MethodPut, base+"/api/v1/crates/new", body, "application/octet-stream"); resp.StatusCode != http.StatusConflict { + t.Fatalf("republish: status %d, want 409", resp.StatusCode) + } + + resp, b := doRequest(t, http.MethodGet, base+"/config.json", nil, "") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"api":"`) { + t.Fatalf("config.json: status %d: %s", resp.StatusCode, b) + } + + resp, b = doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, "") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"cksum":"`) || !strings.Contains(string(b), `"yanked":false`) { + t.Fatalf("index: status %d: %s", resp.StatusCode, b) + } + + resp, b = doRequest(t, http.MethodGet, base+"/crates/e2e-crate/e2e-crate-0.1.0.crate", nil, "") + if resp.StatusCode != http.StatusOK || !bytes.Equal(b, crate) { + t.Fatalf("download: status %d", resp.StatusCode) + } + + if resp, b := doRequest(t, http.MethodDelete, base+"/api/v1/crates/e2e-crate/0.1.0/yank", nil, ""); resp.StatusCode != http.StatusOK { + t.Fatalf("yank: status %d: %s", resp.StatusCode, b) + } + if _, b := doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, ""); !strings.Contains(string(b), `"yanked":true`) { + t.Fatalf("index after yank: %s", b) + } +} diff --git a/internal/api/v1/cargo.go b/internal/api/v1/cargo.go new file mode 100644 index 0000000..73100dc --- /dev/null +++ b/internal/api/v1/cargo.go @@ -0,0 +1,148 @@ +package v1 + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/provider/cargo" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// This file implements the cargo registry Web API (publish, yank, unyank) for +// local cargo repositories. config.json points api at /api/v1/local/, so +// cargo appends its own /api/v1/crates/... below that. + +const maxCratePublish = 64 << 20 + +func (h *ProxyHandler) localCargoRemote(r *http.Request) (*models.Remote, bool) { + remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName")) + if err != nil { + return nil, false + } + return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageCargo +} + +func cargoError(w http.ResponseWriter, status int, format string, args ...any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{ + "errors": []map[string]string{{"detail": fmt.Sprintf(format, args...)}}, + }) +} + +func cargoOK(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(v) +} + +func (h *ProxyHandler) cargoPublish(w http.ResponseWriter, r *http.Request) { + remote, ok := h.localCargoRemote(r) + if !ok { + cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName")) + return + } + ctx := r.Context() + + body, err := io.ReadAll(io.LimitReader(r.Body, maxCratePublish+1)) + if err != nil { + cargoError(w, http.StatusBadRequest, "read body: %v", err) + return + } + if len(body) > maxCratePublish { + cargoError(w, http.StatusRequestEntityTooLarge, "publish body exceeds %d bytes", maxCratePublish) + return + } + meta, crate, err := cargo.ParsePublish(body) + if err != nil { + cargoError(w, http.StatusBadRequest, "%v", err) + return + } + line, err := cargo.NewIndexLine(meta, crate) + if err != nil { + cargoError(w, http.StatusBadRequest, "%v", err) + return + } + + existing, err := h.db.ListCargoIndexEntries(ctx, remote.Name, meta.Name) + if err != nil { + cargoError(w, http.StatusInternalServerError, "%v", err) + return + } + for _, e := range existing { + if e.Name != meta.Name { + cargoError(w, http.StatusBadRequest, "crate %q already exists as %q", meta.Name, e.Name) + return + } + if e.Version == meta.Vers { + cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers) + return + } + } + + lineJSON, err := json.Marshal(line) + if err != nil { + cargoError(w, http.StatusInternalServerError, "%v", err) + return + } + + path := cargo.CratePath(meta.Name, meta.Vers) + result, err := h.cas.Store(ctx, bytes.NewReader(crate), "application/gzip") + if err != nil { + cargoError(w, http.StatusInternalServerError, "store failed: %v", err) + return + } + if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, "application/gzip"); err != nil { + cargoError(w, http.StatusInternalServerError, "record blob: %v", err) + return + } + if err := h.db.CreateLocalFile(ctx, remote.Name, path, result.ContentHash); err != nil { + if errors.Is(err, database.ErrAlreadyExists) { + cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers) + return + } + cargoError(w, http.StatusInternalServerError, "record file: %v", err) + return + } + entry := &provider.CargoIndexEntry{RepoName: remote.Name, FilePath: path, Name: meta.Name, Version: meta.Vers, Line: string(lineJSON)} + if err := h.db.InsertCargoIndexEntry(ctx, entry); err != nil { + // Drop the file so a retry is not rejected as a duplicate. + _ = h.db.DeleteLocalFile(ctx, remote.Name, path) + cargoError(w, http.StatusInternalServerError, "record index: %v", err) + return + } + + slog.Info("cargo: published", "repo", remote.Name, "crate", meta.Name, "version", meta.Vers) + cargoOK(w, map[string]any{"warnings": map[string][]string{"invalid_categories": {}, "invalid_badges": {}, "other": {}}}) +} + +func (h *ProxyHandler) cargoYank(yanked bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + remote, ok := h.localCargoRemote(r) + if !ok { + cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName")) + return + } + name, version := chi.URLParam(r, "crate"), chi.URLParam(r, "version") + found, err := h.db.SetCargoYanked(r.Context(), remote.Name, name, version, yanked) + if err != nil { + cargoError(w, http.StatusInternalServerError, "%v", err) + return + } + if !found { + cargoError(w, http.StatusNotFound, "crate version `%s@%s` not found", name, version) + return + } + slog.Info("cargo: yank updated", "repo", remote.Name, "crate", name, "version", version, "yanked", yanked) + cargoOK(w, map[string]bool{"ok": true}) + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index e77f43e..f4a801b 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -41,6 +41,9 @@ func (h *ProxyHandler) Routes() chi.Router { r := chi.NewRouter() r.Get("/remote/{remoteName}/*", h.handleProxy) r.Get("/local/{localName}/*", h.handleLocal) + r.Put("/local/{localName}/api/v1/crates/new", h.cargoPublish) + r.Delete("/local/{localName}/api/v1/crates/{crate}/{version}/yank", h.cargoYank(true)) + r.Put("/local/{localName}/api/v1/crates/{crate}/{version}/unyank", h.cargoYank(false)) r.Get("/virtual/{virtualName}/*", h.handleVirtual) return r } diff --git a/internal/database/cargo_index.go b/internal/database/cargo_index.go new file mode 100644 index 0000000..712ae84 --- /dev/null +++ b/internal/database/cargo_index.go @@ -0,0 +1,56 @@ +package database + +import ( + "context" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func (db *DB) InsertCargoIndexEntry(ctx context.Context, e *provider.CargoIndexEntry) error { + _, err := db.Pool.Exec(ctx, ` + INSERT INTO cargo_index (repo_name, file_path, name, version, line, yanked) + VALUES ($1, $2, $3, $4, $5, $6) + `, e.RepoName, e.FilePath, e.Name, e.Version, e.Line, e.Yanked) + return err +} + +func (db *DB) ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]provider.CargoIndexEntry, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, name, version, line, yanked + FROM cargo_index + WHERE repo_name = $1 AND lower(name) = $2 + ORDER BY id + `, repoName, strings.ToLower(name)) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.CargoIndexEntry + for rows.Next() { + var e provider.CargoIndexEntry + if err := rows.Scan(&e.RepoName, &e.FilePath, &e.Name, &e.Version, &e.Line, &e.Yanked); err != nil { + return nil, err + } + result = append(result, e) + } + return result, rows.Err() +} + +// SetCargoYanked reports false when no such crate version exists. +func (db *DB) SetCargoYanked(ctx context.Context, repoName, name, version string, yanked bool) (bool, error) { + tag, err := db.Pool.Exec(ctx, ` + UPDATE cargo_index SET yanked = $4 + WHERE repo_name = $1 AND lower(name) = $2 AND version = $3 + `, repoName, strings.ToLower(name), version, yanked) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (db *DB) DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM cargo_index WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 2f11049..a908d34 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -425,3 +425,38 @@ func TestRPMMetadata(t *testing.T) { t.Fatalf("list rpm rows: %v %v", len(rows), err) } } + +func TestCargoIndex(t *testing.T) { + requireDB(t) + for _, v := range []string{"0.1.0", "0.2.0"} { + if err := testDB.InsertCargoIndexEntry(ctx(), &provider.CargoIndexEntry{ + RepoName: "cargo-db", FilePath: "crates/Unkin-Hello/Unkin-Hello-" + v + ".crate", + Name: "Unkin-Hello", Version: v, Line: `{"vers":"` + v + `"}`, + }); err != nil { + t.Fatal(err) + } + } + + found, err := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "0.1.0", true) + if err != nil || !found { + t.Fatalf("yank: found=%v err=%v", found, err) + } + if found, _ := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "9.9.9", true); found { + t.Error("yank of a missing version reported found") + } + + entries, err := testDB.ListCargoIndexEntries(ctx(), "cargo-db", "unkin-hello") + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 || entries[0].Version != "0.1.0" || !entries[0].Yanked || entries[1].Yanked { + t.Fatalf("entries = %+v", entries) + } + + if err := testDB.DeleteCargoIndexEntry(ctx(), "cargo-db", entries[0].FilePath); err != nil { + t.Fatal(err) + } + if entries, _ = testDB.ListCargoIndexEntries(ctx(), "cargo-db", "UNKIN-HELLO"); len(entries) != 1 { + t.Fatalf("after delete: %+v", entries) + } +} diff --git a/internal/provider/cargo/local.go b/internal/provider/cargo/local.go new file mode 100644 index 0000000..9e0f985 --- /dev/null +++ b/internal/provider/cargo/local.go @@ -0,0 +1,303 @@ +package cargo + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +// publishDep is a dependency as cargo sends it in the publish metadata. +type publishDep struct { + Name string `json:"name"` + VersionReq string `json:"version_req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + ExplicitNameInToml *string `json:"explicit_name_in_toml"` +} + +// PublishMeta is the JSON half of a `PUT /api/v1/crates/new` body; fields the +// index does not carry are ignored. +type PublishMeta struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []publishDep `json:"deps"` + Features map[string][]string `json:"features"` + Links *string `json:"links"` + RustVersion *string `json:"rust_version"` +} + +type indexDep struct { + Name string `json:"name"` + Req string `json:"req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry,omitempty"` + Package *string `json:"package,omitempty"` +} + +// IndexLine is one line of a sparse index file. +type IndexLine struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []indexDep `json:"deps"` + Cksum string `json:"cksum"` + Features map[string][]string `json:"features"` + Features2 map[string][]string `json:"features2,omitempty"` + Yanked bool `json:"yanked"` + Links *string `json:"links,omitempty"` + V int `json:"v,omitempty"` + RustVersion *string `json:"rust_version,omitempty"` +} + +// ParsePublish splits a publish body: u32le JSON length, JSON, u32le crate length, crate. +func ParsePublish(body []byte) (PublishMeta, []byte, error) { + var meta PublishMeta + jsonBytes, rest, err := lengthPrefixed(body) + if err != nil { + return meta, nil, fmt.Errorf("metadata: %w", err) + } + crate, rest, err := lengthPrefixed(rest) + if err != nil { + return meta, nil, fmt.Errorf("crate: %w", err) + } + if len(rest) != 0 { + return meta, nil, errors.New("trailing bytes after crate") + } + if err := json.Unmarshal(jsonBytes, &meta); err != nil { + return meta, nil, fmt.Errorf("metadata: %w", err) + } + return meta, crate, nil +} + +func lengthPrefixed(b []byte) ([]byte, []byte, error) { + if len(b) < 4 { + return nil, nil, errors.New("truncated length") + } + n := binary.LittleEndian.Uint32(b) + b = b[4:] + if uint64(n) > uint64(len(b)) { + return nil, nil, errors.New("truncated body") + } + return b[:n], b[n:], nil +} + +// NewIndexLine validates the metadata and translates it into index form. +func NewIndexLine(meta PublishMeta, crate []byte) (IndexLine, error) { + if !validName(meta.Name) { + return IndexLine{}, fmt.Errorf("invalid crate name %q", meta.Name) + } + if !validVersion(meta.Vers) { + return IndexLine{}, fmt.Errorf("invalid version %q", meta.Vers) + } + sum := sha256.Sum256(crate) + line := IndexLine{ + Name: meta.Name, + Vers: meta.Vers, + Deps: make([]indexDep, 0, len(meta.Deps)), + Cksum: hex.EncodeToString(sum[:]), + Features: map[string][]string{}, + Links: meta.Links, + RustVersion: meta.RustVersion, + } + for _, d := range meta.Deps { + dep := indexDep{ + Name: d.Name, + Req: d.VersionReq, + Features: d.Features, + Optional: d.Optional, + DefaultFeatures: d.DefaultFeatures, + Target: d.Target, + Kind: d.Kind, + Registry: d.Registry, + } + if dep.Features == nil { + dep.Features = []string{} + } + if dep.Kind == "" { + dep.Kind = "normal" + } + if d.ExplicitNameInToml != nil && *d.ExplicitNameInToml != "" { + pkg := d.Name + dep.Name, dep.Package = *d.ExplicitNameInToml, &pkg + } + line.Deps = append(line.Deps, dep) + } + // Older cargo cannot parse dep:/?/ feature syntax, so those features live in + // features2 behind v:2, as on crates.io. + for name, vals := range meta.Features { + if vals == nil { + vals = []string{} + } + if usesNewFeatureSyntax(vals) { + if line.Features2 == nil { + line.Features2 = map[string][]string{} + } + line.Features2[name] = vals + continue + } + line.Features[name] = vals + } + if line.Features2 != nil { + line.V = 2 + } + return line, nil +} + +func usesNewFeatureSyntax(vals []string) bool { + for _, v := range vals { + if strings.HasPrefix(v, "dep:") || strings.Contains(v, "?/") { + return true + } + } + return false +} + +func validName(name string) bool { + if name == "" || len(name) > 64 { + return false + } + for i, c := range name { + alpha := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + if i == 0 && !alpha { + return false + } + if !alpha && (c < '0' || c > '9') && c != '-' && c != '_' { + return false + } + } + return true +} + +func validVersion(v string) bool { + if v == "" || len(v) > 128 || !strings.ContainsAny(v[:1], "0123456789") { + return false + } + return strings.Trim(v, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-+") == "" +} + +// IndexPath is the sparse-index file path for a crate name. +func IndexPath(name string) string { + n := strings.ToLower(name) + switch len(n) { + case 0: + return "" + case 1: + return "1/" + n + case 2: + return "2/" + n + case 3: + return "3/" + n[:1] + "/" + n + } + return n[:2] + "/" + n[2:4] + "/" + n +} + +// CratePath is where a published .crate is stored, matching config.json's dl. +func CratePath(name, version string) string { + return crateDir + name + "/" + name + "-" + version + ".crate" +} + +// RenderIndex joins stored lines into an index file with each yanked flag applied. +func RenderIndex(entries []provider.CargoIndexEntry) ([]byte, error) { + var b strings.Builder + for _, e := range entries { + var line IndexLine + if err := json.Unmarshal([]byte(e.Line), &line); err != nil { + return nil, fmt.Errorf("%s %s: %w", e.Name, e.Version, err) + } + line.Yanked = e.Yanked + raw, err := json.Marshal(line) + if err != nil { + return nil, err + } + b.Write(raw) + b.WriteByte('\n') + } + return []byte(b.String()), nil +} + +// LocalConfig is config.json for a local registry rooted at base. +func LocalConfig(base string) map[string]string { + return map[string]string{ + "dl": base + "/" + crateDir + "{crate}/{crate}-{version}.crate", + "api": base, + } +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool { + path = strings.TrimLeft(path, "/") + if path == "config.json" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(LocalConfig(requestBase(r) + "/api/v1/local/" + repoName)) + return true + } + name := path[strings.LastIndex(path, "/")+1:] + if name == "" || IndexPath(name) != path { + return false + } + + reader, ok := files.(provider.CargoIndexReader) + if !ok { + http.Error(w, "cargo index not available", http.StatusInternalServerError) + return true + } + entries, err := reader.ListCargoIndexEntries(r.Context(), repoName, name) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + if len(entries) == 0 { + http.Error(w, "not found", http.StatusNotFound) + return true + } + body, err := RenderIndex(entries) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + return true +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) { + return nil, fmt.Errorf("cargo local index generation for virtual repos not supported") +} + +func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error { + deleter, ok := db.(provider.CargoIndexDeleter) + if !ok { + return nil + } + if err := deleter.DeleteCargoIndexEntry(ctx, repoName, storagePath); err != nil { + slog.Error("cargo index: delete failed", "repo", repoName, "path", storagePath, "error", err) + return err + } + return nil +} + +func requestBase(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } else if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" { + scheme = fwd + } + return scheme + "://" + r.Host +} diff --git a/internal/provider/cargo/local_test.go b/internal/provider/cargo/local_test.go new file mode 100644 index 0000000..e6bae0d --- /dev/null +++ b/internal/provider/cargo/local_test.go @@ -0,0 +1,218 @@ +package cargo + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func publishBody(meta string, crate []byte) []byte { + var b []byte + b = binary.LittleEndian.AppendUint32(b, uint32(len(meta))) + b = append(b, meta...) + b = binary.LittleEndian.AppendUint32(b, uint32(len(crate))) + return append(b, crate...) +} + +const sampleMeta = `{ + "name": "unkin-hello", "vers": "0.2.0", + "deps": [ + {"name": "serde", "version_req": "^1", "features": ["derive"], "optional": false, + "default_features": true, "target": null, "kind": "normal", + "registry": "https://github.com/rust-lang/crates.io-index"}, + {"name": "rand_core", "version_req": "^0.6", "features": null, "optional": true, + "default_features": false, "target": "cfg(unix)", "kind": "dev", + "explicit_name_in_toml": "rc"} + ], + "features": {"default": ["std"], "std": [], "extra": ["dep:rc"], "weak": ["serde?/std"]}, + "links": null, "rust_version": "1.70", "description": "ignored" +}` + +func TestParsePublishAndIndexLine(t *testing.T) { + crate := []byte("fake crate bytes") + meta, gotCrate, err := ParsePublish(publishBody(sampleMeta, crate)) + if err != nil { + t.Fatal(err) + } + if string(gotCrate) != string(crate) { + t.Fatalf("crate = %q", gotCrate) + } + line, err := NewIndexLine(meta, crate) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(crate) + if line.Cksum != hex.EncodeToString(sum[:]) { + t.Errorf("cksum = %s", line.Cksum) + } + + raw, _ := json.Marshal(line) + var got map[string]any + _ = json.Unmarshal(raw, &got) + if _, ok := got["version_req"]; ok { + t.Error("publish-only field leaked into index line") + } + if got["v"] != float64(2) { + t.Errorf("v = %v, want 2 with features2 present", got["v"]) + } + if got["rust_version"] != "1.70" { + t.Errorf("rust_version = %v", got["rust_version"]) + } + if _, ok := line.Features["extra"]; ok { + t.Error("dep: feature must move to features2") + } + if len(line.Features2) != 2 || len(line.Features) != 2 { + t.Errorf("features = %v features2 = %v", line.Features, line.Features2) + } + + deps := got["deps"].([]any) + serde := deps[0].(map[string]any) + if serde["req"] != "^1" || serde["registry"] != "https://github.com/rust-lang/crates.io-index" || serde["target"] != nil { + t.Errorf("serde dep = %v", serde) + } + if _, ok := serde["package"]; ok { + t.Error("package set without a rename") + } + renamed := deps[1].(map[string]any) + if renamed["name"] != "rc" || renamed["package"] != "rand_core" || renamed["kind"] != "dev" { + t.Errorf("renamed dep = %v", renamed) + } + if f, ok := renamed["features"].([]any); !ok || len(f) != 0 { + t.Errorf("null features must render as [], got %v", renamed["features"]) + } +} + +func TestIndexLineNoNewFeaturesOmitsV(t *testing.T) { + line, err := NewIndexLine(PublishMeta{Name: "a", Vers: "1.0.0", Features: map[string][]string{"x": nil}}, nil) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(line) + s := string(raw) + for _, bad := range []string{`"v":`, `"features2"`, `"links"`} { + if strings.Contains(s, bad) { + t.Errorf("%s present in %s", bad, s) + } + } + if !strings.Contains(s, `"deps":[]`) || !strings.Contains(s, `"features":{"x":[]}`) { + t.Errorf("deps/features must be non-null: %s", s) + } +} + +func TestParsePublishRejectsMalformed(t *testing.T) { + good := publishBody(`{"name":"a","vers":"1.0.0"}`, []byte("c")) + for name, body := range map[string][]byte{ + "empty": nil, + "short": good[:len(good)-1], + "trailing": append(append([]byte{}, good...), 'x'), + "bad json": publishBody(`{`, []byte("c")), + "no length": good[:6], + } { + if _, _, err := ParsePublish(body); err == nil { + t.Errorf("%s: expected error", name) + } + } +} + +func TestNewIndexLineValidation(t *testing.T) { + for _, m := range []PublishMeta{ + {Name: "", Vers: "1.0.0"}, + {Name: "1abc", Vers: "1.0.0"}, + {Name: "../x", Vers: "1.0.0"}, + {Name: "ok", Vers: "1.0.0/../../x"}, + {Name: "ok", Vers: ""}, + {Name: strings.Repeat("a", 65), Vers: "1.0.0"}, + } { + if _, err := NewIndexLine(m, nil); err == nil { + t.Errorf("%q@%q accepted", m.Name, m.Vers) + } + } + if _, err := NewIndexLine(PublishMeta{Name: "Foo_bar-2", Vers: "1.0.0-rc.1+build.5"}, nil); err != nil { + t.Error(err) + } +} + +func TestIndexPath(t *testing.T) { + for name, want := range map[string]string{ + "a": "1/a", "ab": "2/ab", "abc": "3/a/abc", "Serde": "se/rd/serde", "unkin-hello": "un/ki/unkin-hello", + } { + if got := IndexPath(name); got != want { + t.Errorf("IndexPath(%q) = %q, want %q", name, got, want) + } + } +} + +type fakeIndex struct { + entries []provider.CargoIndexEntry + asked string +} + +func (f *fakeIndex) ListCargoIndexEntries(_ context.Context, _, name string) ([]provider.CargoIndexEntry, error) { + f.asked = name + return f.entries, nil +} +func (f *fakeIndex) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (f *fakeIndex) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestServeLocalIndex(t *testing.T) { + line, _ := NewIndexLine(PublishMeta{Name: "unkin-hello", Vers: "0.1.0"}, []byte("x")) + raw, _ := json.Marshal(line) + files := &fakeIndex{entries: []provider.CargoIndexEntry{ + {Name: "unkin-hello", Version: "0.1.0", Line: string(raw), Yanked: true}, + {Name: "unkin-hello", Version: "0.2.0", Line: strings.Replace(string(raw), "0.1.0", "0.2.0", 1)}, + }} + p := &Provider{} + + rec := httptest.NewRecorder() + if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "http://aa.example/x", nil), files, "crates-local", "un/ki/unkin-hello") { + t.Fatal("index path not served") + } + lines := strings.Split(strings.TrimSpace(rec.Body.String()), "\n") + if len(lines) != 2 || !strings.Contains(lines[0], `"yanked":true`) || !strings.Contains(lines[1], `"yanked":false`) { + t.Errorf("index body = %q", rec.Body.String()) + } + if files.asked != "unkin-hello" { + t.Errorf("looked up %q", files.asked) + } + + rec = httptest.NewRecorder() + req := httptest.NewRequest("GET", "http://aa.example/config.json", nil) + req.Header.Set("X-Forwarded-Proto", "https") + p.ServeLocalIndex(rec, req, files, "crates-local", "config.json") + var cfg map[string]string + _ = json.Unmarshal(rec.Body.Bytes(), &cfg) + if cfg["api"] != "https://aa.example/api/v1/local/crates-local" || + cfg["dl"] != "https://aa.example/api/v1/local/crates-local/crates/{crate}/{crate}-{version}.crate" { + t.Errorf("config = %v", cfg) + } + + for _, path := range []string{"crates/unkin-hello/unkin-hello-0.1.0.crate", "xx/ki/unkin-hello", ""} { + if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), files, "r", path) { + t.Errorf("%q should fall through", path) + } + } + + files.entries = nil + rec = httptest.NewRecorder() + p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), files, "r", "3/a/abc") + if rec.Code != 404 { + t.Errorf("unknown crate code = %d", rec.Code) + } +} + +func TestCratePathMatchesDL(t *testing.T) { + dl := LocalConfig("http://h/api/v1/local/r")["dl"] + want := strings.NewReplacer("{crate}", "unkin-hello", "{version}", "0.1.0").Replace(dl) + if got := "http://h/api/v1/local/r/" + CratePath("unkin-hello", "0.1.0"); got != want { + t.Errorf("CratePath = %q, dl expands to %q", got, want) + } +} diff --git a/internal/provider/cargo_index.go b/internal/provider/cargo_index.go new file mode 100644 index 0000000..3c7f6aa --- /dev/null +++ b/internal/provider/cargo_index.go @@ -0,0 +1,24 @@ +package provider + +import "context" + +// CargoIndexEntry is one published crate version in a local cargo registry. +// Line is the sparse-index JSON line; Yanked is applied when it is served. +type CargoIndexEntry struct { + RepoName string + FilePath string + Name string + Version string + Line string + Yanked bool +} + +// CargoIndexReader lists every version of one crate (matched case-insensitively). +// *database.DB satisfies it and CargoIndexDeleter. +type CargoIndexReader interface { + ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]CargoIndexEntry, error) +} + +type CargoIndexDeleter interface { + DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error +} diff --git a/migrations/0003_cargo_index.sql b/migrations/0003_cargo_index.sql new file mode 100644 index 0000000..74b570d --- /dev/null +++ b/migrations/0003_cargo_index.sql @@ -0,0 +1,15 @@ +-- Local cargo registries: one sparse-index line per published crate version. +-- line holds the index JSON minus yanked, which flips on yank/unyank. +CREATE TABLE IF NOT EXISTS cargo_index ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + line TEXT NOT NULL, + yanked BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_cargo_index_repo_name ON cargo_index(repo_name, lower(name)); diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index 228a924..f7ca43a 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -229,6 +229,25 @@ go mod download`, ]; case 'cargo': + if (isLocal) { + return [ + { + title: 'Add the private registry (real sparse registry)', + language: 'toml', + code: `# $CARGO_HOME/config.toml +[registries.${name}] +index = "sparse+${url}/api/v1/local/${name}/"`, + note: 'cargo requires a token for publish/yank: set CARGO_REGISTRIES_' + name.toUpperCase().replace(/-/g, '_') + '_TOKEN to any value.', + }, + { + title: 'Publish, depend on and yank crates', + language: 'bash', + code: `cargo publish --registry ${name} +cargo add --registry ${name} +cargo yank --version --registry ${name}`, + }, + ]; + } return [ { title: 'Point crates.io at this sparse registry',