diff --git a/README.md b/README.md index 5b4a4e9..7a38cfc 100644 --- a/README.md +++ b/README.md @@ -280,6 +280,20 @@ content-addressable store (deduplicated by digest, reaped by GC once unreferenced); tags are mutable references and re-pushing a tag moves it. Blob uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows. +### Local cargo registry + +A local `cargo` repo is a private crate registry: it serves the sparse index and +the Web API publish, yank and unyank endpoints, so `cargo publish` works against +it directly. + +```toml +[registries.] +index = "sparse+https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local//" +``` + +cargo requires a token for publish/yank (`CARGO_REGISTRIES__TOKEN`); like +other local repos, artifactapi does not check it. + ## Access Control | Field | Default | Behaviour | diff --git a/e2e-docker/README.md b/e2e-docker/README.md index ec2ee0c..b658e0e 100644 --- a/e2e-docker/README.md +++ b/e2e-docker/README.md @@ -27,7 +27,8 @@ already-running stack. mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes verified against the origin fixture. - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/` - index), rpm (real package + **automatic repodata** generation). + index), rpm (real package + **automatic repodata** generation), cargo (publish, + sparse index, download, yank). - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge across two members. - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors diff --git a/e2e-docker/cargo_local_test.go b/e2e-docker/cargo_local_test.go new file mode 100644 index 0000000..c86d32d --- /dev/null +++ b/e2e-docker/cargo_local_test.go @@ -0,0 +1,66 @@ +//go:build dockere2e + +package e2edocker + +import ( + "bytes" + "encoding/binary" + "net/http" + "strings" + "testing" +) + +// TestLocalCargoRegistry publishes through the cargo Web API, then reads the +// sparse index, downloads the crate and yanks it. +func TestLocalCargoRegistry(t *testing.T) { + createRepo(t, `{"name":"local-cargo","package_type":"cargo","repo_type":"local"}`) + defer deleteRepo(t, "local-cargo") + base := api("/api/v1/local/local-cargo") + + meta := `{"name":"e2e-crate","vers":"0.1.0","deps":[],"features":{}}` + crate := []byte("e2e crate bytes") + publish := func(meta string) *http.Response { + var body []byte + body = binary.LittleEndian.AppendUint32(body, uint32(len(meta))) + body = append(body, meta...) + body = binary.LittleEndian.AppendUint32(body, uint32(len(crate))) + body = append(body, crate...) + resp, _ := doRequest(t, http.MethodPut, base+"/api/v1/crates/new", body, "application/octet-stream") + return resp + } + + if resp := publish(meta); resp.StatusCode != http.StatusOK { + t.Fatalf("publish: status %d", resp.StatusCode) + } + if resp := publish(meta); resp.StatusCode != http.StatusConflict { + t.Fatalf("republish: status %d, want 409", resp.StatusCode) + } + if resp := publish(`{"name":"e2e-crate","vers":"0.1.0+other","deps":[],"features":{}}`); resp.StatusCode != http.StatusConflict { + t.Fatalf("build-metadata republish: status %d, want 409", resp.StatusCode) + } + if resp := publish(`{"name":"E2E_crate","vers":"0.2.0","deps":[],"features":{}}`); resp.StatusCode != http.StatusBadRequest { + t.Fatalf("name collision publish: status %d, want 400", resp.StatusCode) + } + + resp, b := doRequest(t, http.MethodGet, base+"/config.json", nil, "") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"api":"`) { + t.Fatalf("config.json: status %d: %s", resp.StatusCode, b) + } + + resp, b = doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, "") + if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"cksum":"`) || !strings.Contains(string(b), `"yanked":false`) { + t.Fatalf("index: status %d: %s", resp.StatusCode, b) + } + + resp, b = doRequest(t, http.MethodGet, base+"/crates/e2e-crate/e2e-crate-0.1.0.crate", nil, "") + if resp.StatusCode != http.StatusOK || !bytes.Equal(b, crate) { + t.Fatalf("download: status %d", resp.StatusCode) + } + + if resp, b := doRequest(t, http.MethodDelete, base+"/api/v1/crates/e2e-crate/0.1.0/yank", nil, ""); resp.StatusCode != http.StatusOK { + t.Fatalf("yank: status %d: %s", resp.StatusCode, b) + } + if _, b := doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, ""); !strings.Contains(string(b), `"yanked":true`) { + t.Fatalf("index after yank: %s", b) + } +} diff --git a/internal/api/v1/cargo.go b/internal/api/v1/cargo.go new file mode 100644 index 0000000..7159116 --- /dev/null +++ b/internal/api/v1/cargo.go @@ -0,0 +1,139 @@ +package v1 + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + + "github.com/go-chi/chi/v5" + + "git.unkin.net/unkin/artifactapi/internal/database" + "git.unkin.net/unkin/artifactapi/internal/provider" + "git.unkin.net/unkin/artifactapi/internal/provider/cargo" + "git.unkin.net/unkin/artifactapi/pkg/models" +) + +// This file implements the cargo registry Web API (publish, yank, unyank) for +// local cargo repositories. config.json points api at /api/v1/local/, so +// cargo appends its own /api/v1/crates/... below that. + +const maxCratePublish = 64 << 20 + +func (h *ProxyHandler) localCargoRemote(r *http.Request) (*models.Remote, bool) { + remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName")) + if err != nil { + return nil, false + } + return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageCargo +} + +func cargoError(w http.ResponseWriter, status int, format string, args ...any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{ + "errors": []map[string]string{{"detail": fmt.Sprintf(format, args...)}}, + }) +} + +func cargoOK(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(v) +} + +func (h *ProxyHandler) cargoPublish(w http.ResponseWriter, r *http.Request) { + remote, ok := h.localCargoRemote(r) + if !ok { + cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName")) + return + } + ctx := r.Context() + + body, err := io.ReadAll(io.LimitReader(r.Body, maxCratePublish+1)) + if err != nil { + cargoError(w, http.StatusBadRequest, "read body: %v", err) + return + } + if len(body) > maxCratePublish { + cargoError(w, http.StatusRequestEntityTooLarge, "publish body exceeds %d bytes", maxCratePublish) + return + } + meta, crate, err := cargo.ParsePublish(body) + if err != nil { + cargoError(w, http.StatusBadRequest, "%v", err) + return + } + line, err := cargo.NewIndexLine(meta, crate) + if err != nil { + cargoError(w, http.StatusBadRequest, "%v", err) + return + } + + lineJSON, err := json.Marshal(line) + if err != nil { + cargoError(w, http.StatusInternalServerError, "%v", err) + return + } + + path := cargo.CratePath(meta.Name, meta.Vers) + result, err := h.cas.Store(ctx, bytes.NewReader(crate), "application/gzip") + if err != nil { + cargoError(w, http.StatusInternalServerError, "store failed: %v", err) + return + } + if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, "application/gzip"); err != nil { + cargoError(w, http.StatusInternalServerError, "record blob: %v", err) + return + } + if err := h.db.CreateLocalFile(ctx, remote.Name, path, result.ContentHash); err != nil { + if errors.Is(err, database.ErrAlreadyExists) { + cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers) + return + } + cargoError(w, http.StatusInternalServerError, "record file: %v", err) + return + } + entry := &provider.CargoIndexEntry{RepoName: remote.Name, FilePath: path, Name: meta.Name, Version: meta.Vers, Line: string(lineJSON)} + if err := h.db.InsertCargoIndexEntry(ctx, entry); err != nil { + // Drop the file so a retry is not rejected as a duplicate. + _ = h.db.DeleteLocalFile(ctx, remote.Name, path) + switch { + case errors.Is(err, database.ErrCargoNameTaken): + cargoError(w, http.StatusBadRequest, "%v", err) + case errors.Is(err, database.ErrAlreadyExists): + cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers) + default: + cargoError(w, http.StatusInternalServerError, "record index: %v", err) + } + return + } + + slog.Info("cargo: published", "repo", remote.Name, "crate", meta.Name, "version", meta.Vers) + cargoOK(w, map[string]any{"warnings": map[string][]string{"invalid_categories": {}, "invalid_badges": {}, "other": {}}}) +} + +func (h *ProxyHandler) cargoYank(yanked bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + remote, ok := h.localCargoRemote(r) + if !ok { + cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName")) + return + } + name, version := chi.URLParam(r, "crate"), chi.URLParam(r, "version") + found, err := h.db.SetCargoYanked(r.Context(), remote.Name, name, version, yanked) + if err != nil { + cargoError(w, http.StatusInternalServerError, "%v", err) + return + } + if !found { + cargoError(w, http.StatusNotFound, "crate version `%s@%s` not found", name, version) + return + } + slog.Info("cargo: yank updated", "repo", remote.Name, "crate", name, "version", version, "yanked", yanked) + cargoOK(w, map[string]bool{"ok": true}) + } +} diff --git a/internal/api/v1/proxy.go b/internal/api/v1/proxy.go index 7429007..d0332f3 100644 --- a/internal/api/v1/proxy.go +++ b/internal/api/v1/proxy.go @@ -42,6 +42,9 @@ func (h *ProxyHandler) Routes() chi.Router { r := chi.NewRouter() r.Get("/remote/{remoteName}/*", h.handleProxy) r.Get("/local/{localName}/*", h.handleLocal) + r.Put("/local/{localName}/api/v1/crates/new", h.cargoPublish) + r.Delete("/local/{localName}/api/v1/crates/{crate}/{version}/yank", h.cargoYank(true)) + r.Put("/local/{localName}/api/v1/crates/{crate}/{version}/unyank", h.cargoYank(false)) r.Get("/virtual/{virtualName}/*", h.handleVirtual) return r } diff --git a/internal/database/cargo_index.go b/internal/database/cargo_index.go new file mode 100644 index 0000000..adb9af0 --- /dev/null +++ b/internal/database/cargo_index.go @@ -0,0 +1,111 @@ +package database + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgconn" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +var ErrCargoNameTaken = errors.New("crate name already taken") + +func cargoNormName(name string) string { + return strings.ReplaceAll(strings.ToLower(name), "-", "_") +} + +// InsertCargoIndexEntry rejects a name that differs from an existing crate only +// by case or -/_ (ErrCargoNameTaken) and a version equal up to build metadata +// (ErrAlreadyExists). A per-crate advisory lock serialises concurrent publishes. +func (db *DB) InsertCargoIndexEntry(ctx context.Context, e *provider.CargoIndexEntry) error { + tx, err := db.Pool.Begin(ctx) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + + norm := cargoNormName(e.Name) + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtext('cargo:' || $1 || '/' || $2))`, e.RepoName, norm); err != nil { + return err + } + rows, err := tx.Query(ctx, ` + SELECT name, version FROM cargo_index + WHERE repo_name = $1 AND replace(lower(name), '-', '_') = $2 + `, e.RepoName, norm) + if err != nil { + return err + } + defer rows.Close() + base, _, _ := strings.Cut(e.Version, "+") + for rows.Next() { + var name, version string + if err := rows.Scan(&name, &version); err != nil { + return err + } + if name != e.Name { + return fmt.Errorf("%w: %q exists as %q", ErrCargoNameTaken, e.Name, name) + } + if v, _, _ := strings.Cut(version, "+"); v == base { + return fmt.Errorf("%w: %s@%s", ErrAlreadyExists, name, version) + } + } + if err := rows.Err(); err != nil { + return err + } + rows.Close() + + if _, err := tx.Exec(ctx, ` + INSERT INTO cargo_index (repo_name, file_path, name, version, line, yanked) + VALUES ($1, $2, $3, $4, $5, $6) + `, e.RepoName, e.FilePath, e.Name, e.Version, e.Line, e.Yanked); err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return fmt.Errorf("%w: %s@%s", ErrAlreadyExists, e.Name, e.Version) + } + return err + } + return tx.Commit(ctx) +} + +func (db *DB) ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]provider.CargoIndexEntry, error) { + rows, err := db.Pool.Query(ctx, ` + SELECT repo_name, file_path, name, version, line, yanked + FROM cargo_index + WHERE repo_name = $1 AND lower(name) = $2 + ORDER BY id + `, repoName, strings.ToLower(name)) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []provider.CargoIndexEntry + for rows.Next() { + var e provider.CargoIndexEntry + if err := rows.Scan(&e.RepoName, &e.FilePath, &e.Name, &e.Version, &e.Line, &e.Yanked); err != nil { + return nil, err + } + result = append(result, e) + } + return result, rows.Err() +} + +// SetCargoYanked reports false when no such crate version exists. +func (db *DB) SetCargoYanked(ctx context.Context, repoName, name, version string, yanked bool) (bool, error) { + tag, err := db.Pool.Exec(ctx, ` + UPDATE cargo_index SET yanked = $4 + WHERE repo_name = $1 AND lower(name) = $2 AND version = $3 + `, repoName, strings.ToLower(name), version, yanked) + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil +} + +func (db *DB) DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error { + _, err := db.Pool.Exec(ctx, `DELETE FROM cargo_index WHERE repo_name = $1 AND file_path = $2`, repoName, filePath) + return err +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 2f11049..c7b4e31 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -2,7 +2,10 @@ package database import ( "context" + "errors" + "fmt" "os" + "sync" "testing" "time" @@ -425,3 +428,115 @@ func TestRPMMetadata(t *testing.T) { t.Fatalf("list rpm rows: %v %v", len(rows), err) } } + +func TestCargoIndex(t *testing.T) { + requireDB(t) + for _, v := range []string{"0.1.0", "0.2.0"} { + if err := testDB.InsertCargoIndexEntry(ctx(), &provider.CargoIndexEntry{ + RepoName: "cargo-db", FilePath: "crates/Unkin-Hello/Unkin-Hello-" + v + ".crate", + Name: "Unkin-Hello", Version: v, Line: `{"vers":"` + v + `"}`, + }); err != nil { + t.Fatal(err) + } + } + + found, err := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "0.1.0", true) + if err != nil || !found { + t.Fatalf("yank: found=%v err=%v", found, err) + } + if found, _ := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "9.9.9", true); found { + t.Error("yank of a missing version reported found") + } + + entries, err := testDB.ListCargoIndexEntries(ctx(), "cargo-db", "unkin-hello") + if err != nil { + t.Fatal(err) + } + if len(entries) != 2 || entries[0].Version != "0.1.0" || !entries[0].Yanked || entries[1].Yanked { + t.Fatalf("entries = %+v", entries) + } + + if err := testDB.DeleteCargoIndexEntry(ctx(), "cargo-db", entries[0].FilePath); err != nil { + t.Fatal(err) + } + if entries, _ = testDB.ListCargoIndexEntries(ctx(), "cargo-db", "UNKIN-HELLO"); len(entries) != 1 { + t.Fatalf("after delete: %+v", entries) + } +} + +func TestCargoIndexPublishConflicts(t *testing.T) { + requireDB(t) + entry := func(name, version string) *provider.CargoIndexEntry { + return &provider.CargoIndexEntry{ + RepoName: "cargo-conflict", FilePath: "crates/" + name + "/" + name + "-" + version + ".crate", + Name: name, Version: version, Line: "{}", + } + } + if err := testDB.InsertCargoIndexEntry(ctx(), entry("foo-bar", "1.0.0+a")); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, version string + want error + }{ + {"foo-bar", "1.0.0+b", ErrAlreadyExists}, + {"foo-bar", "1.0.0", ErrAlreadyExists}, + {"Foo-Bar", "2.0.0", ErrCargoNameTaken}, + {"foo_bar", "2.0.0", ErrCargoNameTaken}, + } { + if err := testDB.InsertCargoIndexEntry(ctx(), entry(tc.name, tc.version)); !errors.Is(err, tc.want) { + t.Errorf("%s@%s: err = %v, want %v", tc.name, tc.version, err, tc.want) + } + } + if err := testDB.InsertCargoIndexEntry(ctx(), entry("foo-bar", "1.0.1")); err != nil { + t.Fatal(err) + } +} + +func TestCargoIndexConcurrentPublish(t *testing.T) { + requireDB(t) + names := []string{"Race", "race", "RACE", "race", "rAce", "RaCe", "racE", "race"} + errs := make(chan error, len(names)) + var wg sync.WaitGroup + for i, name := range names { + wg.Add(1) + go func() { + defer wg.Done() + errs <- testDB.InsertCargoIndexEntry(ctx(), &provider.CargoIndexEntry{ + RepoName: "cargo-race", FilePath: fmt.Sprintf("crates/%s/%d.crate", name, i), + Name: name, Version: "1.0.0", Line: "{}", + }) + }() + } + wg.Wait() + close(errs) + ok := 0 + for err := range errs { + switch { + case err == nil: + ok++ + case !errors.Is(err, ErrAlreadyExists) && !errors.Is(err, ErrCargoNameTaken): + t.Errorf("unexpected error: %v", err) + } + } + if ok != 1 { + t.Fatalf("%d concurrent publishes succeeded, want 1", ok) + } + if entries, _ := testDB.ListCargoIndexEntries(ctx(), "cargo-race", "race"); len(entries) > 1 { + t.Fatalf("entries = %+v", entries) + } +} + +func TestCargoIndexUniqueIndex(t *testing.T) { + requireDB(t) + insert := func(name, version, path string) error { + _, err := testDB.Pool.Exec(ctx(), `INSERT INTO cargo_index (repo_name, file_path, name, version, line) VALUES ('cargo-uniq', $1, $2, $3, '{}')`, path, name, version) + return err + } + if err := insert("a-b", "1.0.0+x", "p1"); err != nil { + t.Fatal(err) + } + if err := insert("A_B", "1.0.0+y", "p2"); err == nil { + t.Fatal("unique index allowed a normalised duplicate") + } +} diff --git a/internal/provider/cargo/local.go b/internal/provider/cargo/local.go new file mode 100644 index 0000000..0a54d16 --- /dev/null +++ b/internal/provider/cargo/local.go @@ -0,0 +1,307 @@ +package cargo + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "regexp" + "strings" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +// publishDep is a dependency as cargo sends it in the publish metadata. +type publishDep struct { + Name string `json:"name"` + VersionReq string `json:"version_req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry"` + ExplicitNameInToml *string `json:"explicit_name_in_toml"` +} + +// PublishMeta is the JSON half of a `PUT /api/v1/crates/new` body; fields the +// index does not carry are ignored. +type PublishMeta struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []publishDep `json:"deps"` + Features map[string][]string `json:"features"` + Links *string `json:"links"` + RustVersion *string `json:"rust_version"` +} + +type indexDep struct { + Name string `json:"name"` + Req string `json:"req"` + Features []string `json:"features"` + Optional bool `json:"optional"` + DefaultFeatures bool `json:"default_features"` + Target *string `json:"target"` + Kind string `json:"kind"` + Registry *string `json:"registry,omitempty"` + Package *string `json:"package,omitempty"` +} + +// IndexLine is one line of a sparse index file. +type IndexLine struct { + Name string `json:"name"` + Vers string `json:"vers"` + Deps []indexDep `json:"deps"` + Cksum string `json:"cksum"` + Features map[string][]string `json:"features"` + Features2 map[string][]string `json:"features2,omitempty"` + Yanked bool `json:"yanked"` + Links *string `json:"links,omitempty"` + V int `json:"v,omitempty"` + RustVersion *string `json:"rust_version,omitempty"` +} + +// ParsePublish splits a publish body: u32le JSON length, JSON, u32le crate length, crate. +func ParsePublish(body []byte) (PublishMeta, []byte, error) { + var meta PublishMeta + jsonBytes, rest, err := lengthPrefixed(body) + if err != nil { + return meta, nil, fmt.Errorf("metadata: %w", err) + } + crate, rest, err := lengthPrefixed(rest) + if err != nil { + return meta, nil, fmt.Errorf("crate: %w", err) + } + if len(rest) != 0 { + return meta, nil, errors.New("trailing bytes after crate") + } + if err := json.Unmarshal(jsonBytes, &meta); err != nil { + return meta, nil, fmt.Errorf("metadata: %w", err) + } + return meta, crate, nil +} + +func lengthPrefixed(b []byte) ([]byte, []byte, error) { + if len(b) < 4 { + return nil, nil, errors.New("truncated length") + } + n := binary.LittleEndian.Uint32(b) + b = b[4:] + if uint64(n) > uint64(len(b)) { + return nil, nil, errors.New("truncated body") + } + return b[:n], b[n:], nil +} + +// NewIndexLine validates the metadata and translates it into index form. +func NewIndexLine(meta PublishMeta, crate []byte) (IndexLine, error) { + if !validName(meta.Name) { + return IndexLine{}, fmt.Errorf("invalid crate name %q", meta.Name) + } + if !validVersion(meta.Vers) { + return IndexLine{}, fmt.Errorf("invalid version %q", meta.Vers) + } + if len(crate) == 0 { + return IndexLine{}, errors.New("empty .crate") + } + sum := sha256.Sum256(crate) + line := IndexLine{ + Name: meta.Name, + Vers: meta.Vers, + Deps: make([]indexDep, 0, len(meta.Deps)), + Cksum: hex.EncodeToString(sum[:]), + Features: map[string][]string{}, + Links: meta.Links, + RustVersion: meta.RustVersion, + } + for _, d := range meta.Deps { + dep := indexDep{ + Name: d.Name, + Req: d.VersionReq, + Features: d.Features, + Optional: d.Optional, + DefaultFeatures: d.DefaultFeatures, + Target: d.Target, + Kind: d.Kind, + Registry: d.Registry, + } + if dep.Features == nil { + dep.Features = []string{} + } + if dep.Kind == "" { + dep.Kind = "normal" + } + if d.ExplicitNameInToml != nil && *d.ExplicitNameInToml != "" { + pkg := d.Name + dep.Name, dep.Package = *d.ExplicitNameInToml, &pkg + } + line.Deps = append(line.Deps, dep) + } + // Older cargo cannot parse dep:/?/ feature syntax, so those features live in + // features2 behind v:2, as on crates.io. + for name, vals := range meta.Features { + if vals == nil { + vals = []string{} + } + if usesNewFeatureSyntax(vals) { + if line.Features2 == nil { + line.Features2 = map[string][]string{} + } + line.Features2[name] = vals + continue + } + line.Features[name] = vals + } + if line.Features2 != nil { + line.V = 2 + } + return line, nil +} + +func usesNewFeatureSyntax(vals []string) bool { + for _, v := range vals { + if strings.HasPrefix(v, "dep:") || strings.Contains(v, "?/") { + return true + } + } + return false +} + +func validName(name string) bool { + if name == "" || len(name) > 64 { + return false + } + for i, c := range name { + alpha := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') + if i == 0 && !alpha { + return false + } + if !alpha && (c < '0' || c > '9') && c != '-' && c != '_' { + return false + } + } + return true +} + +// semverRe is the SemVer 2.0.0 grammar from semver.org. +var semverRe = regexp.MustCompile(`^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`) + +func validVersion(v string) bool { + return len(v) <= 128 && semverRe.MatchString(v) +} + +// IndexPath is the sparse-index file path for a crate name. +func IndexPath(name string) string { + n := strings.ToLower(name) + switch len(n) { + case 0: + return "" + case 1: + return "1/" + n + case 2: + return "2/" + n + case 3: + return "3/" + n[:1] + "/" + n + } + return n[:2] + "/" + n[2:4] + "/" + n +} + +// CratePath is where a published .crate is stored, matching config.json's dl. +func CratePath(name, version string) string { + return crateDir + name + "/" + name + "-" + version + ".crate" +} + +// RenderIndex joins stored lines into an index file with each yanked flag applied. +func RenderIndex(entries []provider.CargoIndexEntry) ([]byte, error) { + var b strings.Builder + for _, e := range entries { + var line IndexLine + if err := json.Unmarshal([]byte(e.Line), &line); err != nil { + return nil, fmt.Errorf("%s %s: %w", e.Name, e.Version, err) + } + line.Yanked = e.Yanked + raw, err := json.Marshal(line) + if err != nil { + return nil, err + } + b.Write(raw) + b.WriteByte('\n') + } + return []byte(b.String()), nil +} + +// LocalConfig is config.json for a local registry rooted at base. +func LocalConfig(base string) map[string]string { + return map[string]string{ + "dl": base + "/" + crateDir + "{crate}/{crate}-{version}.crate", + "api": base, + } +} + +func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool { + path = strings.TrimLeft(path, "/") + if path == "config.json" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(LocalConfig(requestBase(r) + "/api/v1/local/" + repoName)) + return true + } + name := path[strings.LastIndex(path, "/")+1:] + if name == "" || IndexPath(name) != path { + return false + } + + reader, ok := files.(provider.CargoIndexReader) + if !ok { + http.Error(w, "cargo index not available", http.StatusInternalServerError) + return true + } + entries, err := reader.ListCargoIndexEntries(r.Context(), repoName, name) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + if len(entries) == 0 { + http.Error(w, "not found", http.StatusNotFound) + return true + } + body, err := RenderIndex(entries) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return true + } + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(body) + return true +} + +func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) { + return nil, fmt.Errorf("cargo local index generation for virtual repos not supported") +} + +func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error { + deleter, ok := db.(provider.CargoIndexDeleter) + if !ok { + return nil + } + if err := deleter.DeleteCargoIndexEntry(ctx, repoName, storagePath); err != nil { + slog.Error("cargo index: delete failed", "repo", repoName, "path", storagePath, "error", err) + return err + } + return nil +} + +func requestBase(r *http.Request) string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } else if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" { + scheme = fwd + } + return scheme + "://" + r.Host +} diff --git a/internal/provider/cargo/local_test.go b/internal/provider/cargo/local_test.go new file mode 100644 index 0000000..a55594d --- /dev/null +++ b/internal/provider/cargo/local_test.go @@ -0,0 +1,232 @@ +package cargo + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "net/http/httptest" + "strings" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/provider" +) + +func publishBody(meta string, crate []byte) []byte { + var b []byte + b = binary.LittleEndian.AppendUint32(b, uint32(len(meta))) + b = append(b, meta...) + b = binary.LittleEndian.AppendUint32(b, uint32(len(crate))) + return append(b, crate...) +} + +const sampleMeta = `{ + "name": "unkin-hello", "vers": "0.2.0", + "deps": [ + {"name": "serde", "version_req": "^1", "features": ["derive"], "optional": false, + "default_features": true, "target": null, "kind": "normal", + "registry": "https://github.com/rust-lang/crates.io-index"}, + {"name": "rand_core", "version_req": "^0.6", "features": null, "optional": true, + "default_features": false, "target": "cfg(unix)", "kind": "dev", + "explicit_name_in_toml": "rc"} + ], + "features": {"default": ["std"], "std": [], "extra": ["dep:rc"], "weak": ["serde?/std"]}, + "links": null, "rust_version": "1.70", "description": "ignored" +}` + +func TestParsePublishAndIndexLine(t *testing.T) { + crate := []byte("fake crate bytes") + meta, gotCrate, err := ParsePublish(publishBody(sampleMeta, crate)) + if err != nil { + t.Fatal(err) + } + if string(gotCrate) != string(crate) { + t.Fatalf("crate = %q", gotCrate) + } + line, err := NewIndexLine(meta, crate) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(crate) + if line.Cksum != hex.EncodeToString(sum[:]) { + t.Errorf("cksum = %s", line.Cksum) + } + + raw, _ := json.Marshal(line) + var got map[string]any + _ = json.Unmarshal(raw, &got) + if _, ok := got["version_req"]; ok { + t.Error("publish-only field leaked into index line") + } + if got["v"] != float64(2) { + t.Errorf("v = %v, want 2 with features2 present", got["v"]) + } + if got["rust_version"] != "1.70" { + t.Errorf("rust_version = %v", got["rust_version"]) + } + if _, ok := line.Features["extra"]; ok { + t.Error("dep: feature must move to features2") + } + if len(line.Features2) != 2 || len(line.Features) != 2 { + t.Errorf("features = %v features2 = %v", line.Features, line.Features2) + } + + deps := got["deps"].([]any) + serde := deps[0].(map[string]any) + if serde["req"] != "^1" || serde["registry"] != "https://github.com/rust-lang/crates.io-index" || serde["target"] != nil { + t.Errorf("serde dep = %v", serde) + } + if _, ok := serde["package"]; ok { + t.Error("package set without a rename") + } + renamed := deps[1].(map[string]any) + if renamed["name"] != "rc" || renamed["package"] != "rand_core" || renamed["kind"] != "dev" { + t.Errorf("renamed dep = %v", renamed) + } + if f, ok := renamed["features"].([]any); !ok || len(f) != 0 { + t.Errorf("null features must render as [], got %v", renamed["features"]) + } +} + +func TestIndexLineNoNewFeaturesOmitsV(t *testing.T) { + line, err := NewIndexLine(PublishMeta{Name: "a", Vers: "1.0.0", Features: map[string][]string{"x": nil}}, []byte("x")) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(line) + s := string(raw) + for _, bad := range []string{`"v":`, `"features2"`, `"links"`} { + if strings.Contains(s, bad) { + t.Errorf("%s present in %s", bad, s) + } + } + if !strings.Contains(s, `"deps":[]`) || !strings.Contains(s, `"features":{"x":[]}`) { + t.Errorf("deps/features must be non-null: %s", s) + } +} + +func TestParsePublishRejectsMalformed(t *testing.T) { + good := publishBody(`{"name":"a","vers":"1.0.0"}`, []byte("c")) + for name, body := range map[string][]byte{ + "empty": nil, + "short": good[:len(good)-1], + "trailing": append(append([]byte{}, good...), 'x'), + "bad json": publishBody(`{`, []byte("c")), + "no length": good[:6], + } { + if _, _, err := ParsePublish(body); err == nil { + t.Errorf("%s: expected error", name) + } + } +} + +func TestNewIndexLineValidation(t *testing.T) { + for _, m := range []PublishMeta{ + {Name: "", Vers: "1.0.0"}, + {Name: "1abc", Vers: "1.0.0"}, + {Name: "../x", Vers: "1.0.0"}, + {Name: "ok", Vers: "1.0.0/../../x"}, + {Name: "ok", Vers: ""}, + {Name: strings.Repeat("a", 65), Vers: "1.0.0"}, + {Name: "ok", Vers: "1"}, + {Name: "ok", Vers: "1.0"}, + {Name: "ok", Vers: "1..2"}, + {Name: "ok", Vers: "01.0.0"}, + {Name: "ok", Vers: "1.0.0-"}, + {Name: "ok", Vers: "1.0.0-01"}, + {Name: "ok", Vers: "1.0.0+"}, + {Name: "ok", Vers: "1.0.0+a..b"}, + {Name: "ok", Vers: "v1.0.0"}, + } { + if _, err := NewIndexLine(m, []byte("x")); err == nil { + t.Errorf("%q@%q accepted", m.Name, m.Vers) + } + } + for _, v := range []string{"0.0.0", "1.0.0-rc.1+build.5", "1.2.3-alpha.0.x-y", "10.20.30+20261017"} { + if _, err := NewIndexLine(PublishMeta{Name: "Foo_bar-2", Vers: v}, []byte("x")); err != nil { + t.Error(err) + } + } + if _, err := NewIndexLine(PublishMeta{Name: "ok", Vers: "1.0.0"}, nil); err == nil { + t.Error("empty .crate accepted") + } +} + +func TestIndexPath(t *testing.T) { + for name, want := range map[string]string{ + "a": "1/a", "ab": "2/ab", "abc": "3/a/abc", "Serde": "se/rd/serde", "unkin-hello": "un/ki/unkin-hello", + } { + if got := IndexPath(name); got != want { + t.Errorf("IndexPath(%q) = %q, want %q", name, got, want) + } + } +} + +type fakeIndex struct { + entries []provider.CargoIndexEntry + asked string +} + +func (f *fakeIndex) ListCargoIndexEntries(_ context.Context, _, name string) ([]provider.CargoIndexEntry, error) { + f.asked = name + return f.entries, nil +} +func (f *fakeIndex) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) { + return nil, nil +} +func (f *fakeIndex) ListPackages(context.Context, string) ([]string, error) { return nil, nil } + +func TestServeLocalIndex(t *testing.T) { + line, _ := NewIndexLine(PublishMeta{Name: "unkin-hello", Vers: "0.1.0"}, []byte("x")) + raw, _ := json.Marshal(line) + files := &fakeIndex{entries: []provider.CargoIndexEntry{ + {Name: "unkin-hello", Version: "0.1.0", Line: string(raw), Yanked: true}, + {Name: "unkin-hello", Version: "0.2.0", Line: strings.Replace(string(raw), "0.1.0", "0.2.0", 1)}, + }} + p := &Provider{} + + rec := httptest.NewRecorder() + if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "http://aa.example/x", nil), files, "crates-local", "un/ki/unkin-hello") { + t.Fatal("index path not served") + } + lines := strings.Split(strings.TrimSpace(rec.Body.String()), "\n") + if len(lines) != 2 || !strings.Contains(lines[0], `"yanked":true`) || !strings.Contains(lines[1], `"yanked":false`) { + t.Errorf("index body = %q", rec.Body.String()) + } + if files.asked != "unkin-hello" { + t.Errorf("looked up %q", files.asked) + } + + rec = httptest.NewRecorder() + req := httptest.NewRequest("GET", "http://aa.example/config.json", nil) + req.Header.Set("X-Forwarded-Proto", "https") + p.ServeLocalIndex(rec, req, files, "crates-local", "config.json") + var cfg map[string]string + _ = json.Unmarshal(rec.Body.Bytes(), &cfg) + if cfg["api"] != "https://aa.example/api/v1/local/crates-local" || + cfg["dl"] != "https://aa.example/api/v1/local/crates-local/crates/{crate}/{crate}-{version}.crate" { + t.Errorf("config = %v", cfg) + } + + for _, path := range []string{"crates/unkin-hello/unkin-hello-0.1.0.crate", "xx/ki/unkin-hello", ""} { + if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), files, "r", path) { + t.Errorf("%q should fall through", path) + } + } + + files.entries = nil + rec = httptest.NewRecorder() + p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), files, "r", "3/a/abc") + if rec.Code != 404 { + t.Errorf("unknown crate code = %d", rec.Code) + } +} + +func TestCratePathMatchesDL(t *testing.T) { + dl := LocalConfig("http://h/api/v1/local/r")["dl"] + want := strings.NewReplacer("{crate}", "unkin-hello", "{version}", "0.1.0").Replace(dl) + if got := "http://h/api/v1/local/r/" + CratePath("unkin-hello", "0.1.0"); got != want { + t.Errorf("CratePath = %q, dl expands to %q", got, want) + } +} diff --git a/internal/provider/cargo_index.go b/internal/provider/cargo_index.go new file mode 100644 index 0000000..3c7f6aa --- /dev/null +++ b/internal/provider/cargo_index.go @@ -0,0 +1,24 @@ +package provider + +import "context" + +// CargoIndexEntry is one published crate version in a local cargo registry. +// Line is the sparse-index JSON line; Yanked is applied when it is served. +type CargoIndexEntry struct { + RepoName string + FilePath string + Name string + Version string + Line string + Yanked bool +} + +// CargoIndexReader lists every version of one crate (matched case-insensitively). +// *database.DB satisfies it and CargoIndexDeleter. +type CargoIndexReader interface { + ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]CargoIndexEntry, error) +} + +type CargoIndexDeleter interface { + DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error +} diff --git a/migrations/0005_cargo_index.sql b/migrations/0005_cargo_index.sql new file mode 100644 index 0000000..607c9d0 --- /dev/null +++ b/migrations/0005_cargo_index.sql @@ -0,0 +1,19 @@ +-- Local cargo registries: one sparse-index line per published crate version. +-- line holds the index JSON minus yanked, which flips on yank/unyank. +CREATE TABLE IF NOT EXISTS cargo_index ( + id BIGSERIAL PRIMARY KEY, + repo_name TEXT NOT NULL, + file_path TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL, + line TEXT NOT NULL, + yanked BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ DEFAULT NOW(), + UNIQUE(repo_name, file_path) +); + +CREATE INDEX IF NOT EXISTS idx_cargo_index_repo_name ON cargo_index(repo_name, lower(name)); + +-- Cargo treats names equal up to case and -/_, and versions equal up to build metadata. +CREATE UNIQUE INDEX IF NOT EXISTS idx_cargo_index_unique_version + ON cargo_index(repo_name, replace(lower(name), '-', '_'), split_part(version, '+', 1)); diff --git a/ui/src/components/UsageInstructions.tsx b/ui/src/components/UsageInstructions.tsx index 92c1e57..7f58eb2 100644 --- a/ui/src/components/UsageInstructions.tsx +++ b/ui/src/components/UsageInstructions.tsx @@ -229,6 +229,25 @@ go mod download`, ]; case 'cargo': + if (isLocal) { + return [ + { + title: 'Add the private registry (real sparse registry)', + language: 'toml', + code: `# $CARGO_HOME/config.toml +[registries.${name}] +index = "sparse+${url}/api/v1/local/${name}/"`, + note: 'cargo requires a token for publish/yank: set CARGO_REGISTRIES_' + name.toUpperCase().replace(/-/g, '_') + '_TOKEN to any value.', + }, + { + title: 'Publish, depend on and yank crates', + language: 'bash', + code: `cargo publish --registry ${name} +cargo add --registry ${name} +cargo yank --version --registry ${name}`, + }, + ]; + } return [ { title: 'Point crates.io at this sparse registry',