Compare commits

..

2 Commits

Author SHA1 Message Date
unkin-agent 0f826aa331 Retrigger CI
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
2026-10-16 23:09:20 +11:00
unkin-agent 13d693b58a Add ansible Galaxy remote type
ci/woodpecker/pr/build Pipeline failed
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
- add ansible package type proxying the Galaxy v3 API
- rewrite href/links/download_url back to the remote so tarballs flow through it
- forward the query string for providers that opt in, keeping Galaxy pagination working
- add a UI usage snippet and an ansible e2e caching case
2026-10-16 23:06:34 +11:00
21 changed files with 180 additions and 895 deletions
+2 -15
View File
@@ -1,6 +1,6 @@
# ArtifactAPI
Caching proxy for package repositories. Single Go binary, 11 package types, content-addressable storage, managed by Terraform.
Caching proxy for package repositories. Single Go binary, 12 package types, content-addressable storage, managed by Terraform.
## Quick Start
@@ -33,6 +33,7 @@ API: `http://localhost:8000` | Frontend: `http://localhost:5173`
| `terraform` | `*/versions` | `*/download/*/*` |
| `goproxy` | `@v/list`, `@latest` | `.info`, `.mod`, `.zip` |
| `cargo` | sparse index files (`config.json` synthesized) | `crates/*/*.crate` |
| `ansible` | Galaxy `api/` JSON (links rewritten) | `*.tar.gz` |
| `github_rpm` | `repodata/*` (synthesized) | `.rpm` (redirected) |
Providers classify paths automatically. Users only configure what to proxy and TTLs.
@@ -279,20 +280,6 @@ content-addressable store (deduplicated by digest, reaped by GC once
unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
### Local cargo registry
A local `cargo` repo is a private crate registry: it serves the sparse index and
the Web API publish, yank and unyank endpoints, so `cargo publish` works against
it directly.
```toml
[registries.<repo>]
index = "sparse+https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>/"
```
cargo requires a token for publish/yank (`CARGO_REGISTRIES_<REPO>_TOKEN`); like
other local repos, artifactapi does not check it.
## Access Control
| Field | Default | Behaviour |
+2 -3
View File
@@ -23,12 +23,11 @@ already-running stack.
- **Repository lifecycle** — add / change / delete for remote, local and virtual repos.
- **Caching** — one immutable artifact per remote package type (generic, docker,
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo) proxied through the
helm, pypi, npm, rpm, alpine, puppet, terraform, goproxy, cargo, ansible) proxied through the
mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
verified against the origin fixture.
- **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
index), rpm (real package + **automatic repodata** generation), cargo (publish,
sparse index, download, yank).
index), rpm (real package + **automatic repodata** generation).
- **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
across two members.
- **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors
+1
View File
@@ -30,6 +30,7 @@ func TestCachingPerProvider(t *testing.T) {
{"puppet", "puppet-releases/author-mod-1.0.0.tar.gz", "puppet-releases/author-mod-1.0.0.tar.gz"},
{"goproxy", "goproxy/example.com/mod/@v/v1.0.0.zip", "goproxy/example.com/mod/@v/v1.0.0.zip"},
{"cargo", "crates/mycrate/mycrate-1.0.0.crate", "crates/mycrate/mycrate-1.0.0.crate"},
{"ansible", "galaxy/artifacts/ns-col-1.0.0.tar.gz", "galaxy/artifacts/ns-col-1.0.0.tar.gz"},
{"terraform", "hashicorp/aws/download/pkg.zip", "v1/providers/hashicorp/aws/download/pkg.zip"},
{"docker", "library/testimg/blobs/blobdata", "v2/library/testimg/blobs/blobdata"},
}
-56
View File
@@ -1,56 +0,0 @@
//go:build dockere2e
package e2edocker
import (
"bytes"
"encoding/binary"
"net/http"
"strings"
"testing"
)
// TestLocalCargoRegistry publishes through the cargo Web API, then reads the
// sparse index, downloads the crate and yanks it.
func TestLocalCargoRegistry(t *testing.T) {
createRepo(t, `{"name":"local-cargo","package_type":"cargo","repo_type":"local"}`)
defer deleteRepo(t, "local-cargo")
base := api("/api/v1/local/local-cargo")
meta := `{"name":"e2e-crate","vers":"0.1.0","deps":[],"features":{}}`
crate := []byte("e2e crate bytes")
var body []byte
body = binary.LittleEndian.AppendUint32(body, uint32(len(meta)))
body = append(body, meta...)
body = binary.LittleEndian.AppendUint32(body, uint32(len(crate)))
body = append(body, crate...)
if resp, b := doRequest(t, http.MethodPut, base+"/api/v1/crates/new", body, "application/octet-stream"); resp.StatusCode != http.StatusOK {
t.Fatalf("publish: status %d: %s", resp.StatusCode, b)
}
if resp, _ := doRequest(t, http.MethodPut, base+"/api/v1/crates/new", body, "application/octet-stream"); resp.StatusCode != http.StatusConflict {
t.Fatalf("republish: status %d, want 409", resp.StatusCode)
}
resp, b := doRequest(t, http.MethodGet, base+"/config.json", nil, "")
if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"api":"`) {
t.Fatalf("config.json: status %d: %s", resp.StatusCode, b)
}
resp, b = doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, "")
if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"cksum":"`) || !strings.Contains(string(b), `"yanked":false`) {
t.Fatalf("index: status %d: %s", resp.StatusCode, b)
}
resp, b = doRequest(t, http.MethodGet, base+"/crates/e2e-crate/e2e-crate-0.1.0.crate", nil, "")
if resp.StatusCode != http.StatusOK || !bytes.Equal(b, crate) {
t.Fatalf("download: status %d", resp.StatusCode)
}
if resp, b := doRequest(t, http.MethodDelete, base+"/api/v1/crates/e2e-crate/0.1.0/yank", nil, ""); resp.StatusCode != http.StatusOK {
t.Fatalf("yank: status %d: %s", resp.StatusCode, b)
}
if _, b := doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, ""); !strings.Contains(string(b), `"yanked":true`) {
t.Fatalf("index after yank: %s", b)
}
}
-148
View File
@@ -1,148 +0,0 @@
package v1
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"github.com/go-chi/chi/v5"
"git.unkin.net/unkin/artifactapi/internal/database"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/provider/cargo"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
// This file implements the cargo registry Web API (publish, yank, unyank) for
// local cargo repositories. config.json points api at /api/v1/local/<repo>, so
// cargo appends its own /api/v1/crates/... below that.
const maxCratePublish = 64 << 20
func (h *ProxyHandler) localCargoRemote(r *http.Request) (*models.Remote, bool) {
remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
if err != nil {
return nil, false
}
return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageCargo
}
func cargoError(w http.ResponseWriter, status int, format string, args ...any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(map[string]any{
"errors": []map[string]string{{"detail": fmt.Sprintf(format, args...)}},
})
}
func cargoOK(w http.ResponseWriter, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_ = json.NewEncoder(w).Encode(v)
}
func (h *ProxyHandler) cargoPublish(w http.ResponseWriter, r *http.Request) {
remote, ok := h.localCargoRemote(r)
if !ok {
cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName"))
return
}
ctx := r.Context()
body, err := io.ReadAll(io.LimitReader(r.Body, maxCratePublish+1))
if err != nil {
cargoError(w, http.StatusBadRequest, "read body: %v", err)
return
}
if len(body) > maxCratePublish {
cargoError(w, http.StatusRequestEntityTooLarge, "publish body exceeds %d bytes", maxCratePublish)
return
}
meta, crate, err := cargo.ParsePublish(body)
if err != nil {
cargoError(w, http.StatusBadRequest, "%v", err)
return
}
line, err := cargo.NewIndexLine(meta, crate)
if err != nil {
cargoError(w, http.StatusBadRequest, "%v", err)
return
}
existing, err := h.db.ListCargoIndexEntries(ctx, remote.Name, meta.Name)
if err != nil {
cargoError(w, http.StatusInternalServerError, "%v", err)
return
}
for _, e := range existing {
if e.Name != meta.Name {
cargoError(w, http.StatusBadRequest, "crate %q already exists as %q", meta.Name, e.Name)
return
}
if e.Version == meta.Vers {
cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers)
return
}
}
lineJSON, err := json.Marshal(line)
if err != nil {
cargoError(w, http.StatusInternalServerError, "%v", err)
return
}
path := cargo.CratePath(meta.Name, meta.Vers)
result, err := h.cas.Store(ctx, bytes.NewReader(crate), "application/gzip")
if err != nil {
cargoError(w, http.StatusInternalServerError, "store failed: %v", err)
return
}
if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, "application/gzip"); err != nil {
cargoError(w, http.StatusInternalServerError, "record blob: %v", err)
return
}
if err := h.db.CreateLocalFile(ctx, remote.Name, path, result.ContentHash); err != nil {
if errors.Is(err, database.ErrAlreadyExists) {
cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers)
return
}
cargoError(w, http.StatusInternalServerError, "record file: %v", err)
return
}
entry := &provider.CargoIndexEntry{RepoName: remote.Name, FilePath: path, Name: meta.Name, Version: meta.Vers, Line: string(lineJSON)}
if err := h.db.InsertCargoIndexEntry(ctx, entry); err != nil {
// Drop the file so a retry is not rejected as a duplicate.
_ = h.db.DeleteLocalFile(ctx, remote.Name, path)
cargoError(w, http.StatusInternalServerError, "record index: %v", err)
return
}
slog.Info("cargo: published", "repo", remote.Name, "crate", meta.Name, "version", meta.Vers)
cargoOK(w, map[string]any{"warnings": map[string][]string{"invalid_categories": {}, "invalid_badges": {}, "other": {}}})
}
func (h *ProxyHandler) cargoYank(yanked bool) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
remote, ok := h.localCargoRemote(r)
if !ok {
cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName"))
return
}
name, version := chi.URLParam(r, "crate"), chi.URLParam(r, "version")
found, err := h.db.SetCargoYanked(r.Context(), remote.Name, name, version, yanked)
if err != nil {
cargoError(w, http.StatusInternalServerError, "%v", err)
return
}
if !found {
cargoError(w, http.StatusNotFound, "crate version `%s@%s` not found", name, version)
return
}
slog.Info("cargo: yank updated", "repo", remote.Name, "crate", name, "version", version, "yanked", yanked)
cargoOK(w, map[string]bool{"ok": true})
}
}
+4 -3
View File
@@ -41,9 +41,6 @@ func (h *ProxyHandler) Routes() chi.Router {
r := chi.NewRouter()
r.Get("/remote/{remoteName}/*", h.handleProxy)
r.Get("/local/{localName}/*", h.handleLocal)
r.Put("/local/{localName}/api/v1/crates/new", h.cargoPublish)
r.Delete("/local/{localName}/api/v1/crates/{crate}/{version}/yank", h.cargoYank(true))
r.Put("/local/{localName}/api/v1/crates/{crate}/{version}/unyank", h.cargoYank(false))
r.Get("/virtual/{virtualName}/*", h.handleVirtual)
return r
}
@@ -95,6 +92,10 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
}
}
if _, ok := prov.(provider.QueryForwarder); ok && r.URL.RawQuery != "" {
path += "?" + r.URL.RawQuery
}
result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
if err != nil {
var proxyErr *proxy.ProxyError
-56
View File
@@ -1,56 +0,0 @@
package database
import (
"context"
"strings"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertCargoIndexEntry(ctx context.Context, e *provider.CargoIndexEntry) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO cargo_index (repo_name, file_path, name, version, line, yanked)
VALUES ($1, $2, $3, $4, $5, $6)
`, e.RepoName, e.FilePath, e.Name, e.Version, e.Line, e.Yanked)
return err
}
func (db *DB) ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]provider.CargoIndexEntry, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, name, version, line, yanked
FROM cargo_index
WHERE repo_name = $1 AND lower(name) = $2
ORDER BY id
`, repoName, strings.ToLower(name))
if err != nil {
return nil, err
}
defer rows.Close()
var result []provider.CargoIndexEntry
for rows.Next() {
var e provider.CargoIndexEntry
if err := rows.Scan(&e.RepoName, &e.FilePath, &e.Name, &e.Version, &e.Line, &e.Yanked); err != nil {
return nil, err
}
result = append(result, e)
}
return result, rows.Err()
}
// SetCargoYanked reports false when no such crate version exists.
func (db *DB) SetCargoYanked(ctx context.Context, repoName, name, version string, yanked bool) (bool, error) {
tag, err := db.Pool.Exec(ctx, `
UPDATE cargo_index SET yanked = $4
WHERE repo_name = $1 AND lower(name) = $2 AND version = $3
`, repoName, strings.ToLower(name), version, yanked)
if err != nil {
return false, err
}
return tag.RowsAffected() > 0, nil
}
func (db *DB) DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM cargo_index WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
-35
View File
@@ -425,38 +425,3 @@ func TestRPMMetadata(t *testing.T) {
t.Fatalf("list rpm rows: %v %v", len(rows), err)
}
}
func TestCargoIndex(t *testing.T) {
requireDB(t)
for _, v := range []string{"0.1.0", "0.2.0"} {
if err := testDB.InsertCargoIndexEntry(ctx(), &provider.CargoIndexEntry{
RepoName: "cargo-db", FilePath: "crates/Unkin-Hello/Unkin-Hello-" + v + ".crate",
Name: "Unkin-Hello", Version: v, Line: `{"vers":"` + v + `"}`,
}); err != nil {
t.Fatal(err)
}
}
found, err := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "0.1.0", true)
if err != nil || !found {
t.Fatalf("yank: found=%v err=%v", found, err)
}
if found, _ := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "9.9.9", true); found {
t.Error("yank of a missing version reported found")
}
entries, err := testDB.ListCargoIndexEntries(ctx(), "cargo-db", "unkin-hello")
if err != nil {
t.Fatal(err)
}
if len(entries) != 2 || entries[0].Version != "0.1.0" || !entries[0].Yanked || entries[1].Yanked {
t.Fatalf("entries = %+v", entries)
}
if err := testDB.DeleteCargoIndexEntry(ctx(), "cargo-db", entries[0].FilePath); err != nil {
t.Fatal(err)
}
if entries, _ = testDB.ListCargoIndexEntries(ctx(), "cargo-db", "UNKIN-HELLO"); len(entries) != 1 {
t.Fatalf("after delete: %+v", entries)
}
}
+65
View File
@@ -0,0 +1,65 @@
// Package ansible proxies an Ansible Galaxy server (v3 collections API):
// mutable JSON metadata whose links are rewritten back to the remote, plus
// immutable collection tarballs.
package ansible
import (
"context"
"encoding/json"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func init() {
provider.Register(&Provider{})
}
type Provider struct{}
func (p *Provider) Type() models.PackageType { return models.PackageAnsible }
func (p *Provider) Classify(path string) provider.Mutability {
if isArtifact(path) {
return provider.Immutable
}
return provider.Mutable
}
func (p *Provider) ContentType(path string) string {
if isArtifact(path) {
return "application/gzip"
}
return "application/json"
}
func (p *Provider) UpstreamURL(remote models.Remote, path string) string {
return strings.TrimRight(remote.BaseURL, "/") + "/" + strings.TrimLeft(path, "/")
}
// ForwardQuery keeps limit/offset pagination in the proxied path.
func (p *Provider) ForwardQuery() {}
// RewriteResponse points root-relative hrefs/links and absolute download_url
// values at the remote. Links stay root-relative, which ansible-galaxy joins
// onto the server URL, so the cached body is independent of the request host.
func (p *Provider) RewriteResponse(body []byte, remote models.Remote, _ string) ([]byte, error) {
if !json.Valid(body) {
return nil, nil
}
prefix := `"/api/v1/remote/` + remote.Name + `/`
content := strings.ReplaceAll(string(body), `"/api/`, prefix+`api/`)
content = strings.ReplaceAll(content, `"`+strings.TrimRight(remote.BaseURL, "/")+`/`, prefix)
return []byte(content), nil
}
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
func isArtifact(path string) bool {
return strings.HasSuffix(path, ".tar.gz")
}
+85
View File
@@ -0,0 +1,85 @@
package ansible
import (
"encoding/json"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
func TestClassify(t *testing.T) {
p := &Provider{}
tests := []struct {
path string
want provider.Mutability
}{
{"api/", provider.Mutable},
{"api/v3/collections/community/general/versions/?limit=100", provider.Mutable},
{"api/v3/collections/ansible/posix/versions/2.0.0/", provider.Mutable},
{"api/v3/plugin/ansible/content/published/collections/artifacts/ansible-posix-2.0.0.tar.gz", provider.Immutable},
}
for _, tt := range tests {
if got := p.Classify(tt.path); got != tt.want {
t.Errorf("Classify(%q) = %v, want %v", tt.path, got, tt.want)
}
}
}
func TestUpstreamURLKeepsQuery(t *testing.T) {
p := &Provider{}
got := p.UpstreamURL(models.Remote{BaseURL: "https://galaxy.ansible.com/"}, "api/v3/collections/a/b/versions/?limit=100&offset=100")
want := "https://galaxy.ansible.com/api/v3/collections/a/b/versions/?limit=100&offset=100"
if got != want {
t.Errorf("UpstreamURL = %q, want %q", got, want)
}
}
func TestRewriteResponse(t *testing.T) {
p := &Provider{}
remote := models.Remote{Name: "galaxy", BaseURL: "https://galaxy.ansible.com/"}
body := `{"links":{"next":"/api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10","previous":null},` +
`"href":"/api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/",` +
`"download_url":"https://galaxy.ansible.com/api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz",` +
`"metadata":{"repository":"https://github.com/a/b"}}`
out, err := p.RewriteResponse([]byte(body), remote, "")
if err != nil {
t.Fatal(err)
}
var got struct {
Links map[string]any `json:"links"`
Href string `json:"href"`
DownloadURL string `json:"download_url"`
Metadata map[string]any `json:"metadata"`
}
if err := json.Unmarshal(out, &got); err != nil {
t.Fatalf("rewritten body is not JSON: %v", err)
}
pre := "/api/v1/remote/galaxy/"
if want := pre + "api/v3/plugin/ansible/content/published/collections/index/a/b/versions/?limit=10&offset=10"; got.Links["next"] != want {
t.Errorf("next = %v, want %q", got.Links["next"], want)
}
if want := pre + "api/content/published/v3/plugin/ansible/content/published/collections/index/a/b/versions/1.0.0/"; got.Href != want {
t.Errorf("href = %q, want %q", got.Href, want)
}
if want := pre + "api/v3/plugin/ansible/content/published/collections/artifacts/a-b-1.0.0.tar.gz"; got.DownloadURL != want {
t.Errorf("download_url = %q, want %q", got.DownloadURL, want)
}
if got.Metadata["repository"] != "https://github.com/a/b" {
t.Errorf("unrelated URL rewritten: %v", got.Metadata["repository"])
}
}
func TestRewriteResponseSkipsNonJSON(t *testing.T) {
out, err := (&Provider{}).RewriteResponse([]byte("<html>"), models.Remote{Name: "g"}, "")
if err != nil || out != nil {
t.Errorf("got %q, %v; want nil, nil", out, err)
}
}
func TestForwardsQuery(t *testing.T) {
if _, ok := any(&Provider{}).(provider.QueryForwarder); !ok {
t.Error("ansible provider must forward the query string for pagination")
}
}
-303
View File
@@ -1,303 +0,0 @@
package cargo
import (
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"strings"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
// publishDep is a dependency as cargo sends it in the publish metadata.
type publishDep struct {
Name string `json:"name"`
VersionReq string `json:"version_req"`
Features []string `json:"features"`
Optional bool `json:"optional"`
DefaultFeatures bool `json:"default_features"`
Target *string `json:"target"`
Kind string `json:"kind"`
Registry *string `json:"registry"`
ExplicitNameInToml *string `json:"explicit_name_in_toml"`
}
// PublishMeta is the JSON half of a `PUT /api/v1/crates/new` body; fields the
// index does not carry are ignored.
type PublishMeta struct {
Name string `json:"name"`
Vers string `json:"vers"`
Deps []publishDep `json:"deps"`
Features map[string][]string `json:"features"`
Links *string `json:"links"`
RustVersion *string `json:"rust_version"`
}
type indexDep struct {
Name string `json:"name"`
Req string `json:"req"`
Features []string `json:"features"`
Optional bool `json:"optional"`
DefaultFeatures bool `json:"default_features"`
Target *string `json:"target"`
Kind string `json:"kind"`
Registry *string `json:"registry,omitempty"`
Package *string `json:"package,omitempty"`
}
// IndexLine is one line of a sparse index file.
type IndexLine struct {
Name string `json:"name"`
Vers string `json:"vers"`
Deps []indexDep `json:"deps"`
Cksum string `json:"cksum"`
Features map[string][]string `json:"features"`
Features2 map[string][]string `json:"features2,omitempty"`
Yanked bool `json:"yanked"`
Links *string `json:"links,omitempty"`
V int `json:"v,omitempty"`
RustVersion *string `json:"rust_version,omitempty"`
}
// ParsePublish splits a publish body: u32le JSON length, JSON, u32le crate length, crate.
func ParsePublish(body []byte) (PublishMeta, []byte, error) {
var meta PublishMeta
jsonBytes, rest, err := lengthPrefixed(body)
if err != nil {
return meta, nil, fmt.Errorf("metadata: %w", err)
}
crate, rest, err := lengthPrefixed(rest)
if err != nil {
return meta, nil, fmt.Errorf("crate: %w", err)
}
if len(rest) != 0 {
return meta, nil, errors.New("trailing bytes after crate")
}
if err := json.Unmarshal(jsonBytes, &meta); err != nil {
return meta, nil, fmt.Errorf("metadata: %w", err)
}
return meta, crate, nil
}
func lengthPrefixed(b []byte) ([]byte, []byte, error) {
if len(b) < 4 {
return nil, nil, errors.New("truncated length")
}
n := binary.LittleEndian.Uint32(b)
b = b[4:]
if uint64(n) > uint64(len(b)) {
return nil, nil, errors.New("truncated body")
}
return b[:n], b[n:], nil
}
// NewIndexLine validates the metadata and translates it into index form.
func NewIndexLine(meta PublishMeta, crate []byte) (IndexLine, error) {
if !validName(meta.Name) {
return IndexLine{}, fmt.Errorf("invalid crate name %q", meta.Name)
}
if !validVersion(meta.Vers) {
return IndexLine{}, fmt.Errorf("invalid version %q", meta.Vers)
}
sum := sha256.Sum256(crate)
line := IndexLine{
Name: meta.Name,
Vers: meta.Vers,
Deps: make([]indexDep, 0, len(meta.Deps)),
Cksum: hex.EncodeToString(sum[:]),
Features: map[string][]string{},
Links: meta.Links,
RustVersion: meta.RustVersion,
}
for _, d := range meta.Deps {
dep := indexDep{
Name: d.Name,
Req: d.VersionReq,
Features: d.Features,
Optional: d.Optional,
DefaultFeatures: d.DefaultFeatures,
Target: d.Target,
Kind: d.Kind,
Registry: d.Registry,
}
if dep.Features == nil {
dep.Features = []string{}
}
if dep.Kind == "" {
dep.Kind = "normal"
}
if d.ExplicitNameInToml != nil && *d.ExplicitNameInToml != "" {
pkg := d.Name
dep.Name, dep.Package = *d.ExplicitNameInToml, &pkg
}
line.Deps = append(line.Deps, dep)
}
// Older cargo cannot parse dep:/?/ feature syntax, so those features live in
// features2 behind v:2, as on crates.io.
for name, vals := range meta.Features {
if vals == nil {
vals = []string{}
}
if usesNewFeatureSyntax(vals) {
if line.Features2 == nil {
line.Features2 = map[string][]string{}
}
line.Features2[name] = vals
continue
}
line.Features[name] = vals
}
if line.Features2 != nil {
line.V = 2
}
return line, nil
}
func usesNewFeatureSyntax(vals []string) bool {
for _, v := range vals {
if strings.HasPrefix(v, "dep:") || strings.Contains(v, "?/") {
return true
}
}
return false
}
func validName(name string) bool {
if name == "" || len(name) > 64 {
return false
}
for i, c := range name {
alpha := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
if i == 0 && !alpha {
return false
}
if !alpha && (c < '0' || c > '9') && c != '-' && c != '_' {
return false
}
}
return true
}
func validVersion(v string) bool {
if v == "" || len(v) > 128 || !strings.ContainsAny(v[:1], "0123456789") {
return false
}
return strings.Trim(v, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-+") == ""
}
// IndexPath is the sparse-index file path for a crate name.
func IndexPath(name string) string {
n := strings.ToLower(name)
switch len(n) {
case 0:
return ""
case 1:
return "1/" + n
case 2:
return "2/" + n
case 3:
return "3/" + n[:1] + "/" + n
}
return n[:2] + "/" + n[2:4] + "/" + n
}
// CratePath is where a published .crate is stored, matching config.json's dl.
func CratePath(name, version string) string {
return crateDir + name + "/" + name + "-" + version + ".crate"
}
// RenderIndex joins stored lines into an index file with each yanked flag applied.
func RenderIndex(entries []provider.CargoIndexEntry) ([]byte, error) {
var b strings.Builder
for _, e := range entries {
var line IndexLine
if err := json.Unmarshal([]byte(e.Line), &line); err != nil {
return nil, fmt.Errorf("%s %s: %w", e.Name, e.Version, err)
}
line.Yanked = e.Yanked
raw, err := json.Marshal(line)
if err != nil {
return nil, err
}
b.Write(raw)
b.WriteByte('\n')
}
return []byte(b.String()), nil
}
// LocalConfig is config.json for a local registry rooted at base.
func LocalConfig(base string) map[string]string {
return map[string]string{
"dl": base + "/" + crateDir + "{crate}/{crate}-{version}.crate",
"api": base,
}
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
path = strings.TrimLeft(path, "/")
if path == "config.json" {
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(LocalConfig(requestBase(r) + "/api/v1/local/" + repoName))
return true
}
name := path[strings.LastIndex(path, "/")+1:]
if name == "" || IndexPath(name) != path {
return false
}
reader, ok := files.(provider.CargoIndexReader)
if !ok {
http.Error(w, "cargo index not available", http.StatusInternalServerError)
return true
}
entries, err := reader.ListCargoIndexEntries(r.Context(), repoName, name)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
if len(entries) == 0 {
http.Error(w, "not found", http.StatusNotFound)
return true
}
body, err := RenderIndex(entries)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(body)
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
return nil, fmt.Errorf("cargo local index generation for virtual repos not supported")
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
deleter, ok := db.(provider.CargoIndexDeleter)
if !ok {
return nil
}
if err := deleter.DeleteCargoIndexEntry(ctx, repoName, storagePath); err != nil {
slog.Error("cargo index: delete failed", "repo", repoName, "path", storagePath, "error", err)
return err
}
return nil
}
func requestBase(r *http.Request) string {
scheme := "http"
if r.TLS != nil {
scheme = "https"
} else if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
scheme = fwd
}
return scheme + "://" + r.Host
}
-218
View File
@@ -1,218 +0,0 @@
package cargo
import (
"context"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"encoding/json"
"net/http/httptest"
"strings"
"testing"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func publishBody(meta string, crate []byte) []byte {
var b []byte
b = binary.LittleEndian.AppendUint32(b, uint32(len(meta)))
b = append(b, meta...)
b = binary.LittleEndian.AppendUint32(b, uint32(len(crate)))
return append(b, crate...)
}
const sampleMeta = `{
"name": "unkin-hello", "vers": "0.2.0",
"deps": [
{"name": "serde", "version_req": "^1", "features": ["derive"], "optional": false,
"default_features": true, "target": null, "kind": "normal",
"registry": "https://github.com/rust-lang/crates.io-index"},
{"name": "rand_core", "version_req": "^0.6", "features": null, "optional": true,
"default_features": false, "target": "cfg(unix)", "kind": "dev",
"explicit_name_in_toml": "rc"}
],
"features": {"default": ["std"], "std": [], "extra": ["dep:rc"], "weak": ["serde?/std"]},
"links": null, "rust_version": "1.70", "description": "ignored"
}`
func TestParsePublishAndIndexLine(t *testing.T) {
crate := []byte("fake crate bytes")
meta, gotCrate, err := ParsePublish(publishBody(sampleMeta, crate))
if err != nil {
t.Fatal(err)
}
if string(gotCrate) != string(crate) {
t.Fatalf("crate = %q", gotCrate)
}
line, err := NewIndexLine(meta, crate)
if err != nil {
t.Fatal(err)
}
sum := sha256.Sum256(crate)
if line.Cksum != hex.EncodeToString(sum[:]) {
t.Errorf("cksum = %s", line.Cksum)
}
raw, _ := json.Marshal(line)
var got map[string]any
_ = json.Unmarshal(raw, &got)
if _, ok := got["version_req"]; ok {
t.Error("publish-only field leaked into index line")
}
if got["v"] != float64(2) {
t.Errorf("v = %v, want 2 with features2 present", got["v"])
}
if got["rust_version"] != "1.70" {
t.Errorf("rust_version = %v", got["rust_version"])
}
if _, ok := line.Features["extra"]; ok {
t.Error("dep: feature must move to features2")
}
if len(line.Features2) != 2 || len(line.Features) != 2 {
t.Errorf("features = %v features2 = %v", line.Features, line.Features2)
}
deps := got["deps"].([]any)
serde := deps[0].(map[string]any)
if serde["req"] != "^1" || serde["registry"] != "https://github.com/rust-lang/crates.io-index" || serde["target"] != nil {
t.Errorf("serde dep = %v", serde)
}
if _, ok := serde["package"]; ok {
t.Error("package set without a rename")
}
renamed := deps[1].(map[string]any)
if renamed["name"] != "rc" || renamed["package"] != "rand_core" || renamed["kind"] != "dev" {
t.Errorf("renamed dep = %v", renamed)
}
if f, ok := renamed["features"].([]any); !ok || len(f) != 0 {
t.Errorf("null features must render as [], got %v", renamed["features"])
}
}
func TestIndexLineNoNewFeaturesOmitsV(t *testing.T) {
line, err := NewIndexLine(PublishMeta{Name: "a", Vers: "1.0.0", Features: map[string][]string{"x": nil}}, nil)
if err != nil {
t.Fatal(err)
}
raw, _ := json.Marshal(line)
s := string(raw)
for _, bad := range []string{`"v":`, `"features2"`, `"links"`} {
if strings.Contains(s, bad) {
t.Errorf("%s present in %s", bad, s)
}
}
if !strings.Contains(s, `"deps":[]`) || !strings.Contains(s, `"features":{"x":[]}`) {
t.Errorf("deps/features must be non-null: %s", s)
}
}
func TestParsePublishRejectsMalformed(t *testing.T) {
good := publishBody(`{"name":"a","vers":"1.0.0"}`, []byte("c"))
for name, body := range map[string][]byte{
"empty": nil,
"short": good[:len(good)-1],
"trailing": append(append([]byte{}, good...), 'x'),
"bad json": publishBody(`{`, []byte("c")),
"no length": good[:6],
} {
if _, _, err := ParsePublish(body); err == nil {
t.Errorf("%s: expected error", name)
}
}
}
func TestNewIndexLineValidation(t *testing.T) {
for _, m := range []PublishMeta{
{Name: "", Vers: "1.0.0"},
{Name: "1abc", Vers: "1.0.0"},
{Name: "../x", Vers: "1.0.0"},
{Name: "ok", Vers: "1.0.0/../../x"},
{Name: "ok", Vers: ""},
{Name: strings.Repeat("a", 65), Vers: "1.0.0"},
} {
if _, err := NewIndexLine(m, nil); err == nil {
t.Errorf("%q@%q accepted", m.Name, m.Vers)
}
}
if _, err := NewIndexLine(PublishMeta{Name: "Foo_bar-2", Vers: "1.0.0-rc.1+build.5"}, nil); err != nil {
t.Error(err)
}
}
func TestIndexPath(t *testing.T) {
for name, want := range map[string]string{
"a": "1/a", "ab": "2/ab", "abc": "3/a/abc", "Serde": "se/rd/serde", "unkin-hello": "un/ki/unkin-hello",
} {
if got := IndexPath(name); got != want {
t.Errorf("IndexPath(%q) = %q, want %q", name, got, want)
}
}
}
type fakeIndex struct {
entries []provider.CargoIndexEntry
asked string
}
func (f *fakeIndex) ListCargoIndexEntries(_ context.Context, _, name string) ([]provider.CargoIndexEntry, error) {
f.asked = name
return f.entries, nil
}
func (f *fakeIndex) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (f *fakeIndex) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestServeLocalIndex(t *testing.T) {
line, _ := NewIndexLine(PublishMeta{Name: "unkin-hello", Vers: "0.1.0"}, []byte("x"))
raw, _ := json.Marshal(line)
files := &fakeIndex{entries: []provider.CargoIndexEntry{
{Name: "unkin-hello", Version: "0.1.0", Line: string(raw), Yanked: true},
{Name: "unkin-hello", Version: "0.2.0", Line: strings.Replace(string(raw), "0.1.0", "0.2.0", 1)},
}}
p := &Provider{}
rec := httptest.NewRecorder()
if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "http://aa.example/x", nil), files, "crates-local", "un/ki/unkin-hello") {
t.Fatal("index path not served")
}
lines := strings.Split(strings.TrimSpace(rec.Body.String()), "\n")
if len(lines) != 2 || !strings.Contains(lines[0], `"yanked":true`) || !strings.Contains(lines[1], `"yanked":false`) {
t.Errorf("index body = %q", rec.Body.String())
}
if files.asked != "unkin-hello" {
t.Errorf("looked up %q", files.asked)
}
rec = httptest.NewRecorder()
req := httptest.NewRequest("GET", "http://aa.example/config.json", nil)
req.Header.Set("X-Forwarded-Proto", "https")
p.ServeLocalIndex(rec, req, files, "crates-local", "config.json")
var cfg map[string]string
_ = json.Unmarshal(rec.Body.Bytes(), &cfg)
if cfg["api"] != "https://aa.example/api/v1/local/crates-local" ||
cfg["dl"] != "https://aa.example/api/v1/local/crates-local/crates/{crate}/{crate}-{version}.crate" {
t.Errorf("config = %v", cfg)
}
for _, path := range []string{"crates/unkin-hello/unkin-hello-0.1.0.crate", "xx/ki/unkin-hello", ""} {
if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), files, "r", path) {
t.Errorf("%q should fall through", path)
}
}
files.entries = nil
rec = httptest.NewRecorder()
p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), files, "r", "3/a/abc")
if rec.Code != 404 {
t.Errorf("unknown crate code = %d", rec.Code)
}
}
func TestCratePathMatchesDL(t *testing.T) {
dl := LocalConfig("http://h/api/v1/local/r")["dl"]
want := strings.NewReplacer("{crate}", "unkin-hello", "{version}", "0.1.0").Replace(dl)
if got := "http://h/api/v1/local/r/" + CratePath("unkin-hello", "0.1.0"); got != want {
t.Errorf("CratePath = %q, dl expands to %q", got, want)
}
}
-24
View File
@@ -1,24 +0,0 @@
package provider
import "context"
// CargoIndexEntry is one published crate version in a local cargo registry.
// Line is the sparse-index JSON line; Yanked is applied when it is served.
type CargoIndexEntry struct {
RepoName string
FilePath string
Name string
Version string
Line string
Yanked bool
}
// CargoIndexReader lists every version of one crate (matched case-insensitively).
// *database.DB satisfies it and CargoIndexDeleter.
type CargoIndexReader interface {
ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]CargoIndexEntry, error)
}
type CargoIndexDeleter interface {
DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error
}
+7
View File
@@ -65,6 +65,13 @@ type MetadataStore interface {
InsertDebMetadata(ctx context.Context, meta *DebMetadata) error
}
// QueryForwarder marks a remote provider whose upstream answers depend on the
// query string (e.g. paginated JSON APIs); the query is kept as part of the
// proxied and cached path.
type QueryForwarder interface {
ForwardQuery()
}
// RemoteServer lets a remote provider fully answer a request itself instead of
// going through the byte-proxy engine. It is the remote-side analog of
// LocalIndexer: a metadata-only remote (e.g. github_rpm) uses it to synthesize
+1
View File
@@ -22,6 +22,7 @@ import (
"git.unkin.net/unkin/artifactapi/internal/gc"
"git.unkin.net/unkin/artifactapi/internal/githubauth"
"git.unkin.net/unkin/artifactapi/internal/provider/alpine"
_ "git.unkin.net/unkin/artifactapi/internal/provider/ansible"
_ "git.unkin.net/unkin/artifactapi/internal/provider/cargo"
"git.unkin.net/unkin/artifactapi/internal/provider/deb"
_ "git.unkin.net/unkin/artifactapi/internal/provider/docker"
-15
View File
@@ -1,15 +0,0 @@
-- Local cargo registries: one sparse-index line per published crate version.
-- line holds the index JSON minus yanked, which flips on yank/unyank.
CREATE TABLE IF NOT EXISTS cargo_index (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
line TEXT NOT NULL,
yanked BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE INDEX IF NOT EXISTS idx_cargo_index_repo_name ON cargo_index(repo_name, lower(name));
+2
View File
@@ -17,6 +17,7 @@ const (
PackageTerraform PackageType = "terraform"
PackageGoProxy PackageType = "goproxy"
PackageCargo PackageType = "cargo"
PackageAnsible PackageType = "ansible"
PackageGitHubRPM PackageType = "github_rpm"
PackageGitHubDeb PackageType = "github_deb"
PackageGitHubAlpine PackageType = "github_alpine"
@@ -35,6 +36,7 @@ var validPackageTypes = map[PackageType]bool{
PackageTerraform: true,
PackageGoProxy: true,
PackageCargo: true,
PackageAnsible: true,
PackageGitHubRPM: true,
PackageGitHubDeb: true,
PackageGitHubAlpine: true,
+1
View File
@@ -19,6 +19,7 @@ func TestPackageTypeValid(t *testing.T) {
models.PackageTerraform,
models.PackageGoProxy,
models.PackageCargo,
models.PackageAnsible,
models.PackageGitHubRPM,
models.PackageGitHubDeb,
models.PackageGitHubAlpine,
+9 -19
View File
@@ -229,25 +229,6 @@ go mod download`,
];
case 'cargo':
if (isLocal) {
return [
{
title: 'Add the private registry (real sparse registry)',
language: 'toml',
code: `# $CARGO_HOME/config.toml
[registries.${name}]
index = "sparse+${url}/api/v1/local/${name}/"`,
note: 'cargo requires a token for publish/yank: set CARGO_REGISTRIES_' + name.toUpperCase().replace(/-/g, '_') + '_TOKEN to any value.',
},
{
title: 'Publish, depend on and yank crates',
language: 'bash',
code: `cargo publish --registry ${name}
cargo add <crate> --registry ${name}
cargo yank --version <version> --registry ${name}`,
},
];
}
return [
{
title: 'Point crates.io at this sparse registry',
@@ -262,6 +243,15 @@ index = "sparse+${proxy}/"`,
},
];
case 'ansible':
return [
{
title: 'Install collections through this Galaxy proxy',
language: 'bash',
code: `ansible-galaxy collection install community.general -s ${proxy}/api/`,
},
];
case 'puppet':
return [
{
+1
View File
@@ -18,6 +18,7 @@ const typeColors: Record<string, 'blue' | 'green' | 'yellow' | 'red' | 'default'
terraform: 'blue',
goproxy: 'green',
cargo: 'yellow',
ansible: 'red',
};
export function Remotes() {