Treat a clean EOF with unknown total as the whole asset

Wrap package parse errors in provider.ErrInvalidPackage so only transient
asset and insert errors fail the scan; cover rpm, deb and alpine.

README.md

@@ -279,20 +279,6 @@ content-addressable store (deduplicated by digest, reaped by GC once
 unreferenced); tags are mutable references and re-pushing a tag moves it. Blob
 uploads support both the monolithic and chunked (`POST`/`PATCH`/`PUT`) flows.
 
-### Local cargo registry
-
-A local `cargo` repo is a private crate registry: it serves the sparse index and
-the Web API publish, yank and unyank endpoints, so `cargo publish` works against
-it directly.
-
-```toml
-[registries.<repo>]
-index = "sparse+https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>/"
-```
-
-cargo requires a token for publish/yank (`CARGO_REGISTRIES_<REPO>_TOKEN`); like
-other local repos, artifactapi does not check it.
-
 ## Access Control
 
 | Field | Default | Behaviour |
@@ -385,8 +371,6 @@ additive and idempotent for the same reason; a test enforces it.
 | `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
 | `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
 | `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
-| `ARTIFACTAPI_OIDC_ISSUER` | | OIDC issuer URL (e.g. Vault `https://vault.service.consul:8200/v1/identity/oidc`). When set, `POST /api/v2/remotes`, `POST /api/v2/virtuals` and `PUT`/`DELETE` on `/api/v2/{remotes,virtuals}/{name}` require `Authorization: Bearer <jwt>` verified against the issuer's JWKS. Uploads, object/file deletes, probe, reads and `/v2` stay open. Unset = no auth |
-| `ARTIFACTAPI_OIDC_AUDIENCE` | | Comma-separated allowed `aud` values (e.g. the Vault OIDC role's client_id). Required when `ARTIFACTAPI_OIDC_ISSUER` is set |
 
 ## Development
 

e2e-docker/README.md

@@ -27,8 +27,7 @@ already-running stack.
   mock upstream: first fetch `X-Artifact-Source: remote`, second `cache`, bytes
   verified against the origin fixture.
 - **Local uploads** — generic (upload/download), pypi (wheel + generated `simple/`
-  index), rpm (real package + **automatic repodata** generation), cargo (publish,
-  sparse index, download, yank).
+  index), rpm (real package + **automatic repodata** generation).
 - **Virtual repositories** — pypi simple-index merge and helm `index.yaml` merge
   across two members.
 - **Mirrorlist** — an rpm remote with a `mirrorlist` of extra upstream mirrors

e2e-docker/cargo_local_test.go

@@ -1,56 +0,0 @@
-//go:build dockere2e
-
-package e2edocker
-
-import (
-	"bytes"
-	"encoding/binary"
-	"net/http"
-	"strings"
-	"testing"
-)
-
-// TestLocalCargoRegistry publishes through the cargo Web API, then reads the
-// sparse index, downloads the crate and yanks it.
-func TestLocalCargoRegistry(t *testing.T) {
-	createRepo(t, `{"name":"local-cargo","package_type":"cargo","repo_type":"local"}`)
-	defer deleteRepo(t, "local-cargo")
-	base := api("/api/v1/local/local-cargo")
-
-	meta := `{"name":"e2e-crate","vers":"0.1.0","deps":[],"features":{}}`
-	crate := []byte("e2e crate bytes")
-	var body []byte
-	body = binary.LittleEndian.AppendUint32(body, uint32(len(meta)))
-	body = append(body, meta...)
-	body = binary.LittleEndian.AppendUint32(body, uint32(len(crate)))
-	body = append(body, crate...)
-
-	if resp, b := doRequest(t, http.MethodPut, base+"/api/v1/crates/new", body, "application/octet-stream"); resp.StatusCode != http.StatusOK {
-		t.Fatalf("publish: status %d: %s", resp.StatusCode, b)
-	}
-	if resp, _ := doRequest(t, http.MethodPut, base+"/api/v1/crates/new", body, "application/octet-stream"); resp.StatusCode != http.StatusConflict {
-		t.Fatalf("republish: status %d, want 409", resp.StatusCode)
-	}
-
-	resp, b := doRequest(t, http.MethodGet, base+"/config.json", nil, "")
-	if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"api":"`) {
-		t.Fatalf("config.json: status %d: %s", resp.StatusCode, b)
-	}
-
-	resp, b = doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, "")
-	if resp.StatusCode != http.StatusOK || !strings.Contains(string(b), `"cksum":"`) || !strings.Contains(string(b), `"yanked":false`) {
-		t.Fatalf("index: status %d: %s", resp.StatusCode, b)
-	}
-
-	resp, b = doRequest(t, http.MethodGet, base+"/crates/e2e-crate/e2e-crate-0.1.0.crate", nil, "")
-	if resp.StatusCode != http.StatusOK || !bytes.Equal(b, crate) {
-		t.Fatalf("download: status %d", resp.StatusCode)
-	}
-
-	if resp, b := doRequest(t, http.MethodDelete, base+"/api/v1/crates/e2e-crate/0.1.0/yank", nil, ""); resp.StatusCode != http.StatusOK {
-		t.Fatalf("yank: status %d: %s", resp.StatusCode, b)
-	}
-	if _, b := doRequest(t, http.MethodGet, base+"/e2/e-/e2e-crate", nil, ""); !strings.Contains(string(b), `"yanked":true`) {
-		t.Fatalf("index after yank: %s", b)
-	}
-}

go.mod

@@ -7,9 +7,7 @@ require (
 	github.com/cavaliergopher/rpm v1.3.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
-	github.com/coreos/go-oidc/v3 v3.21.0
 	github.com/go-chi/chi/v5 v5.3.0
-	github.com/go-jose/go-jose/v4 v4.1.4
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/klauspost/compress v1.19.2
@@ -104,7 +102,6 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/net v0.56.0 // indirect
-	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/text v0.40.0 // indirect
 	gopkg.in/ini.v1 v1.67.2 // indirect

go.sum

@@ -44,8 +44,6 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
 github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
-github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
-github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
 github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
 github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
@@ -69,8 +67,6 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO
 github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
 github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
 github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
-github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
-github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
 github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -229,8 +225,6 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
 golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
 golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
-golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
-golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
 golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/api/v1/cargo.go

@@ -1,148 +0,0 @@
-package v1
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"net/http"
-
-	"github.com/go-chi/chi/v5"
-
-	"git.unkin.net/unkin/artifactapi/internal/database"
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-	"git.unkin.net/unkin/artifactapi/internal/provider/cargo"
-	"git.unkin.net/unkin/artifactapi/pkg/models"
-)
-
-// This file implements the cargo registry Web API (publish, yank, unyank) for
-// local cargo repositories. config.json points api at /api/v1/local/<repo>, so
-// cargo appends its own /api/v1/crates/... below that.
-
-const maxCratePublish = 64 << 20
-
-func (h *ProxyHandler) localCargoRemote(r *http.Request) (*models.Remote, bool) {
-	remote, err := h.db.GetRemote(r.Context(), chi.URLParam(r, "localName"))
-	if err != nil {
-		return nil, false
-	}
-	return remote, remote.RepoType == models.RepoTypeLocal && remote.PackageType == models.PackageCargo
-}
-
-func cargoError(w http.ResponseWriter, status int, format string, args ...any) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(status)
-	_ = json.NewEncoder(w).Encode(map[string]any{
-		"errors": []map[string]string{{"detail": fmt.Sprintf(format, args...)}},
-	})
-}
-
-func cargoOK(w http.ResponseWriter, v any) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(http.StatusOK)
-	_ = json.NewEncoder(w).Encode(v)
-}
-
-func (h *ProxyHandler) cargoPublish(w http.ResponseWriter, r *http.Request) {
-	remote, ok := h.localCargoRemote(r)
-	if !ok {
-		cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName"))
-		return
-	}
-	ctx := r.Context()
-
-	body, err := io.ReadAll(io.LimitReader(r.Body, maxCratePublish+1))
-	if err != nil {
-		cargoError(w, http.StatusBadRequest, "read body: %v", err)
-		return
-	}
-	if len(body) > maxCratePublish {
-		cargoError(w, http.StatusRequestEntityTooLarge, "publish body exceeds %d bytes", maxCratePublish)
-		return
-	}
-	meta, crate, err := cargo.ParsePublish(body)
-	if err != nil {
-		cargoError(w, http.StatusBadRequest, "%v", err)
-		return
-	}
-	line, err := cargo.NewIndexLine(meta, crate)
-	if err != nil {
-		cargoError(w, http.StatusBadRequest, "%v", err)
-		return
-	}
-
-	existing, err := h.db.ListCargoIndexEntries(ctx, remote.Name, meta.Name)
-	if err != nil {
-		cargoError(w, http.StatusInternalServerError, "%v", err)
-		return
-	}
-	for _, e := range existing {
-		if e.Name != meta.Name {
-			cargoError(w, http.StatusBadRequest, "crate %q already exists as %q", meta.Name, e.Name)
-			return
-		}
-		if e.Version == meta.Vers {
-			cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers)
-			return
-		}
-	}
-
-	lineJSON, err := json.Marshal(line)
-	if err != nil {
-		cargoError(w, http.StatusInternalServerError, "%v", err)
-		return
-	}
-
-	path := cargo.CratePath(meta.Name, meta.Vers)
-	result, err := h.cas.Store(ctx, bytes.NewReader(crate), "application/gzip")
-	if err != nil {
-		cargoError(w, http.StatusInternalServerError, "store failed: %v", err)
-		return
-	}
-	if err := h.db.UpsertBlob(ctx, result.ContentHash, result.S3Key, result.SizeBytes, "application/gzip"); err != nil {
-		cargoError(w, http.StatusInternalServerError, "record blob: %v", err)
-		return
-	}
-	if err := h.db.CreateLocalFile(ctx, remote.Name, path, result.ContentHash); err != nil {
-		if errors.Is(err, database.ErrAlreadyExists) {
-			cargoError(w, http.StatusConflict, "crate version `%s@%s` already exists", meta.Name, meta.Vers)
-			return
-		}
-		cargoError(w, http.StatusInternalServerError, "record file: %v", err)
-		return
-	}
-	entry := &provider.CargoIndexEntry{RepoName: remote.Name, FilePath: path, Name: meta.Name, Version: meta.Vers, Line: string(lineJSON)}
-	if err := h.db.InsertCargoIndexEntry(ctx, entry); err != nil {
-		// Drop the file so a retry is not rejected as a duplicate.
-		_ = h.db.DeleteLocalFile(ctx, remote.Name, path)
-		cargoError(w, http.StatusInternalServerError, "record index: %v", err)
-		return
-	}
-
-	slog.Info("cargo: published", "repo", remote.Name, "crate", meta.Name, "version", meta.Vers)
-	cargoOK(w, map[string]any{"warnings": map[string][]string{"invalid_categories": {}, "invalid_badges": {}, "other": {}}})
-}
-
-func (h *ProxyHandler) cargoYank(yanked bool) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		remote, ok := h.localCargoRemote(r)
-		if !ok {
-			cargoError(w, http.StatusNotFound, "no local cargo registry %q", chi.URLParam(r, "localName"))
-			return
-		}
-		name, version := chi.URLParam(r, "crate"), chi.URLParam(r, "version")
-		found, err := h.db.SetCargoYanked(r.Context(), remote.Name, name, version, yanked)
-		if err != nil {
-			cargoError(w, http.StatusInternalServerError, "%v", err)
-			return
-		}
-		if !found {
-			cargoError(w, http.StatusNotFound, "crate version `%s@%s` not found", name, version)
-			return
-		}
-		slog.Info("cargo: yank updated", "repo", remote.Name, "crate", name, "version", version, "yanked", yanked)
-		cargoOK(w, map[string]bool{"ok": true})
-	}
-}

internal/api/v1/proxy.go

@@ -41,9 +41,6 @@ func (h *ProxyHandler) Routes() chi.Router {
 	r := chi.NewRouter()
 	r.Get("/remote/{remoteName}/*", h.handleProxy)
 	r.Get("/local/{localName}/*", h.handleLocal)
-	r.Put("/local/{localName}/api/v1/crates/new", h.cargoPublish)
-	r.Delete("/local/{localName}/api/v1/crates/{crate}/{version}/yank", h.cargoYank(true))
-	r.Put("/local/{localName}/api/v1/crates/{crate}/{version}/unyank", h.cargoYank(false))
 	r.Get("/virtual/{virtualName}/*", h.handleVirtual)
 	return r
 }

internal/auth/oidc.go

@@ -1,158 +0,0 @@
-package auth
-
-import (
-	"context"
-	"errors"
-	"log/slog"
-	"net/http"
-	"slices"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/coreos/go-oidc/v3/oidc"
-)
-
-// BearerOIDC verifies `Authorization: Bearer <jwt>` against an OIDC issuer's
-// JWKS. Discovery is lazy so startup never depends on the issuer being
-// reachable; a failed discovery is not retried until the backoff elapses.
-type BearerOIDC struct {
-	issuer    string
-	audiences []string
-	timeout   time.Duration
-	backoff   time.Duration
-	discover  func(context.Context, string) (*oidc.Provider, error)
-
-	mu       sync.Mutex
-	verifier *oidc.IDTokenVerifier
-	inflight chan struct{}
-	failedAt time.Time
-}
-
-var errIssuerUnavailable = errors.New("oidc issuer unavailable")
-
-// NewBearerOIDC returns nil when issuer is empty, meaning auth is disabled.
-func NewBearerOIDC(issuer string, audiences []string) *BearerOIDC {
-	if issuer == "" {
-		return nil
-	}
-	return &BearerOIDC{issuer: issuer, audiences: audiences, timeout: 10 * time.Second, backoff: 15 * time.Second, discover: oidc.NewProvider}
-}
-
-// getVerifier runs at most one discovery at a time without holding the lock
-// across network I/O; concurrent callers wait for it, and only callers within
-// the post-failure backoff fail fast.
-func (b *BearerOIDC) getVerifier(ctx context.Context) (*oidc.IDTokenVerifier, error) {
-	b.mu.Lock()
-	if v := b.verifier; v != nil {
-		b.mu.Unlock()
-		return v, nil
-	}
-	if done := b.inflight; done != nil {
-		b.mu.Unlock()
-		select {
-		case <-done:
-		case <-ctx.Done():
-			return nil, ctx.Err()
-		}
-		b.mu.Lock()
-		defer b.mu.Unlock()
-		if b.verifier == nil {
-			return nil, errIssuerUnavailable
-		}
-		return b.verifier, nil
-	}
-	if time.Since(b.failedAt) < b.backoff {
-		b.mu.Unlock()
-		return nil, errIssuerUnavailable
-	}
-	done := make(chan struct{})
-	b.inflight = done
-	b.mu.Unlock()
-
-	v, err := b.discoverOnce(done)
-	if err != nil {
-		slog.Error("oidc discovery failed", "issuer", b.issuer, "error", err)
-		return nil, err
-	}
-	return v, nil
-}
-
-// discoverOnce releases inflight and waiters on every exit, a panic included,
-// recording failedAt unless discovery succeeded.
-func (b *BearerOIDC) discoverOnce(done chan struct{}) (v *oidc.IDTokenVerifier, err error) {
-	defer func() {
-		b.mu.Lock()
-		b.inflight = nil
-		if v == nil {
-			b.failedAt = time.Now()
-		}
-		b.verifier = v
-		b.mu.Unlock()
-		close(done)
-	}()
-	// The provider keeps this context and client for JWKS refreshes, so the
-	// deadline lives on the client rather than the context.
-	client := &http.Client{Timeout: b.timeout}
-	p, err := b.discover(oidc.ClientContext(context.Background(), client), b.issuer)
-	if err != nil {
-		return nil, err
-	}
-	return p.Verifier(&oidc.Config{SkipClientIDCheck: true}), nil
-}
-
-// Require gates every request through next on a valid token. A nil receiver
-// passes everything through.
-func (b *BearerOIDC) Require(next http.Handler) http.Handler {
-	if b == nil {
-		return next
-	}
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		scheme, raw, _ := strings.Cut(strings.TrimSpace(r.Header.Get("Authorization")), " ")
-		raw = strings.TrimSpace(raw)
-		if !strings.EqualFold(scheme, "Bearer") || raw == "" {
-			unauthorized(w, "missing bearer token")
-			return
-		}
-		v, err := b.getVerifier(r.Context())
-		if err != nil {
-			w.Header().Set("Content-Type", "application/json")
-			w.WriteHeader(http.StatusServiceUnavailable)
-			_, _ = w.Write([]byte(`{"error":"token issuer unavailable"}`))
-			return
-		}
-		tok, err := v.Verify(r.Context(), raw)
-		if err != nil {
-			slog.Info("bearer token rejected", "error", err)
-			unauthorized(w, "invalid token")
-			return
-		}
-		if len(b.audiences) > 0 && !slices.ContainsFunc(tok.Audience, func(a string) bool { return slices.Contains(b.audiences, a) }) {
-			slog.Info("bearer token rejected", "error", "audience not allowed", "aud", tok.Audience)
-			unauthorized(w, "invalid token")
-			return
-		}
-		slog.Info("bearer token accepted", "sub", tok.Subject, "method", r.Method, "path", r.URL.Path)
-		next.ServeHTTP(w, r)
-	})
-}
-
-// RequireWrites gates only mutating methods.
-func (b *BearerOIDC) RequireWrites(next http.Handler) http.Handler {
-	gated := b.Require(next)
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		switch r.Method {
-		case http.MethodGet, http.MethodHead, http.MethodOptions:
-			next.ServeHTTP(w, r)
-		default:
-			gated.ServeHTTP(w, r)
-		}
-	})
-}
-
-func unauthorized(w http.ResponseWriter, msg string) {
-	w.Header().Set("Content-Type", "application/json")
-	w.Header().Set("WWW-Authenticate", "Bearer")
-	w.WriteHeader(http.StatusUnauthorized)
-	_, _ = w.Write([]byte(`{"error":"` + msg + `"}`))
-}

internal/auth/oidc_test.go

@@ -1,255 +0,0 @@
-package auth
-
-import (
-	"context"
-	"crypto/rand"
-	"crypto/rsa"
-	"encoding/base64"
-	"net/http"
-	"net/http/httptest"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/coreos/go-oidc/v3/oidc"
-	"github.com/go-jose/go-jose/v4"
-	"github.com/go-jose/go-jose/v4/jwt"
-
-	"git.unkin.net/unkin/artifactapi/internal/testsupport"
-)
-
-func serve(h http.Handler, method, token string) int {
-	if token == "" {
-		return serveAuth(h, method, "")
-	}
-	return serveAuth(h, method, "Bearer "+token)
-}
-
-func serveAuth(h http.Handler, method, authz string) int {
-	r := httptest.NewRequest(method, "/x", nil)
-	if authz != "" {
-		r.Header.Set("Authorization", authz)
-	}
-	w := httptest.NewRecorder()
-	h.ServeHTTP(w, r)
-	return w.Code
-}
-
-func claims(iss string) jwt.Claims {
-	now := time.Now()
-	return jwt.Claims{
-		Issuer:   iss,
-		Subject:  "ci",
-		Audience: jwt.Audience{"artifactapi"},
-		IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
-		Expiry:   jwt.NewNumericDate(now.Add(time.Hour)),
-	}
-}
-
-func hs256Token(t *testing.T, c jwt.Claims) string {
-	sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: []byte("0123456789abcdef0123456789abcdef")},
-		(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	tok, err := jwt.Signed(sig).Claims(c).Serialize()
-	if err != nil {
-		t.Fatal(err)
-	}
-	return tok
-}
-
-func noneToken(iss string) string {
-	enc := base64.RawURLEncoding.EncodeToString
-	return enc([]byte(`{"alg":"none","typ":"JWT","kid":"k1"}`)) + "." +
-		enc([]byte(`{"iss":"`+iss+`","sub":"ci","aud":"artifactapi","exp":`+strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)+`}`)) + "."
-}
-
-var ok = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
-
-func TestBearerOIDC(t *testing.T) {
-	iss := testsupport.NewOIDCIssuer()
-	defer iss.Close()
-	h := NewBearerOIDC(iss.URL, []string{"other", "artifactapi"}).Require(ok)
-
-	otherKey, _ := rsa.GenerateKey(rand.Reader, 2048)
-	cases := []struct {
-		name  string
-		token string
-		want  int
-	}{
-		{"valid", iss.Token(nil, "artifactapi", "ci", time.Hour), http.StatusNoContent},
-		{"missing", "", http.StatusUnauthorized},
-		{"garbage", "not.a.jwt", http.StatusUnauthorized},
-		{"bad signature", iss.Token(otherKey, "artifactapi", "ci", time.Hour), http.StatusUnauthorized},
-		{"wrong aud", iss.Token(nil, "nope", "ci", time.Hour), http.StatusUnauthorized},
-		{"expired", iss.Token(nil, "artifactapi", "ci", -time.Hour), http.StatusUnauthorized},
-		{"wrong iss", iss.Sign(nil, claims("https://evil.example")), http.StatusUnauthorized},
-		{"hs256", hs256Token(t, claims(iss.URL)), http.StatusUnauthorized},
-		{"alg none", noneToken(iss.URL), http.StatusUnauthorized},
-	}
-	for _, c := range cases {
-		if got := serve(h, http.MethodPost, c.token); got != c.want {
-			t.Errorf("%s: status %d, want %d", c.name, got, c.want)
-		}
-	}
-
-	valid := iss.Token(nil, "artifactapi", "ci", time.Hour)
-	headers := []struct {
-		name  string
-		authz string
-		want  int
-	}{
-		{"lowercase scheme", "bearer " + valid, http.StatusNoContent},
-		{"extra whitespace", "  Bearer   " + valid + " ", http.StatusNoContent},
-		{"empty value", "Bearer ", http.StatusUnauthorized},
-		{"scheme only", "Bearer", http.StatusUnauthorized},
-		{"basic scheme", "Basic " + valid, http.StatusUnauthorized},
-	}
-	for _, c := range headers {
-		if got := serveAuth(h, http.MethodPost, c.authz); got != c.want {
-			t.Errorf("%s: status %d, want %d", c.name, got, c.want)
-		}
-	}
-}
-
-func TestBearerOIDCHangingIssuerBacksOff(t *testing.T) {
-	var hits atomic.Int32
-	release := make(chan struct{})
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		hits.Add(1)
-		select {
-		case <-r.Context().Done():
-		case <-release:
-		}
-	}))
-	defer srv.Close()
-	defer close(release)
-
-	b := NewBearerOIDC(srv.URL, nil)
-	b.timeout = 100 * time.Millisecond
-	b.backoff = time.Hour
-	h := b.Require(ok)
-
-	start := time.Now()
-	if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
-		t.Fatalf("hanging issuer: status %d, want 503", got)
-	}
-	if d := time.Since(start); d > 2*time.Second {
-		t.Fatalf("hanging issuer: took %s, timeout not applied", d)
-	}
-	if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
-		t.Fatalf("during backoff: status %d, want 503", got)
-	}
-	if n := hits.Load(); n != 1 {
-		t.Fatalf("issuer hit %d times, want 1", n)
-	}
-}
-
-func TestBearerOIDCPanickingDiscoveryBacksOff(t *testing.T) {
-	b := NewBearerOIDC("http://issuer.invalid", nil)
-	b.backoff = time.Hour
-	b.discover = func(context.Context, string) (*oidc.Provider, error) { panic("boom") }
-	h := b.Require(ok)
-
-	func() {
-		defer func() { _ = recover() }()
-		serve(h, http.MethodPost, "x.y.z")
-	}()
-
-	got := make(chan int, 1)
-	go func() { got <- serve(h, http.MethodPost, "x.y.z") }()
-	select {
-	case code := <-got:
-		if code != http.StatusServiceUnavailable {
-			t.Fatalf("after panic: status %d, want 503", code)
-		}
-	case <-time.After(2 * time.Second):
-		t.Fatal("after panic: request hung on stale inflight discovery")
-	}
-}
-
-func TestBearerOIDCConcurrentFirstRequestsWaitForDiscovery(t *testing.T) {
-	iss := testsupport.NewOIDCIssuer()
-	defer iss.Close()
-	var hits atomic.Int32
-	inner := iss.Config.Handler
-	iss.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.URL.Path == "/.well-known/openid-configuration" {
-			hits.Add(1)
-			time.Sleep(200 * time.Millisecond)
-		}
-		inner.ServeHTTP(w, r)
-	})
-
-	h := NewBearerOIDC(iss.URL, nil).Require(ok)
-	tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
-	var wg sync.WaitGroup
-	codes := make([]int, 20)
-	for i := range codes {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			codes[i] = serve(h, http.MethodPost, tok)
-		}()
-	}
-	wg.Wait()
-	for i, c := range codes {
-		if c != http.StatusNoContent {
-			t.Errorf("request %d: status %d, want 204", i, c)
-		}
-	}
-	if n := hits.Load(); n != 1 {
-		t.Fatalf("discovery hit %d times, want 1", n)
-	}
-}
-
-func TestBearerOIDCRetriesAfterBackoff(t *testing.T) {
-	iss := testsupport.NewOIDCIssuer()
-	defer iss.Close()
-	b := NewBearerOIDC(iss.URL, nil)
-	b.failedAt = time.Now()
-	b.backoff = 50 * time.Millisecond
-	tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
-	if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
-		t.Fatalf("within backoff: status %d, want 503", got)
-	}
-	time.Sleep(60 * time.Millisecond)
-	if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusNoContent {
-		t.Fatalf("after backoff: status %d, want 204", got)
-	}
-}
-
-func TestBearerOIDCIssuerDown(t *testing.T) {
-	iss := testsupport.NewOIDCIssuer()
-	b := NewBearerOIDC(iss.URL, []string{"artifactapi"})
-	tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
-	iss.Close()
-	if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
-		t.Errorf("issuer down: status %d, want 503", got)
-	}
-}
-
-func TestBearerOIDCDisabled(t *testing.T) {
-	if got := serve(NewBearerOIDC("", nil).RequireWrites(ok), http.MethodDelete, ""); got != http.StatusNoContent {
-		t.Errorf("disabled: status %d, want 204", got)
-	}
-}
-
-func TestRequireWritesLeavesReadsOpen(t *testing.T) {
-	iss := testsupport.NewOIDCIssuer()
-	defer iss.Close()
-	h := NewBearerOIDC(iss.URL, []string{"artifactapi"}).RequireWrites(ok)
-	for _, m := range []string{http.MethodGet, http.MethodHead, http.MethodOptions} {
-		if got := serve(h, m, ""); got != http.StatusNoContent {
-			t.Errorf("%s: status %d, want 204", m, got)
-		}
-	}
-	for _, m := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
-		if got := serve(h, m, ""); got != http.StatusUnauthorized {
-			t.Errorf("%s: status %d, want 401", m, got)
-		}
-	}
-}

internal/config/env.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"os"
 	"strconv"
-	"strings"
 )
 
 type Config struct {
@@ -57,10 +56,6 @@ type Config struct {
 	GitHubAppInstallationID string
 	GitHubAppPrivateKey     string
 	GitHubAppPrivateKeyPath string
-
-	// Bearer-JWT auth on configuration writes; empty issuer disables it.
-	OIDCIssuer    string
-	OIDCAudiences []string
 }
 
 func (c *Config) DatabaseDSN() string {
@@ -95,17 +90,6 @@ func Load() (*Config, error) {
 		return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
 	}
 
-	oidcIssuer := getenv("ARTIFACTAPI_OIDC_ISSUER", "")
-	var oidcAudiences []string
-	for _, a := range strings.Split(getenv("ARTIFACTAPI_OIDC_AUDIENCE", ""), ",") {
-		if a = strings.TrimSpace(a); a != "" {
-			oidcAudiences = append(oidcAudiences, a)
-		}
-	}
-	if oidcIssuer != "" && len(oidcAudiences) == 0 {
-		return nil, fmt.Errorf("ARTIFACTAPI_OIDC_AUDIENCE is required when ARTIFACTAPI_OIDC_ISSUER is set")
-	}
-
 	cfg := &Config{
 		ListenAddr: getenv("LISTEN_ADDR", ":8000"),
 
@@ -139,9 +123,6 @@ func Load() (*Config, error) {
 		GitHubAppInstallationID: getenv("GITHUB_APP_INSTALLATION_ID", ""),
 		GitHubAppPrivateKey:     getenv("GITHUB_APP_PRIVATE_KEY", ""),
 		GitHubAppPrivateKeyPath: getenv("GITHUB_APP_PRIVATE_KEY_PATH", ""),
-
-		OIDCIssuer:    oidcIssuer,
-		OIDCAudiences: oidcAudiences,
 	}
 
 	return cfg, nil

internal/config/env_test.go

@@ -64,20 +64,3 @@ func TestLoadInvalidPort(t *testing.T) {
 		t.Error("expected error for invalid DBPORT")
 	}
 }
-
-func TestLoadOIDC(t *testing.T) {
-	t.Setenv("ARTIFACTAPI_OIDC_ISSUER", "https://vault.example/v1/identity/oidc")
-	t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", " a, ,b ")
-	cfg, err := Load()
-	if err != nil {
-		t.Fatalf("load: %v", err)
-	}
-	if len(cfg.OIDCAudiences) != 2 || cfg.OIDCAudiences[0] != "a" || cfg.OIDCAudiences[1] != "b" {
-		t.Errorf("audiences = %q", cfg.OIDCAudiences)
-	}
-
-	t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", "")
-	if _, err := Load(); err == nil {
-		t.Error("expected error for issuer without audience")
-	}
-}

internal/database/cargo_index.go

@@ -1,56 +0,0 @@
-package database
-
-import (
-	"context"
-	"strings"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-)
-
-func (db *DB) InsertCargoIndexEntry(ctx context.Context, e *provider.CargoIndexEntry) error {
-	_, err := db.Pool.Exec(ctx, `
-		INSERT INTO cargo_index (repo_name, file_path, name, version, line, yanked)
-		VALUES ($1, $2, $3, $4, $5, $6)
-	`, e.RepoName, e.FilePath, e.Name, e.Version, e.Line, e.Yanked)
-	return err
-}
-
-func (db *DB) ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]provider.CargoIndexEntry, error) {
-	rows, err := db.Pool.Query(ctx, `
-		SELECT repo_name, file_path, name, version, line, yanked
-		FROM cargo_index
-		WHERE repo_name = $1 AND lower(name) = $2
-		ORDER BY id
-	`, repoName, strings.ToLower(name))
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	var result []provider.CargoIndexEntry
-	for rows.Next() {
-		var e provider.CargoIndexEntry
-		if err := rows.Scan(&e.RepoName, &e.FilePath, &e.Name, &e.Version, &e.Line, &e.Yanked); err != nil {
-			return nil, err
-		}
-		result = append(result, e)
-	}
-	return result, rows.Err()
-}
-
-// SetCargoYanked reports false when no such crate version exists.
-func (db *DB) SetCargoYanked(ctx context.Context, repoName, name, version string, yanked bool) (bool, error) {
-	tag, err := db.Pool.Exec(ctx, `
-		UPDATE cargo_index SET yanked = $4
-		WHERE repo_name = $1 AND lower(name) = $2 AND version = $3
-	`, repoName, strings.ToLower(name), version, yanked)
-	if err != nil {
-		return false, err
-	}
-	return tag.RowsAffected() > 0, nil
-}
-
-func (db *DB) DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error {
-	_, err := db.Pool.Exec(ctx, `DELETE FROM cargo_index WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
-	return err
-}

internal/database/database_test.go

@@ -425,38 +425,3 @@ func TestRPMMetadata(t *testing.T) {
 		t.Fatalf("list rpm rows: %v %v", len(rows), err)
 	}
 }
-
-func TestCargoIndex(t *testing.T) {
-	requireDB(t)
-	for _, v := range []string{"0.1.0", "0.2.0"} {
-		if err := testDB.InsertCargoIndexEntry(ctx(), &provider.CargoIndexEntry{
-			RepoName: "cargo-db", FilePath: "crates/Unkin-Hello/Unkin-Hello-" + v + ".crate",
-			Name: "Unkin-Hello", Version: v, Line: `{"vers":"` + v + `"}`,
-		}); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	found, err := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "0.1.0", true)
-	if err != nil || !found {
-		t.Fatalf("yank: found=%v err=%v", found, err)
-	}
-	if found, _ := testDB.SetCargoYanked(ctx(), "cargo-db", "unkin-hello", "9.9.9", true); found {
-		t.Error("yank of a missing version reported found")
-	}
-
-	entries, err := testDB.ListCargoIndexEntries(ctx(), "cargo-db", "unkin-hello")
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(entries) != 2 || entries[0].Version != "0.1.0" || !entries[0].Yanked || entries[1].Yanked {
-		t.Fatalf("entries = %+v", entries)
-	}
-
-	if err := testDB.DeleteCargoIndexEntry(ctx(), "cargo-db", entries[0].FilePath); err != nil {
-		t.Fatal(err)
-	}
-	if entries, _ = testDB.ListCargoIndexEntries(ctx(), "cargo-db", "UNKIN-HELLO"); len(entries) != 1 {
-		t.Fatalf("after delete: %+v", entries)
-	}
-}

internal/database/github_sync_test.go

@@ -207,3 +207,72 @@ func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
 		t.Fatal("claimed inside freshness window after a successful sync")
 	}
 }
+
+// Deleting a remote drops its sync state, so a recreated remote of the same name
+// starts clean instead of inheriting the old ETag, sync time and retry state.
+func TestGitHubSyncStateDeletedWithRemote(t *testing.T) {
+	requireDB(t)
+	for _, tc := range []struct {
+		table   string
+		pt      models.PackageType
+		claim   func(name string) (bool, string, error)
+		release func(name string, res provider.SyncResult) error
+	}{
+		{"github_rpm_sync_state", models.PackageGitHubRPM,
+			func(n string) (bool, string, error) {
+				return testDB.ClaimGitHubSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+			},
+			func(n string, res provider.SyncResult) error {
+				return testDB.ReleaseGitHubSyncLease(ctx(), n, "r1", res)
+			}},
+		{"github_deb_sync_state", models.PackageGitHubDeb,
+			func(n string) (bool, string, error) {
+				return testDB.ClaimGitHubDebSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+			},
+			func(n string, res provider.SyncResult) error {
+				return testDB.ReleaseGitHubDebSyncLease(ctx(), n, "r1", res)
+			}},
+		{"github_alpine_sync_state", models.PackageGitHubAlpine,
+			func(n string) (bool, string, error) {
+				return testDB.ClaimGitHubAlpineSyncLease(ctx(), n, "r1", time.Hour, time.Minute)
+			},
+			func(n string, res provider.SyncResult) error {
+				return testDB.ReleaseGitHubAlpineSyncLease(ctx(), n, "r1", res)
+			}},
+	} {
+		t.Run(tc.table, func(t *testing.T) {
+			name := "gh-recreate-" + time.Now().Format("150405.000000")
+			create := func() {
+				t.Helper()
+				if err := testDB.CreateRemote(ctx(), &models.Remote{
+					Name: name, PackageType: tc.pt, RepoType: models.RepoTypeRemote,
+					BaseURL: "https://api.github.com/repos/acme/tools", ReleasesRemote: "github", MutableTTL: 3600,
+				}); err != nil {
+					t.Fatalf("create remote: %v", err)
+				}
+			}
+
+			create()
+			if ok, _, err := tc.claim(name); err != nil || !ok {
+				t.Fatalf("claim: ok=%v err=%v", ok, err)
+			}
+			if err := tc.release(name, provider.SyncResult{Etag: `"old"`}); err != nil {
+				t.Fatalf("release: %v", err)
+			}
+
+			if err := testDB.DeleteRemote(ctx(), name); err != nil {
+				t.Fatalf("delete remote: %v", err)
+			}
+			var rows int
+			if err := testDB.Pool.QueryRow(ctx(), `SELECT count(*) FROM `+tc.table+` WHERE remote_name = $1`, name).Scan(&rows); err != nil || rows != 0 {
+				t.Fatalf("sync state left after delete: rows=%d err=%v", rows, err)
+			}
+
+			create()
+			ok, etag, err := tc.claim(name)
+			if err != nil || !ok || etag != "" {
+				t.Fatalf("recreated remote claim ok=%v etag=%q err=%v, want a fresh claim with no etag", ok, etag, err)
+			}
+		})
+	}
+}

internal/provider/alpine/github.go

@@ -372,6 +372,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
+	// A transiently failed asset fails the scan so the ETag is not advanced and
+	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
+	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -400,10 +403,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_alpine: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				if !errors.Is(err, provider.ErrInvalidPackage) {
+					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
+				}
 				continue
 			}
 			if err := inserter.InsertAlpineMetadata(ctx, meta); err != nil {
 				slog.Error("github_alpine: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_alpine: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -415,7 +422,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = deleter.DeleteAlpineMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, nil
+	return newEtag, true, errors.Join(failed...)
 }
 
 type ghRelease struct {
@@ -496,7 +503,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
 		return nil, err
 	}
 	if meta.Name == "" || meta.Arch == "" {
-		return nil, errors.New(".PKGINFO missing pkgname/arch")
+		return nil, fmt.Errorf("%w: .PKGINFO missing pkgname/arch", provider.ErrInvalidPackage)
 	}
 
 	meta.RepoName = remote.Name
@@ -526,13 +533,13 @@ func (p *GitHubProvider) fetchPkginfo(ctx context.Context, remote models.Remote,
 		}
 		meta, complete, perr := pkginfoFromPrefix(body)
 		if perr != nil {
-			return nil, fmt.Errorf("parse apk .PKGINFO: %w", perr)
+			return nil, fmt.Errorf("%w: parse apk .PKGINFO: %w", provider.ErrInvalidPackage, perr)
 		}
 		if complete {
 			return meta, nil
 		}
 		if full || n >= p.headerMax {
-			return nil, fmt.Errorf(".PKGINFO not found within %d bytes of %s", n, downloadURL)
+			return nil, fmt.Errorf("%w: .PKGINFO not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
 		}
 		n *= 2
 		if n > p.headerMax {
@@ -594,7 +601,7 @@ func pkginfoFromPrefix(prefix []byte) (meta *provider.AlpineMetadata, complete b
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response body was shorter than n (i.e. we already have the whole object).
+// response holds the whole object.
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -616,15 +623,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
+		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	full := int64(len(body)) < n
-	return body, full, nil
+	return provider.ReadPrefix(resp, n)
 }
 
 // assetPath is the package's location relative to github.com — the path the

internal/provider/alpine/github_test.go

@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -88,15 +89,17 @@ type githubFixture struct {
 	notModHit   int
 	releaseAuth string
 	assetAuth   string
+	assetFail   map[string]int // asset filename -> downloads left to answer 500
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		apkBytes: map[string][]byte{},
-		rangeHit: map[string]int{},
-		fullHit:  map[string]int{},
+		apkBytes:  map[string][]byte{},
+		rangeHit:  map[string]int{},
+		fullHit:   map[string]int{},
+		assetFail: map[string]int{},
 	}
 	f.apkBytes["demo-1.2.3-r0.apk"] = testsupport.MinimalApk("demo", "1.2.3-r0", "x86_64")
 
@@ -146,6 +149,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
+		if f.assetFail[name] > 0 {
+			f.assetFail[name]--
+			f.mu.Unlock()
+			http.Error(w, "boom", http.StatusInternalServerError)
+			return
+		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -495,3 +504,13 @@ func readAPKIndex(t *testing.T, gzBytes []byte) string {
 		}
 	}
 }
+
+func TestGitHubDeriveAssetMissingPkgnameIsInvalid(t *testing.T) {
+	fx := newGitHubFixture(t, true)
+	fx.apkBytes["nameless-1-r0.apk"] = testsupport.MinimalApk("", "1-r0", "x86_64")
+	asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2.3/nameless-1-r0.apk"}
+	_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
+	if !errors.Is(err, provider.ErrInvalidPackage) {
+		t.Fatalf("want ErrInvalidPackage, got %v", err)
+	}
+}

internal/provider/alpine/syncer_test.go

@@ -308,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
 		t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
 	}
 }
+
+// A transiently failed asset fails the scan: the ETag is not advanced, a retry
+// is scheduled, and the retry against the unchanged release list derives it. An
+// invalid package is skipped instead, since no retry can fix it.
+func TestSyncerFailedAssetRetry(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		corrupt   bool
+		wantRetry bool
+	}{
+		{name: "transient", wantRetry: true},
+		{name: "invalid package", corrupt: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			fx := newGitHubFixture(t, true)
+			fx.etag = `"v1"`
+			if tc.corrupt {
+				fx.apkBytes["other-9-r0.apk"] = []byte("not a package")
+			} else {
+				fx.apkBytes["other-9-r0.apk"] = testsupport.MinimalApk("other", "9-r0", "aarch64")
+				fx.assetFail["other-9-r0.apk"] = 1
+			}
+			store := newFakeSyncStore()
+			s := newSyncer(store, newTestProvider(), testSyncConfig())
+			remote := fx.remote()
+			bg := context.Background()
+
+			s.process(bg, syncJob{remote: remote, prime: true})
+			_, pending := store.retryAt[remote.Name]
+			if pending != tc.wantRetry {
+				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
+			}
+			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
+				t.Fatalf("etag saved=%v after first scan", saved)
+			}
+			if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 1 {
+				t.Fatalf("healthy asset not served: %d rows", len(rows))
+			}
+			if !tc.wantRetry {
+				return
+			}
+
+			store.mu.Lock()
+			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
+			store.mu.Unlock()
+			s.process(bg, syncJob{remote: remote})
+			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
+				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
+			}
+			if rows, _ := store.ListAlpineMetadataEntries(bg, remote.Name); len(rows) != 2 {
+				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
+			}
+		})
+	}
+}

internal/provider/cargo/local.go

@@ -1,303 +0,0 @@
-package cargo
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/binary"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log/slog"
-	"net/http"
-	"strings"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-)
-
-// publishDep is a dependency as cargo sends it in the publish metadata.
-type publishDep struct {
-	Name               string   `json:"name"`
-	VersionReq         string   `json:"version_req"`
-	Features           []string `json:"features"`
-	Optional           bool     `json:"optional"`
-	DefaultFeatures    bool     `json:"default_features"`
-	Target             *string  `json:"target"`
-	Kind               string   `json:"kind"`
-	Registry           *string  `json:"registry"`
-	ExplicitNameInToml *string  `json:"explicit_name_in_toml"`
-}
-
-// PublishMeta is the JSON half of a `PUT /api/v1/crates/new` body; fields the
-// index does not carry are ignored.
-type PublishMeta struct {
-	Name        string              `json:"name"`
-	Vers        string              `json:"vers"`
-	Deps        []publishDep        `json:"deps"`
-	Features    map[string][]string `json:"features"`
-	Links       *string             `json:"links"`
-	RustVersion *string             `json:"rust_version"`
-}
-
-type indexDep struct {
-	Name            string   `json:"name"`
-	Req             string   `json:"req"`
-	Features        []string `json:"features"`
-	Optional        bool     `json:"optional"`
-	DefaultFeatures bool     `json:"default_features"`
-	Target          *string  `json:"target"`
-	Kind            string   `json:"kind"`
-	Registry        *string  `json:"registry,omitempty"`
-	Package         *string  `json:"package,omitempty"`
-}
-
-// IndexLine is one line of a sparse index file.
-type IndexLine struct {
-	Name        string              `json:"name"`
-	Vers        string              `json:"vers"`
-	Deps        []indexDep          `json:"deps"`
-	Cksum       string              `json:"cksum"`
-	Features    map[string][]string `json:"features"`
-	Features2   map[string][]string `json:"features2,omitempty"`
-	Yanked      bool                `json:"yanked"`
-	Links       *string             `json:"links,omitempty"`
-	V           int                 `json:"v,omitempty"`
-	RustVersion *string             `json:"rust_version,omitempty"`
-}
-
-// ParsePublish splits a publish body: u32le JSON length, JSON, u32le crate length, crate.
-func ParsePublish(body []byte) (PublishMeta, []byte, error) {
-	var meta PublishMeta
-	jsonBytes, rest, err := lengthPrefixed(body)
-	if err != nil {
-		return meta, nil, fmt.Errorf("metadata: %w", err)
-	}
-	crate, rest, err := lengthPrefixed(rest)
-	if err != nil {
-		return meta, nil, fmt.Errorf("crate: %w", err)
-	}
-	if len(rest) != 0 {
-		return meta, nil, errors.New("trailing bytes after crate")
-	}
-	if err := json.Unmarshal(jsonBytes, &meta); err != nil {
-		return meta, nil, fmt.Errorf("metadata: %w", err)
-	}
-	return meta, crate, nil
-}
-
-func lengthPrefixed(b []byte) ([]byte, []byte, error) {
-	if len(b) < 4 {
-		return nil, nil, errors.New("truncated length")
-	}
-	n := binary.LittleEndian.Uint32(b)
-	b = b[4:]
-	if uint64(n) > uint64(len(b)) {
-		return nil, nil, errors.New("truncated body")
-	}
-	return b[:n], b[n:], nil
-}
-
-// NewIndexLine validates the metadata and translates it into index form.
-func NewIndexLine(meta PublishMeta, crate []byte) (IndexLine, error) {
-	if !validName(meta.Name) {
-		return IndexLine{}, fmt.Errorf("invalid crate name %q", meta.Name)
-	}
-	if !validVersion(meta.Vers) {
-		return IndexLine{}, fmt.Errorf("invalid version %q", meta.Vers)
-	}
-	sum := sha256.Sum256(crate)
-	line := IndexLine{
-		Name:        meta.Name,
-		Vers:        meta.Vers,
-		Deps:        make([]indexDep, 0, len(meta.Deps)),
-		Cksum:       hex.EncodeToString(sum[:]),
-		Features:    map[string][]string{},
-		Links:       meta.Links,
-		RustVersion: meta.RustVersion,
-	}
-	for _, d := range meta.Deps {
-		dep := indexDep{
-			Name:            d.Name,
-			Req:             d.VersionReq,
-			Features:        d.Features,
-			Optional:        d.Optional,
-			DefaultFeatures: d.DefaultFeatures,
-			Target:          d.Target,
-			Kind:            d.Kind,
-			Registry:        d.Registry,
-		}
-		if dep.Features == nil {
-			dep.Features = []string{}
-		}
-		if dep.Kind == "" {
-			dep.Kind = "normal"
-		}
-		if d.ExplicitNameInToml != nil && *d.ExplicitNameInToml != "" {
-			pkg := d.Name
-			dep.Name, dep.Package = *d.ExplicitNameInToml, &pkg
-		}
-		line.Deps = append(line.Deps, dep)
-	}
-	// Older cargo cannot parse dep:/?/ feature syntax, so those features live in
-	// features2 behind v:2, as on crates.io.
-	for name, vals := range meta.Features {
-		if vals == nil {
-			vals = []string{}
-		}
-		if usesNewFeatureSyntax(vals) {
-			if line.Features2 == nil {
-				line.Features2 = map[string][]string{}
-			}
-			line.Features2[name] = vals
-			continue
-		}
-		line.Features[name] = vals
-	}
-	if line.Features2 != nil {
-		line.V = 2
-	}
-	return line, nil
-}
-
-func usesNewFeatureSyntax(vals []string) bool {
-	for _, v := range vals {
-		if strings.HasPrefix(v, "dep:") || strings.Contains(v, "?/") {
-			return true
-		}
-	}
-	return false
-}
-
-func validName(name string) bool {
-	if name == "" || len(name) > 64 {
-		return false
-	}
-	for i, c := range name {
-		alpha := (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
-		if i == 0 && !alpha {
-			return false
-		}
-		if !alpha && (c < '0' || c > '9') && c != '-' && c != '_' {
-			return false
-		}
-	}
-	return true
-}
-
-func validVersion(v string) bool {
-	if v == "" || len(v) > 128 || !strings.ContainsAny(v[:1], "0123456789") {
-		return false
-	}
-	return strings.Trim(v, "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-+") == ""
-}
-
-// IndexPath is the sparse-index file path for a crate name.
-func IndexPath(name string) string {
-	n := strings.ToLower(name)
-	switch len(n) {
-	case 0:
-		return ""
-	case 1:
-		return "1/" + n
-	case 2:
-		return "2/" + n
-	case 3:
-		return "3/" + n[:1] + "/" + n
-	}
-	return n[:2] + "/" + n[2:4] + "/" + n
-}
-
-// CratePath is where a published .crate is stored, matching config.json's dl.
-func CratePath(name, version string) string {
-	return crateDir + name + "/" + name + "-" + version + ".crate"
-}
-
-// RenderIndex joins stored lines into an index file with each yanked flag applied.
-func RenderIndex(entries []provider.CargoIndexEntry) ([]byte, error) {
-	var b strings.Builder
-	for _, e := range entries {
-		var line IndexLine
-		if err := json.Unmarshal([]byte(e.Line), &line); err != nil {
-			return nil, fmt.Errorf("%s %s: %w", e.Name, e.Version, err)
-		}
-		line.Yanked = e.Yanked
-		raw, err := json.Marshal(line)
-		if err != nil {
-			return nil, err
-		}
-		b.Write(raw)
-		b.WriteByte('\n')
-	}
-	return []byte(b.String()), nil
-}
-
-// LocalConfig is config.json for a local registry rooted at base.
-func LocalConfig(base string) map[string]string {
-	return map[string]string{
-		"dl":  base + "/" + crateDir + "{crate}/{crate}-{version}.crate",
-		"api": base,
-	}
-}
-
-func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, path string) bool {
-	path = strings.TrimLeft(path, "/")
-	if path == "config.json" {
-		w.Header().Set("Content-Type", "application/json")
-		_ = json.NewEncoder(w).Encode(LocalConfig(requestBase(r) + "/api/v1/local/" + repoName))
-		return true
-	}
-	name := path[strings.LastIndex(path, "/")+1:]
-	if name == "" || IndexPath(name) != path {
-		return false
-	}
-
-	reader, ok := files.(provider.CargoIndexReader)
-	if !ok {
-		http.Error(w, "cargo index not available", http.StatusInternalServerError)
-		return true
-	}
-	entries, err := reader.ListCargoIndexEntries(r.Context(), repoName, name)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return true
-	}
-	if len(entries) == 0 {
-		http.Error(w, "not found", http.StatusNotFound)
-		return true
-	}
-	body, err := RenderIndex(entries)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
-		return true
-	}
-	w.Header().Set("Content-Type", "text/plain")
-	w.WriteHeader(http.StatusOK)
-	_, _ = w.Write(body)
-	return true
-}
-
-func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, path string) ([]byte, error) {
-	return nil, fmt.Errorf("cargo local index generation for virtual repos not supported")
-}
-
-func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
-	deleter, ok := db.(provider.CargoIndexDeleter)
-	if !ok {
-		return nil
-	}
-	if err := deleter.DeleteCargoIndexEntry(ctx, repoName, storagePath); err != nil {
-		slog.Error("cargo index: delete failed", "repo", repoName, "path", storagePath, "error", err)
-		return err
-	}
-	return nil
-}
-
-func requestBase(r *http.Request) string {
-	scheme := "http"
-	if r.TLS != nil {
-		scheme = "https"
-	} else if fwd := r.Header.Get("X-Forwarded-Proto"); fwd != "" {
-		scheme = fwd
-	}
-	return scheme + "://" + r.Host
-}

internal/provider/cargo/local_test.go

@@ -1,218 +0,0 @@
-package cargo
-
-import (
-	"context"
-	"crypto/sha256"
-	"encoding/binary"
-	"encoding/hex"
-	"encoding/json"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"git.unkin.net/unkin/artifactapi/internal/provider"
-)
-
-func publishBody(meta string, crate []byte) []byte {
-	var b []byte
-	b = binary.LittleEndian.AppendUint32(b, uint32(len(meta)))
-	b = append(b, meta...)
-	b = binary.LittleEndian.AppendUint32(b, uint32(len(crate)))
-	return append(b, crate...)
-}
-
-const sampleMeta = `{
-  "name": "unkin-hello", "vers": "0.2.0",
-  "deps": [
-    {"name": "serde", "version_req": "^1", "features": ["derive"], "optional": false,
-     "default_features": true, "target": null, "kind": "normal",
-     "registry": "https://github.com/rust-lang/crates.io-index"},
-    {"name": "rand_core", "version_req": "^0.6", "features": null, "optional": true,
-     "default_features": false, "target": "cfg(unix)", "kind": "dev",
-     "explicit_name_in_toml": "rc"}
-  ],
-  "features": {"default": ["std"], "std": [], "extra": ["dep:rc"], "weak": ["serde?/std"]},
-  "links": null, "rust_version": "1.70", "description": "ignored"
-}`
-
-func TestParsePublishAndIndexLine(t *testing.T) {
-	crate := []byte("fake crate bytes")
-	meta, gotCrate, err := ParsePublish(publishBody(sampleMeta, crate))
-	if err != nil {
-		t.Fatal(err)
-	}
-	if string(gotCrate) != string(crate) {
-		t.Fatalf("crate = %q", gotCrate)
-	}
-	line, err := NewIndexLine(meta, crate)
-	if err != nil {
-		t.Fatal(err)
-	}
-	sum := sha256.Sum256(crate)
-	if line.Cksum != hex.EncodeToString(sum[:]) {
-		t.Errorf("cksum = %s", line.Cksum)
-	}
-
-	raw, _ := json.Marshal(line)
-	var got map[string]any
-	_ = json.Unmarshal(raw, &got)
-	if _, ok := got["version_req"]; ok {
-		t.Error("publish-only field leaked into index line")
-	}
-	if got["v"] != float64(2) {
-		t.Errorf("v = %v, want 2 with features2 present", got["v"])
-	}
-	if got["rust_version"] != "1.70" {
-		t.Errorf("rust_version = %v", got["rust_version"])
-	}
-	if _, ok := line.Features["extra"]; ok {
-		t.Error("dep: feature must move to features2")
-	}
-	if len(line.Features2) != 2 || len(line.Features) != 2 {
-		t.Errorf("features = %v features2 = %v", line.Features, line.Features2)
-	}
-
-	deps := got["deps"].([]any)
-	serde := deps[0].(map[string]any)
-	if serde["req"] != "^1" || serde["registry"] != "https://github.com/rust-lang/crates.io-index" || serde["target"] != nil {
-		t.Errorf("serde dep = %v", serde)
-	}
-	if _, ok := serde["package"]; ok {
-		t.Error("package set without a rename")
-	}
-	renamed := deps[1].(map[string]any)
-	if renamed["name"] != "rc" || renamed["package"] != "rand_core" || renamed["kind"] != "dev" {
-		t.Errorf("renamed dep = %v", renamed)
-	}
-	if f, ok := renamed["features"].([]any); !ok || len(f) != 0 {
-		t.Errorf("null features must render as [], got %v", renamed["features"])
-	}
-}
-
-func TestIndexLineNoNewFeaturesOmitsV(t *testing.T) {
-	line, err := NewIndexLine(PublishMeta{Name: "a", Vers: "1.0.0", Features: map[string][]string{"x": nil}}, nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	raw, _ := json.Marshal(line)
-	s := string(raw)
-	for _, bad := range []string{`"v":`, `"features2"`, `"links"`} {
-		if strings.Contains(s, bad) {
-			t.Errorf("%s present in %s", bad, s)
-		}
-	}
-	if !strings.Contains(s, `"deps":[]`) || !strings.Contains(s, `"features":{"x":[]}`) {
-		t.Errorf("deps/features must be non-null: %s", s)
-	}
-}
-
-func TestParsePublishRejectsMalformed(t *testing.T) {
-	good := publishBody(`{"name":"a","vers":"1.0.0"}`, []byte("c"))
-	for name, body := range map[string][]byte{
-		"empty":     nil,
-		"short":     good[:len(good)-1],
-		"trailing":  append(append([]byte{}, good...), 'x'),
-		"bad json":  publishBody(`{`, []byte("c")),
-		"no length": good[:6],
-	} {
-		if _, _, err := ParsePublish(body); err == nil {
-			t.Errorf("%s: expected error", name)
-		}
-	}
-}
-
-func TestNewIndexLineValidation(t *testing.T) {
-	for _, m := range []PublishMeta{
-		{Name: "", Vers: "1.0.0"},
-		{Name: "1abc", Vers: "1.0.0"},
-		{Name: "../x", Vers: "1.0.0"},
-		{Name: "ok", Vers: "1.0.0/../../x"},
-		{Name: "ok", Vers: ""},
-		{Name: strings.Repeat("a", 65), Vers: "1.0.0"},
-	} {
-		if _, err := NewIndexLine(m, nil); err == nil {
-			t.Errorf("%q@%q accepted", m.Name, m.Vers)
-		}
-	}
-	if _, err := NewIndexLine(PublishMeta{Name: "Foo_bar-2", Vers: "1.0.0-rc.1+build.5"}, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestIndexPath(t *testing.T) {
-	for name, want := range map[string]string{
-		"a": "1/a", "ab": "2/ab", "abc": "3/a/abc", "Serde": "se/rd/serde", "unkin-hello": "un/ki/unkin-hello",
-	} {
-		if got := IndexPath(name); got != want {
-			t.Errorf("IndexPath(%q) = %q, want %q", name, got, want)
-		}
-	}
-}
-
-type fakeIndex struct {
-	entries []provider.CargoIndexEntry
-	asked   string
-}
-
-func (f *fakeIndex) ListCargoIndexEntries(_ context.Context, _, name string) ([]provider.CargoIndexEntry, error) {
-	f.asked = name
-	return f.entries, nil
-}
-func (f *fakeIndex) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
-	return nil, nil
-}
-func (f *fakeIndex) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
-
-func TestServeLocalIndex(t *testing.T) {
-	line, _ := NewIndexLine(PublishMeta{Name: "unkin-hello", Vers: "0.1.0"}, []byte("x"))
-	raw, _ := json.Marshal(line)
-	files := &fakeIndex{entries: []provider.CargoIndexEntry{
-		{Name: "unkin-hello", Version: "0.1.0", Line: string(raw), Yanked: true},
-		{Name: "unkin-hello", Version: "0.2.0", Line: strings.Replace(string(raw), "0.1.0", "0.2.0", 1)},
-	}}
-	p := &Provider{}
-
-	rec := httptest.NewRecorder()
-	if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "http://aa.example/x", nil), files, "crates-local", "un/ki/unkin-hello") {
-		t.Fatal("index path not served")
-	}
-	lines := strings.Split(strings.TrimSpace(rec.Body.String()), "\n")
-	if len(lines) != 2 || !strings.Contains(lines[0], `"yanked":true`) || !strings.Contains(lines[1], `"yanked":false`) {
-		t.Errorf("index body = %q", rec.Body.String())
-	}
-	if files.asked != "unkin-hello" {
-		t.Errorf("looked up %q", files.asked)
-	}
-
-	rec = httptest.NewRecorder()
-	req := httptest.NewRequest("GET", "http://aa.example/config.json", nil)
-	req.Header.Set("X-Forwarded-Proto", "https")
-	p.ServeLocalIndex(rec, req, files, "crates-local", "config.json")
-	var cfg map[string]string
-	_ = json.Unmarshal(rec.Body.Bytes(), &cfg)
-	if cfg["api"] != "https://aa.example/api/v1/local/crates-local" ||
-		cfg["dl"] != "https://aa.example/api/v1/local/crates-local/crates/{crate}/{crate}-{version}.crate" {
-		t.Errorf("config = %v", cfg)
-	}
-
-	for _, path := range []string{"crates/unkin-hello/unkin-hello-0.1.0.crate", "xx/ki/unkin-hello", ""} {
-		if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), files, "r", path) {
-			t.Errorf("%q should fall through", path)
-		}
-	}
-
-	files.entries = nil
-	rec = httptest.NewRecorder()
-	p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), files, "r", "3/a/abc")
-	if rec.Code != 404 {
-		t.Errorf("unknown crate code = %d", rec.Code)
-	}
-}
-
-func TestCratePathMatchesDL(t *testing.T) {
-	dl := LocalConfig("http://h/api/v1/local/r")["dl"]
-	want := strings.NewReplacer("{crate}", "unkin-hello", "{version}", "0.1.0").Replace(dl)
-	if got := "http://h/api/v1/local/r/" + CratePath("unkin-hello", "0.1.0"); got != want {
-		t.Errorf("CratePath = %q, dl expands to %q", got, want)
-	}
-}

internal/provider/cargo_index.go

@@ -1,24 +0,0 @@
-package provider
-
-import "context"
-
-// CargoIndexEntry is one published crate version in a local cargo registry.
-// Line is the sparse-index JSON line; Yanked is applied when it is served.
-type CargoIndexEntry struct {
-	RepoName string
-	FilePath string
-	Name     string
-	Version  string
-	Line     string
-	Yanked   bool
-}
-
-// CargoIndexReader lists every version of one crate (matched case-insensitively).
-// *database.DB satisfies it and CargoIndexDeleter.
-type CargoIndexReader interface {
-	ListCargoIndexEntries(ctx context.Context, repoName, name string) ([]CargoIndexEntry, error)
-}
-
-type CargoIndexDeleter interface {
-	DeleteCargoIndexEntry(ctx context.Context, repoName, filePath string) error
-}

internal/provider/deb/github.go

@@ -333,6 +333,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
+	// A transiently failed asset fails the scan so the ETag is not advanced and
+	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
+	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -361,10 +364,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_deb: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				if !errors.Is(err, provider.ErrInvalidPackage) {
+					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
+				}
 				continue
 			}
 			if err := store.InsertDebMetadata(ctx, meta); err != nil {
 				slog.Error("github_deb: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_deb: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Architecture)
@@ -376,7 +383,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = store.DeleteDebMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, nil
+	return newEtag, true, errors.Join(failed...)
 }
 
 type ghRelease struct {
@@ -468,7 +475,7 @@ func (p *GitHubProvider) deriveAsset(ctx context.Context, remote models.Remote,
 		Size:         asset.Size,
 	}
 	if meta.Name == "" {
-		return nil, errors.New("control missing Package field")
+		return nil, fmt.Errorf("%w: control missing Package field", provider.ErrInvalidPackage)
 	}
 
 	// The Packages SHA256 must be the sha256 of the whole .deb. Prefer GitHub's
@@ -503,13 +510,13 @@ func (p *GitHubProvider) fetchControl(ctx context.Context, remote models.Remote,
 		}
 		control, complete, perr := controlFromPrefix(body)
 		if perr != nil {
-			return "", fmt.Errorf("parse deb control: %w", perr)
+			return "", fmt.Errorf("%w: parse deb control: %w", provider.ErrInvalidPackage, perr)
 		}
 		if complete {
 			return control, nil
 		}
 		if full || n >= p.headerMax {
-			return "", fmt.Errorf("control.tar not found within %d bytes of %s", n, downloadURL)
+			return "", fmt.Errorf("%w: control.tar not found within %d bytes of %s", provider.ErrInvalidPackage, n, downloadURL)
 		}
 		n *= 2
 		if n > p.headerMax {
@@ -567,7 +574,7 @@ func controlFromPrefix(prefix []byte) (control string, complete bool, err error)
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response body was shorter than n (i.e. we already have the whole object).
+// response holds the whole object.
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -589,15 +596,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
+		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	full := int64(len(body)) < n
-	return body, full, nil
+	return provider.ReadPrefix(resp, n)
 }
 
 func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -620,7 +621,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
+		return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
 
 	h := sha256.New()

internal/provider/deb/github_test.go

@@ -7,6 +7,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -81,15 +82,17 @@ type githubFixture struct {
 	notModHit   int
 	releaseAuth string
 	assetAuth   string
+	assetFail   map[string]int // asset filename -> downloads left to answer 500
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		debBytes: map[string][]byte{},
-		rangeHit: map[string]int{},
-		fullHit:  map[string]int{},
+		debBytes:  map[string][]byte{},
+		rangeHit:  map[string]int{},
+		fullHit:   map[string]int{},
+		assetFail: map[string]int{},
 	}
 	f.debBytes["demo_1.2-3_amd64.deb"] = testsupport.MinimalDeb("demo", "1.2-3", "amd64")
 
@@ -139,6 +142,12 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
+		if f.assetFail[name] > 0 {
+			f.assetFail[name]--
+			f.mu.Unlock()
+			http.Error(w, "boom", http.StatusInternalServerError)
+			return
+		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -460,3 +469,13 @@ func TestGitHubAssetPatternFilter(t *testing.T) {
 		t.Fatalf("pattern filter failed, rows=%+v", rows)
 	}
 }
+
+func TestGitHubDeriveAssetMissingPackageIsInvalid(t *testing.T) {
+	fx := newGitHubFixture(t, true)
+	fx.debBytes["nameless_1_amd64.deb"] = testsupport.MinimalDeb("", "1", "amd64")
+	asset := ghAsset{BrowserDownloadURL: fx.srv.URL + "/acme/tools/releases/download/v1.2-3/nameless_1_amd64.deb", Digest: "sha256:00"}
+	_, err := newTestProvider().deriveAsset(context.Background(), fx.remote(), asset, "fp")
+	if !errors.Is(err, provider.ErrInvalidPackage) {
+		t.Fatalf("want ErrInvalidPackage, got %v", err)
+	}
+}

internal/provider/deb/syncer_test.go

@@ -308,3 +308,58 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
 		t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
 	}
 }
+
+// A transiently failed asset fails the scan: the ETag is not advanced, a retry
+// is scheduled, and the retry against the unchanged release list derives it. An
+// invalid package is skipped instead, since no retry can fix it.
+func TestSyncerFailedAssetRetry(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		corrupt   bool
+		wantRetry bool
+	}{
+		{name: "transient", wantRetry: true},
+		{name: "invalid package", corrupt: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			fx := newGitHubFixture(t, true)
+			fx.etag = `"v1"`
+			if tc.corrupt {
+				fx.debBytes["other_9-9_arm64.deb"] = []byte("not a package")
+			} else {
+				fx.debBytes["other_9-9_arm64.deb"] = testsupport.MinimalDeb("other", "9-9", "arm64")
+				fx.assetFail["other_9-9_arm64.deb"] = 1
+			}
+			store := newFakeSyncStore()
+			s := newSyncer(store, newTestProvider(), testSyncConfig())
+			remote := fx.remote()
+			bg := context.Background()
+
+			s.process(bg, syncJob{remote: remote, prime: true})
+			_, pending := store.retryAt[remote.Name]
+			if pending != tc.wantRetry {
+				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
+			}
+			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
+				t.Fatalf("etag saved=%v after first scan", saved)
+			}
+			if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 1 {
+				t.Fatalf("healthy asset not served: %d rows", len(rows))
+			}
+			if !tc.wantRetry {
+				return
+			}
+
+			store.mu.Lock()
+			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
+			store.mu.Unlock()
+			s.process(bg, syncJob{remote: remote})
+			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
+				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
+			}
+			if rows, _ := store.ListDebMetadataEntries(bg, remote.Name); len(rows) != 2 {
+				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
+			}
+		})
+	}
+}

internal/provider/rpm/github.go

@@ -343,6 +343,9 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 		return newEtag, false, err
 	}
 
+	// A transiently failed asset fails the scan so the ETag is not advanced and
+	// the retry re-derives it; an invalid package is skipped as no retry fixes it.
+	var failed []error
 	seen := map[string]bool{}
 	for _, rel := range releases {
 		if rel.Draft {
@@ -373,10 +376,14 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			meta, err := p.deriveAsset(ctx, remote, asset, fp)
 			if err != nil {
 				slog.Warn("github_rpm: derive asset failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				if !errors.Is(err, provider.ErrInvalidPackage) {
+					failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
+				}
 				continue
 			}
 			if err := store.InsertRPMMetadata(ctx, meta); err != nil {
 				slog.Error("github_rpm: insert metadata failed", "remote", remote.Name, "asset", asset.Name, "error", err)
+				failed = append(failed, fmt.Errorf("%s: %w", asset.Name, err))
 				continue
 			}
 			slog.Info("github_rpm: derived asset", "remote", remote.Name, "name", meta.Name, "version", meta.Version, "arch", meta.Arch)
@@ -388,7 +395,7 @@ func (p *GitHubProvider) scanWithState(ctx context.Context, remote models.Remote
 			_ = store.DeleteRPMMetadata(ctx, remote.Name, fp)
 		}
 	}
-	return newEtag, true, nil
+	return newEtag, true, errors.Join(failed...)
 }
 
 type ghRelease struct {
@@ -569,12 +576,12 @@ func (p *GitHubProvider) fetchHeader(ctx context.Context, remote models.Remote,
 			}
 			continue
 		}
-		return nil, fmt.Errorf("parse rpm header: %w", perr)
+		return nil, fmt.Errorf("%w: parse rpm header: %w", provider.ErrInvalidPackage, perr)
 	}
 }
 
 // rangeGet returns the first n bytes of downloadURL. full is true when the
-// response body was shorter than n (i.e. we already have the whole object).
+// response holds the whole object.
 func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, downloadURL string, n int64) ([]byte, bool, error) {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
 	if err != nil {
@@ -596,15 +603,9 @@ func (p *GitHubProvider) rangeGet(ctx context.Context, remote models.Remote, dow
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
-		return nil, false, fmt.Errorf("range GET %s: status %d", downloadURL, resp.StatusCode)
+		return nil, false, fmt.Errorf("range GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, n))
-	if err != nil {
-		return nil, false, err
-	}
-	full := int64(len(body)) < n
-	return body, full, nil
+	return provider.ReadPrefix(resp, n)
 }
 
 func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote, downloadURL string) (string, error) {
@@ -627,7 +628,7 @@ func (p *GitHubProvider) computeSHA256(ctx context.Context, remote models.Remote
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET %s: status %d", downloadURL, resp.StatusCode)
+		return "", fmt.Errorf("GET: %w", provider.AssetStatusError(downloadURL, resp))
 	}
 
 	h := sha256.New()

internal/provider/rpm/github_test.go

@@ -1,11 +1,13 @@
 package rpm
 
 import (
+	"cmp"
 	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -24,15 +26,22 @@ import (
 // fakeStore is an in-memory provider.RemoteMetadataStore keyed by file_path,
 // mirroring the (repo_name, file_path) uniqueness of the real table.
 type fakeStore struct {
-	mu   sync.Mutex
-	rows map[string]provider.RPMMetadata
+	mu         sync.Mutex
+	rows       map[string]provider.RPMMetadata
+	insertFail map[string]int // package name -> inserts left to fail
 }
 
-func newFakeStore() *fakeStore { return &fakeStore{rows: map[string]provider.RPMMetadata{}} }
+func newFakeStore() *fakeStore {
+	return &fakeStore{rows: map[string]provider.RPMMetadata{}, insertFail: map[string]int{}}
+}
 
 func (f *fakeStore) InsertRPMMetadata(_ context.Context, m *provider.RPMMetadata) error {
 	f.mu.Lock()
 	defer f.mu.Unlock()
+	if f.insertFail[m.Name] > 0 {
+		f.insertFail[m.Name]--
+		return errors.New("insert failed")
+	}
 	if _, ok := f.rows[m.FilePath]; ok {
 		return nil // ON CONFLICT DO NOTHING
 	}
@@ -79,15 +88,20 @@ type githubFixture struct {
 	releaseAuth string            // Authorization header seen on the last releases request
 	assetAuth   string            // Authorization header seen on the last asset request
 	failStatus  int               // when set, the releases list answers this status (rate-limit style)
+	assetFail   map[string]int    // asset filename -> downloads left to answer failCode
+	failCode    int               // status for assetFail downloads; 0 = 500
+	shortRead   map[string]int    // asset filename -> ranged GETs left to close early
 	mu          sync.Mutex
 }
 
 func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 	t.Helper()
 	f := &githubFixture{
-		rpmBytes: map[string][]byte{},
-		rangeHit: map[string]int{},
-		fullHit:  map[string]int{},
+		rpmBytes:  map[string][]byte{},
+		rangeHit:  map[string]int{},
+		fullHit:   map[string]int{},
+		assetFail: map[string]int{},
+		shortRead: map[string]int{},
 	}
 	f.rpmBytes["demo-1.2-3.x86_64.rpm"] = testsupport.MinimalRPM("demo", "1.2", "3", "x86_64")
 
@@ -145,6 +159,17 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 		}
 		rng := r.Header.Get("Range")
 		f.mu.Lock()
+		if f.assetFail[name] > 0 {
+			f.assetFail[name]--
+			code := cmp.Or(f.failCode, http.StatusInternalServerError)
+			f.mu.Unlock()
+			http.Error(w, "boom", code)
+			return
+		}
+		short := rng != "" && f.shortRead[name] > 0
+		if short {
+			f.shortRead[name]--
+		}
 		f.assetAuth = r.Header.Get("Authorization")
 		if rng != "" {
 			f.rangeHit[name]++
@@ -165,6 +190,9 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
 			end = len(body) - 1
 		}
 		w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-%d/%d", end, len(body)))
+		if short {
+			end /= 2
+		}
 		w.Header().Set("Content-Length", strconv.Itoa(end+1))
 		w.WriteHeader(http.StatusPartialContent)
 		w.Write(body[:end+1])

internal/provider/rpm/syncer_test.go

@@ -399,3 +399,74 @@ func TestSyncerFailedScanRetriesAfterBackoff(t *testing.T) {
 		t.Fatalf("recovery did not refresh metadata: %d rows", len(rows))
 	}
 }
+
+// A transiently failed asset fails the scan: the ETag is not advanced, a retry
+// is scheduled, and the retry against the unchanged release list derives it. An
+// invalid package is skipped instead, since no retry can fix it.
+func TestSyncerFailedAssetRetry(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		corrupt   bool
+		failCode  int
+		short     bool
+		insert    bool
+		wantRetry bool
+	}{
+		{name: "transient", wantRetry: true},
+		{name: "rate limited", failCode: http.StatusForbidden, wantRetry: true},
+		{name: "short read", short: true, wantRetry: true},
+		{name: "insert failure", insert: true, wantRetry: true},
+		{name: "invalid package", corrupt: true},
+		{name: "asset not found", failCode: http.StatusNotFound},
+		{name: "asset gone", failCode: http.StatusGone},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			const other = "other-9-9.aarch64.rpm"
+			fx := newGitHubFixture(t, true)
+			fx.etag = `"v1"`
+			fx.rpmBytes[other] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
+			store := newFakeSyncStore()
+			switch {
+			case tc.corrupt:
+				fx.rpmBytes[other] = []byte("not a package")
+			case tc.short:
+				fx.shortRead[other] = 1
+			case tc.insert:
+				store.insertFail["other"] = 1
+			case tc.failCode != 0 && !tc.wantRetry:
+				fx.failCode, fx.assetFail[other] = tc.failCode, 1<<30
+			default:
+				fx.failCode, fx.assetFail[other] = tc.failCode, 1
+			}
+			s := newSyncer(store, newTestProvider(), testSyncConfig())
+			remote := fx.remote()
+			bg := context.Background()
+
+			s.process(bg, syncJob{remote: remote, prime: true})
+			_, pending := store.retryAt[remote.Name]
+			if pending != tc.wantRetry {
+				t.Fatalf("retry pending=%v, want %v", pending, tc.wantRetry)
+			}
+			if _, saved := store.etags[remote.Name]; saved == tc.wantRetry {
+				t.Fatalf("etag saved=%v after first scan", saved)
+			}
+			if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
+				t.Fatalf("healthy asset not served: %d rows", len(rows))
+			}
+			if !tc.wantRetry {
+				return
+			}
+
+			store.mu.Lock()
+			store.retryAt[remote.Name] = time.Now().Add(-time.Second)
+			store.mu.Unlock()
+			s.process(bg, syncJob{remote: remote})
+			if _, pending := store.retryAt[remote.Name]; pending || store.etags[remote.Name] != `"v1"` {
+				t.Fatalf("retry not recorded: pending=%v etag=%q", pending, store.etags[remote.Name])
+			}
+			if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 2 {
+				t.Fatalf("retry did not derive the failed asset: %d rows", len(rows))
+			}
+		})
+	}
+}

internal/provider/syncretry.go

@@ -3,8 +3,10 @@ package provider
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -43,6 +45,49 @@ func NewUpstreamStatusError(url string, resp *http.Response) *UpstreamStatusErro
 	return e
 }
 
+// ErrInvalidPackage marks an asset whose bytes are not a valid package. It is
+// permanent, so a scan skips the asset instead of failing and retrying.
+var ErrInvalidPackage = errors.New("invalid package")
+
+// AssetStatusError classifies a non-success asset download. A missing or
+// unsatisfiable asset is permanent; anything else (403/429 rate limits, 5xx) is
+// transient.
+func AssetStatusError(url string, resp *http.Response) error {
+	switch resp.StatusCode {
+	case http.StatusNotFound, http.StatusGone, http.StatusRequestedRangeNotSatisfiable:
+		return fmt.Errorf("%w: %s: status %d", ErrInvalidPackage, url, resp.StatusCode)
+	}
+	return NewUpstreamStatusError(url, resp)
+}
+
+// ReadPrefix reads up to n bytes of a 200 or 206 asset response. full reports
+// that the body is the whole object (per Content-Length or the Content-Range
+// total, or a clean EOF before n when the total is unknown); a body ending before n bytes that is not the whole object is a
+// transient short read.
+func ReadPrefix(resp *http.Response, n int64) (body []byte, full bool, err error) {
+	body, err = io.ReadAll(io.LimitReader(resp.Body, n))
+	if err != nil {
+		return nil, false, err
+	}
+	total := resp.ContentLength
+	unknown := resp.StatusCode == http.StatusOK && total == -1
+	if resp.StatusCode == http.StatusPartialContent {
+		total = -1
+		if _, t, ok := strings.Cut(resp.Header.Get("Content-Range"), "/"); ok {
+			if v, perr := strconv.ParseInt(t, 10, 64); perr == nil {
+				total = v
+			}
+			unknown = t == "*"
+		}
+	}
+	// net/http errors on a premature close, so a clean EOF before n with an unknown total is the whole object.
+	full = total == int64(len(body)) || (unknown && int64(len(body)) < n)
+	if !full && int64(len(body)) < n {
+		return nil, false, fmt.Errorf("short read: got %d of %d bytes", len(body), n)
+	}
+	return body, full, nil
+}
+
 // SyncResult is a background scan's outcome, recorded when its sync lease is
 // released. A failed scan keeps the prior sync time and ETag and schedules a
 // retry after Backoff, doubled per consecutive failure up to MaxBackoff, and

internal/provider/syncretry_test.go

@@ -3,7 +3,9 @@ package provider
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
+	"net/http/httptest"
 	"strconv"
 	"testing"
 	"time"
@@ -61,3 +63,76 @@ func TestNewSyncResult(t *testing.T) {
 		t.Fatalf("wrapped hint not clamped to ttl: retry in %v", until)
 	}
 }
+
+func TestAssetStatusErrorClassification(t *testing.T) {
+	for status, permanent := range map[int]bool{
+		http.StatusNotFound: true, http.StatusGone: true, http.StatusRequestedRangeNotSatisfiable: true,
+		http.StatusForbidden: false, http.StatusTooManyRequests: false, http.StatusInternalServerError: false, http.StatusBadGateway: false,
+	} {
+		err := AssetStatusError("u", &http.Response{StatusCode: status, Header: http.Header{}})
+		if errors.Is(err, ErrInvalidPackage) != permanent {
+			t.Errorf("status %d: permanent=%v, want %v", status, !permanent, permanent)
+		}
+	}
+}
+
+func TestReadPrefix(t *testing.T) {
+	cases := []struct {
+		name     string
+		status   int
+		cl       int64
+		crange   string
+		body     string
+		n        int64
+		wantFull bool
+		wantErr  bool
+	}{
+		{name: "200 whole object", status: 200, cl: 5, body: "hello", n: 32, wantFull: true},
+		{name: "200 larger than n", status: 200, cl: 5, body: "hello", n: 4},
+		{name: "200 chunked shorter than n", status: 200, cl: -1, body: "hello", n: 32, wantFull: true},
+		{name: "200 chunked larger than n", status: 200, cl: -1, body: "hello", n: 4},
+		{name: "206 whole object", status: 206, crange: "bytes 0-4/5", body: "hello", n: 32, wantFull: true},
+		{name: "206 prefix", status: 206, crange: "bytes 0-3/100", body: "hell", n: 4},
+		{name: "206 short read", status: 206, crange: "bytes 0-31/100", body: "hello", n: 32, wantErr: true},
+		{name: "206 no content-range short", status: 206, body: "hello", n: 32, wantErr: true},
+		{name: "206 unknown total shorter than n", status: 206, cl: -1, crange: "bytes 0-4/*", body: "hello", n: 32, wantFull: true},
+		{name: "206 unknown total prefix", status: 206, cl: -1, crange: "bytes 0-3/*", body: "hell", n: 4},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+				if tc.cl > 0 {
+					w.Header().Set("Content-Length", strconv.FormatInt(tc.cl, 10))
+				}
+				if tc.crange != "" {
+					w.Header().Set("Content-Range", tc.crange)
+				}
+				w.WriteHeader(tc.status)
+				_, _ = io.WriteString(w, tc.body)
+				w.(http.Flusher).Flush()
+			}))
+			defer srv.Close()
+			resp, err := http.Get(srv.URL)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer resp.Body.Close()
+			if tc.cl == -1 && resp.ContentLength != -1 {
+				t.Fatalf("response not chunked: ContentLength=%d", resp.ContentLength)
+			}
+			body, full, err := ReadPrefix(resp, tc.n)
+			if (err != nil) != tc.wantErr {
+				t.Fatalf("err=%v, wantErr=%v", err, tc.wantErr)
+			}
+			if err != nil {
+				if errors.Is(err, ErrInvalidPackage) {
+					t.Fatalf("short read must be transient: %v", err)
+				}
+				return
+			}
+			if full != tc.wantFull || int64(len(body)) > tc.n {
+				t.Fatalf("full=%v len=%d, want full=%v", full, len(body), tc.wantFull)
+			}
+		})
+	}
+}

internal/server/middleware.go

@@ -12,7 +12,7 @@ func cors(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Access-Control-Allow-Origin", "*")
 		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
-		w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization")
+		w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
 		if r.Method == http.MethodOptions {
 			w.WriteHeader(http.StatusNoContent)
 			return

internal/server/oidc_test.go

@@ -1,80 +0,0 @@
-package server
-
-import (
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-	"time"
-
-	"git.unkin.net/unkin/artifactapi/internal/testsupport"
-)
-
-func TestServerOIDCWriteAuth(t *testing.T) {
-	requireStack(t)
-	iss := testsupport.NewOIDCIssuer()
-	defer iss.Close()
-
-	cfg := *testSrv.cfg
-	cfg.OIDCIssuer = iss.URL
-	cfg.OIDCAudiences = []string{"artifactapi"}
-	s := *testSrv
-	s.cfg = &cfg
-	ts := httptest.NewServer(s.routes())
-	defer ts.Close()
-
-	good := iss.Token(nil, "artifactapi", "ci", time.Hour)
-	do := func(method, path, body, token string) int {
-		t.Helper()
-		rq, _ := http.NewRequest(method, ts.URL+path, strings.NewReader(body))
-		rq.Header.Set("Content-Type", "application/json")
-		if token != "" {
-			rq.Header.Set("Authorization", "Bearer "+token)
-		}
-		resp, err := http.DefaultClient.Do(rq)
-		if err != nil {
-			t.Fatalf("%s %s: %v", method, path, err)
-		}
-		_, _ = io.Copy(io.Discard, resp.Body)
-		_ = resp.Body.Close()
-		return resp.StatusCode
-	}
-	expect := func(want int, method, path, body, token string) {
-		t.Helper()
-		if got := do(method, path, body, token); got != want {
-			t.Errorf("%s %s: status %d, want %d", method, path, got, want)
-		}
-	}
-
-	remote := `{"name":"srv-oidc","package_type":"generic","repo_type":"local"}`
-	virtual := `{"name":"srv-oidc-v","package_type":"generic","members":["srv-oidc"]}`
-
-	expect(401, "POST", "/api/v2/remotes", remote, "")
-	expect(401, "POST", "/api/v2/remotes", remote, iss.Token(nil, "other", "ci", time.Hour))
-	expect(201, "POST", "/api/v2/remotes", remote, good)
-	expect(401, "PUT", "/api/v2/remotes/srv-oidc", `{"package_type":"generic"}`, "")
-	expect(401, "DELETE", "/api/v2/remotes/srv-oidc", "", "")
-	expect(401, "POST", "/api/v2/virtuals", virtual, "")
-	expect(201, "POST", "/api/v2/virtuals", virtual, good)
-	expect(401, "PUT", "/api/v2/virtuals/srv-oidc-v", virtual, "")
-	expect(401, "DELETE", "/api/v2/virtuals/srv-oidc-v", "", "")
-
-	expect(200, "GET", "/api/v2/remotes", "", "")
-	expect(200, "GET", "/api/v2/remotes/srv-oidc", "", "")
-	expect(200, "GET", "/api/v2/virtuals/srv-oidc-v", "", "")
-	expect(201, "PUT", "/api/v2/remotes/srv-oidc/files/a.bin", "payload", "")
-	expect(200, "GET", "/api/v2/remotes/srv-oidc/files/a.bin", "", "")
-	if got := do("DELETE", "/api/v2/remotes/srv-oidc/files/a.bin", "", ""); got >= 400 {
-		t.Errorf("delete file: status %d", got)
-	}
-	if got := do("POST", "/api/v2/probe", `{}`, ""); got == 401 {
-		t.Error("probe gated")
-	}
-	if got := do("GET", "/v2/", "", ""); got == 401 {
-		t.Error("docker /v2 gated")
-	}
-
-	expect(204, "DELETE", "/api/v2/virtuals/srv-oidc-v", "", good)
-	expect(204, "DELETE", "/api/v2/remotes/srv-oidc", "", good)
-}

internal/server/server.go

@@ -15,7 +15,6 @@ import (
 	tfregistry "git.unkin.net/unkin/artifactapi/internal/api/terraform"
 	v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
 	v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
-	"git.unkin.net/unkin/artifactapi/internal/auth"
 	"git.unkin.net/unkin/artifactapi/internal/cache"
 	"git.unkin.net/unkin/artifactapi/internal/config"
 	"git.unkin.net/unkin/artifactapi/internal/database"
@@ -187,11 +186,10 @@ func (s *Server) routes() chi.Router {
 	statsHandler := v2.NewStatsHandler(s.db)
 	eventsHandler := v2.NewEventsHandler()
 	probeHandler := v2.NewProbeHandler(s.engine, s.db)
-	writeAuth := auth.NewBearerOIDC(s.cfg.OIDCIssuer, s.cfg.OIDCAudiences)
 
 	r.Route("/api/v2", func(r chi.Router) {
-		r.Mount("/remotes", writeAuth.RequireWrites(remotesHandler.Routes()))
-		r.Mount("/virtuals", writeAuth.RequireWrites(virtualsHandler.Routes()))
+		r.Mount("/remotes", remotesHandler.Routes())
+		r.Mount("/virtuals", virtualsHandler.Routes())
 		r.Mount("/health", healthHandler.Routes())
 		r.Mount("/stats", statsHandler.Routes())
 		r.Mount("/events", eventsHandler.Routes())

internal/testsupport/oidc.go

@@ -1,73 +0,0 @@
-package testsupport
-
-import (
-	"crypto/rand"
-	"crypto/rsa"
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"time"
-
-	"github.com/go-jose/go-jose/v4"
-	"github.com/go-jose/go-jose/v4/jwt"
-)
-
-// OIDCIssuer is a minimal OIDC issuer serving discovery and JWKS, signing
-// RS256 tokens with a generated key.
-type OIDCIssuer struct {
-	*httptest.Server
-	key *rsa.PrivateKey
-}
-
-func NewOIDCIssuer() *OIDCIssuer {
-	key, err := rsa.GenerateKey(rand.Reader, 2048)
-	if err != nil {
-		panic(err)
-	}
-	iss := &OIDCIssuer{key: key}
-	mux := http.NewServeMux()
-	mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
-		_ = json.NewEncoder(w).Encode(map[string]any{
-			"issuer":                                iss.URL,
-			"jwks_uri":                              iss.URL + "/keys",
-			"id_token_signing_alg_values_supported": []string{"RS256"},
-		})
-	})
-	mux.HandleFunc("/keys", func(w http.ResponseWriter, r *http.Request) {
-		_ = json.NewEncoder(w).Encode(jose.JSONWebKeySet{Keys: []jose.JSONWebKey{
-			{Key: &key.PublicKey, KeyID: "k1", Algorithm: "RS256", Use: "sig"},
-		}})
-	})
-	iss.Server = httptest.NewServer(mux)
-	return iss
-}
-
-// Token signs a token for aud and sub. Pass a nil key to use the issuer's own;
-// expiry is relative to now.
-func (o *OIDCIssuer) Token(key *rsa.PrivateKey, aud, sub string, expiry time.Duration) string {
-	now := time.Now()
-	return o.Sign(key, jwt.Claims{
-		Issuer:   o.URL,
-		Subject:  sub,
-		Audience: jwt.Audience{aud},
-		IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
-		Expiry:   jwt.NewNumericDate(now.Add(expiry)),
-	})
-}
-
-// Sign signs arbitrary claims with RS256. Pass a nil key to use the issuer's own.
-func (o *OIDCIssuer) Sign(key *rsa.PrivateKey, claims jwt.Claims) string {
-	if key == nil {
-		key = o.key
-	}
-	sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: key},
-		(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
-	if err != nil {
-		panic(err)
-	}
-	tok, err := jwt.Signed(sig).Claims(claims).Serialize()
-	if err != nil {
-		panic(err)
-	}
-	return tok
-}

migrations/0003_cargo_index.sql

@@ -1,15 +0,0 @@
--- Local cargo registries: one sparse-index line per published crate version.
--- line holds the index JSON minus yanked, which flips on yank/unyank.
-CREATE TABLE IF NOT EXISTS cargo_index (
-	id           BIGSERIAL PRIMARY KEY,
-	repo_name    TEXT NOT NULL,
-	file_path    TEXT NOT NULL,
-	name         TEXT NOT NULL,
-	version      TEXT NOT NULL,
-	line         TEXT NOT NULL,
-	yanked       BOOLEAN NOT NULL DEFAULT FALSE,
-	created_at   TIMESTAMPTZ DEFAULT NOW(),
-	UNIQUE(repo_name, file_path)
-);
-
-CREATE INDEX IF NOT EXISTS idx_cargo_index_repo_name ON cargo_index(repo_name, lower(name));

migrations/0003_github_sync_state_cascade.sql

@@ -0,0 +1,12 @@
+-- Sync state is dropped with its remote, so a recreated remote of the same name
+-- starts with no inherited ETag or timing. NOT VALID skips checking rows left by
+-- remotes deleted before this migration, which would otherwise fail it.
+ALTER TABLE github_rpm_sync_state    DROP CONSTRAINT IF EXISTS github_rpm_sync_state_remote_fk;
+ALTER TABLE github_rpm_sync_state    ADD CONSTRAINT github_rpm_sync_state_remote_fk
+	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;
+ALTER TABLE github_deb_sync_state    DROP CONSTRAINT IF EXISTS github_deb_sync_state_remote_fk;
+ALTER TABLE github_deb_sync_state    ADD CONSTRAINT github_deb_sync_state_remote_fk
+	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;
+ALTER TABLE github_alpine_sync_state DROP CONSTRAINT IF EXISTS github_alpine_sync_state_remote_fk;
+ALTER TABLE github_alpine_sync_state ADD CONSTRAINT github_alpine_sync_state_remote_fk
+	FOREIGN KEY (remote_name) REFERENCES remotes(name) ON DELETE CASCADE NOT VALID;

ui/src/components/UsageInstructions.tsx

@@ -229,25 +229,6 @@ go mod download`,
       ];
 
     case 'cargo':
-      if (isLocal) {
-        return [
-          {
-            title: 'Add the private registry (real sparse registry)',
-            language: 'toml',
-            code: `# $CARGO_HOME/config.toml
-[registries.${name}]
-index = "sparse+${url}/api/v1/local/${name}/"`,
-            note: 'cargo requires a token for publish/yank: set CARGO_REGISTRIES_' + name.toUpperCase().replace(/-/g, '_') + '_TOKEN to any value.',
-          },
-          {
-            title: 'Publish, depend on and yank crates',
-            language: 'bash',
-            code: `cargo publish --registry ${name}
-cargo add <crate> --registry ${name}
-cargo yank --version <version> --registry ${name}`,
-          },
-        ];
-      }
       return [
         {
           title: 'Point crates.io at this sparse registry',