Compare commits

..

6 Commits

Author SHA1 Message Date
unkin-agent 7539f93c01 Retrigger CI
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
ci/woodpecker/pr/build Pipeline was successful
2026-10-16 23:25:41 +11:00
unkin-agent dbb1af26d5 Retrigger CI
ci/woodpecker/pr/build Pipeline failed
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
2026-10-16 23:18:29 +11:00
unkin-agent 18e7f000f7 Add helm local repositories
ci/woodpecker/pr/build Pipeline failed
ci/woodpecker/pr/pre-commit Pipeline was successful
ci/woodpecker/pr/test Pipeline was successful
2026-10-16 23:12:12 +11:00
unkin-agent 9161558334 Add optional OIDC bearer auth on remote/virtual config writes (#136)
ci/woodpecker/tag/docker Pipeline was successful
Remote and virtual configuration writes are unauthenticated, so anything that can reach the API can repoint or delete repos. Vault identity OIDC tokens give callers a verifiable identity.

- add `ARTIFACTAPI_OIDC_ISSUER`/`ARTIFACTAPI_OIDC_AUDIENCE`; unset issuer keeps auth off
- require a JWKS-verified bearer token on POST/PUT/DELETE of `/api/v2/remotes` and `/api/v2/virtuals`, 401 otherwise
- leave uploads, object deletes, probe, reads and `/v2` open
- discover the issuer lazily with a 10s client timeout; fail fast with 503 for 15s after a failed discovery
- log the verified `sub`; allow `Authorization` in CORS

Reviewed-on: #136
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-10-16 22:08:20 +11:00
unkin-agent 492607a164 Retry failed GitHub release scans with backoff (#133)
ci/woodpecker/tag/docker Pipeline was successful
Releasing a GitHub sync lease always advanced `last_synced_at`, even after a failed scan (e.g. a rate-limit 403). A remote that failed once waited a full `mutable_ttl` before retrying, and a cold remote kept returning 503 until then.

- record scan outcomes in one shared lease helper for github_rpm/deb/alpine
- keep `last_synced_at` and the ETag on failure; retry from 60s with exponential backoff, capped at min(10m, ttl/4)
- honour `Retry-After` / `X-RateLimit-Reset`, clamped to `mutable_ttl`
- add `sync_failures` / `next_retry_at` columns (migration 0002)

Reviewed-on: #133
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-10-09 23:15:55 +11:00
unkin-agent 1780b3d77c Push release images to artifactapi docker-internal (#134)
Gitea's registry rejects blob HEADs with 401, so the v3.13.0 tag pipeline cannot push. Release images move to the in-cluster artifactapi docker-internal registry, which trusts the runner without credentials.

- push docker-api to docker-internal/artifactapi and docker-web to docker-internal/artifactapi-ui
- use the CA-baked docker-internal/plugin-docker-buildx image
- drop the droneci credentials
- set kubernetes resources and service account on both steps

Reviewed-on: #134
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>
2026-10-09 23:08:22 +11:00
39 changed files with 1717 additions and 190 deletions
+26 -12
View File
@@ -4,31 +4,45 @@ when:
steps:
- name: docker-api
image: woodpeckerci/plugin-docker-buildx
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
settings:
registry: git.unkin.net
repo: git.unkin.net/unkin/artifactapi
registry: artifactapi.k8s.syd1.au.unkin.net
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi
build_args:
VERSION: ${CI_COMMIT_TAG}
username: droneci
password:
from_secret: DRONECI_PASSWORD
tags:
- ${CI_COMMIT_TAG}
- latest
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 1Gi
cpu: 1
limits:
memory: 4Gi
cpu: 2
- name: docker-web
image: woodpeckerci/plugin-docker-buildx
image: artifactapi.k8s.syd1.au.unkin.net/docker-internal/plugin-docker-buildx:latest
settings:
registry: git.unkin.net
repo: git.unkin.net/unkin/artifactapi-ui
registry: artifactapi.k8s.syd1.au.unkin.net
repo: artifactapi.k8s.syd1.au.unkin.net/docker-internal/artifactapi-ui
dockerfile: ui/Dockerfile.ui
context: ui
build_args:
BASE_PATH: /ui
username: droneci
password:
from_secret: DRONECI_PASSWORD
tags:
- ${CI_COMMIT_TAG}
- latest
backend_options:
kubernetes:
serviceAccountName: default
resources:
requests:
memory: 1Gi
cpu: 1
limits:
memory: 4Gi
cpu: 2
+17
View File
@@ -261,6 +261,21 @@ there's nothing to provision. To bring your own key instead, point
`TF_SIGNING_KEY_PASSPHRASE`), which takes precedence over the generated one.
`TF_PROVIDER_PROTOCOLS` (default `5.0,6.0`) sets the advertised plugin protocols.
### Local helm repository
A local `helm` repo is a real chart repository: `PUT` a packaged chart to
`/api/v2/remotes/<repo>/files/<chart>-<version>.tgz` and `index.yaml` is
generated from each chart's `Chart.yaml` (with sha256 digests) at
`/api/v1/local/<repo>/index.yaml`. Add the local as a member of a helm virtual
to serve it alongside remotes.
```bash
helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \
https://artifactapi.k8s.syd1.au.unkin.net/api/v2/remotes/<repo>/files/mychart-0.1.0.tgz
helm repo add <repo> https://artifactapi.k8s.syd1.au.unkin.net/api/v1/local/<repo>
```
### Local docker registry
A local `docker` repo is a real container registry, not a mirror: it serves the
@@ -371,6 +386,8 @@ additive and idempotent for the same reason; a test enforces it.
| `GITHUB_APP_INSTALLATION_ID` | | GitHub App installation id |
| `GITHUB_APP_PRIVATE_KEY` | | GitHub App private key, inline PEM |
| `GITHUB_APP_PRIVATE_KEY_PATH` | | GitHub App private key, file path (alternative to inline PEM) |
| `ARTIFACTAPI_OIDC_ISSUER` | | OIDC issuer URL (e.g. Vault `https://vault.service.consul:8200/v1/identity/oidc`). When set, `POST /api/v2/remotes`, `POST /api/v2/virtuals` and `PUT`/`DELETE` on `/api/v2/{remotes,virtuals}/{name}` require `Authorization: Bearer <jwt>` verified against the issuer's JWKS. Uploads, object/file deletes, probe, reads and `/v2` stay open. Unset = no auth |
| `ARTIFACTAPI_OIDC_AUDIENCE` | | Comma-separated allowed `aud` values (e.g. the Vault OIDC role's client_id). Required when `ARTIFACTAPI_OIDC_ISSUER` is set |
## Development
+71
View File
@@ -0,0 +1,71 @@
//go:build dockere2e
package e2edocker
import (
"net/http"
"os"
"os/exec"
"strings"
"testing"
"time"
)
// TestLocalHelmRealClient packages a chart with the real helm CLI, publishes it
// to a helm local, then drives helm repo add/search/pull/template against the
// local and against a helm virtual that has the local as a member.
func TestLocalHelmRealClient(t *testing.T) {
network := os.Getenv("COMPOSE_NETWORK")
internal := strings.TrimRight(os.Getenv("ARTIFACTAPI_INTERNAL"), "/")
if network == "" || internal == "" {
t.Skip("COMPOSE_NETWORK/ARTIFACTAPI_INTERNAL not set; run via scripts/docker-e2e.sh")
}
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker not available on the test host")
}
createRepo(t, `{"name":"local-helm","package_type":"helm","repo_type":"local"}`)
defer deleteRepo(t, "local-helm")
createVirtual(t, `{"name":"vhelm","package_type":"helm","members":["local-helm"]}`)
defer deleteVirtual(t, "vhelm")
helm := func(script string) string {
t.Helper()
out, err := exec.Command("docker", "run", "--rm", "--network", network, "--entrypoint", "sh",
"alpine/helm:3.16.4", "-c", "set -eu; "+script).Output()
if err != nil {
stderr := ""
if ee, ok := err.(*exec.ExitError); ok {
stderr = string(ee.Stderr)
}
t.Fatalf("helm script failed: %v\n%s%s", err, out, stderr)
}
return string(out)
}
chart := helm("cd /tmp && helm create e2echart >/dev/null && helm package e2echart >/dev/null && cat e2echart-0.1.0.tgz")
uploadFile(t, "local-helm", "e2echart-0.1.0.tgz", []byte(chart), "application/gzip")
// Chart metadata is recorded asynchronously after upload; poll the index.
var body []byte
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(500 * time.Millisecond) {
var resp *http.Response
resp, body = doRequest(t, http.MethodGet, api("/api/v1/local/local-helm/index.yaml"), nil, "")
if resp.StatusCode == http.StatusOK && strings.Contains(string(body), "e2echart-0.1.0.tgz") {
break
}
}
if !strings.Contains(string(body), "e2echart-0.1.0.tgz") || !strings.Contains(string(body), "digest: ") {
t.Fatalf("index.yaml never listed the chart:\n%s", body)
}
for _, repoURL := range []string{internal + "/api/v1/local/local-helm", internal + "/api/v1/virtual/vhelm"} {
out := helm("helm repo add e2e " + repoURL + " >/dev/null && " +
"helm search repo e2e/e2echart && " +
"cd /tmp && helm pull e2e/e2echart --version 0.1.0 && " +
"helm template rel e2echart-0.1.0.tgz | grep -c 'kind: Deployment'")
if !strings.Contains(out, "e2e/e2echart") || !strings.Contains(out, "0.1.0") || !strings.HasSuffix(strings.TrimSpace(out), "1") {
t.Fatalf("helm via %s:\n%s", repoURL, out)
}
}
}
+3
View File
@@ -7,7 +7,9 @@ require (
github.com/cavaliergopher/rpm v1.3.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
github.com/coreos/go-oidc/v3 v3.21.0
github.com/go-chi/chi/v5 v5.3.0
github.com/go-jose/go-jose/v4 v4.1.4
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.10.0
github.com/klauspost/compress v1.19.2
@@ -102,6 +104,7 @@ require (
go.uber.org/atomic v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/net v0.56.0 // indirect
golang.org/x/oauth2 v0.36.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/text v0.40.0 // indirect
gopkg.in/ini.v1 v1.67.2 // indirect
+6
View File
@@ -44,6 +44,8 @@ github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
github.com/coreos/go-oidc/v3 v3.21.0 h1:wZo4Q9Pum8dYEj0eMUPrqR+kvuGkeUplbLpNCkBqoWM=
github.com/coreos/go-oidc/v3 v3.21.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4=
github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s=
@@ -67,6 +69,8 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/go-chi/chi/v5 v5.3.0 h1:halUjDxhshgXHMrao5bB8eNBXo/rnzwr8m5m36glehM=
github.com/go-chi/chi/v5 v5.3.0/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA=
github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
@@ -225,6 +229,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+158
View File
@@ -0,0 +1,158 @@
package auth
import (
"context"
"errors"
"log/slog"
"net/http"
"slices"
"strings"
"sync"
"time"
"github.com/coreos/go-oidc/v3/oidc"
)
// BearerOIDC verifies `Authorization: Bearer <jwt>` against an OIDC issuer's
// JWKS. Discovery is lazy so startup never depends on the issuer being
// reachable; a failed discovery is not retried until the backoff elapses.
type BearerOIDC struct {
issuer string
audiences []string
timeout time.Duration
backoff time.Duration
discover func(context.Context, string) (*oidc.Provider, error)
mu sync.Mutex
verifier *oidc.IDTokenVerifier
inflight chan struct{}
failedAt time.Time
}
var errIssuerUnavailable = errors.New("oidc issuer unavailable")
// NewBearerOIDC returns nil when issuer is empty, meaning auth is disabled.
func NewBearerOIDC(issuer string, audiences []string) *BearerOIDC {
if issuer == "" {
return nil
}
return &BearerOIDC{issuer: issuer, audiences: audiences, timeout: 10 * time.Second, backoff: 15 * time.Second, discover: oidc.NewProvider}
}
// getVerifier runs at most one discovery at a time without holding the lock
// across network I/O; concurrent callers wait for it, and only callers within
// the post-failure backoff fail fast.
func (b *BearerOIDC) getVerifier(ctx context.Context) (*oidc.IDTokenVerifier, error) {
b.mu.Lock()
if v := b.verifier; v != nil {
b.mu.Unlock()
return v, nil
}
if done := b.inflight; done != nil {
b.mu.Unlock()
select {
case <-done:
case <-ctx.Done():
return nil, ctx.Err()
}
b.mu.Lock()
defer b.mu.Unlock()
if b.verifier == nil {
return nil, errIssuerUnavailable
}
return b.verifier, nil
}
if time.Since(b.failedAt) < b.backoff {
b.mu.Unlock()
return nil, errIssuerUnavailable
}
done := make(chan struct{})
b.inflight = done
b.mu.Unlock()
v, err := b.discoverOnce(done)
if err != nil {
slog.Error("oidc discovery failed", "issuer", b.issuer, "error", err)
return nil, err
}
return v, nil
}
// discoverOnce releases inflight and waiters on every exit, a panic included,
// recording failedAt unless discovery succeeded.
func (b *BearerOIDC) discoverOnce(done chan struct{}) (v *oidc.IDTokenVerifier, err error) {
defer func() {
b.mu.Lock()
b.inflight = nil
if v == nil {
b.failedAt = time.Now()
}
b.verifier = v
b.mu.Unlock()
close(done)
}()
// The provider keeps this context and client for JWKS refreshes, so the
// deadline lives on the client rather than the context.
client := &http.Client{Timeout: b.timeout}
p, err := b.discover(oidc.ClientContext(context.Background(), client), b.issuer)
if err != nil {
return nil, err
}
return p.Verifier(&oidc.Config{SkipClientIDCheck: true}), nil
}
// Require gates every request through next on a valid token. A nil receiver
// passes everything through.
func (b *BearerOIDC) Require(next http.Handler) http.Handler {
if b == nil {
return next
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
scheme, raw, _ := strings.Cut(strings.TrimSpace(r.Header.Get("Authorization")), " ")
raw = strings.TrimSpace(raw)
if !strings.EqualFold(scheme, "Bearer") || raw == "" {
unauthorized(w, "missing bearer token")
return
}
v, err := b.getVerifier(r.Context())
if err != nil {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusServiceUnavailable)
_, _ = w.Write([]byte(`{"error":"token issuer unavailable"}`))
return
}
tok, err := v.Verify(r.Context(), raw)
if err != nil {
slog.Info("bearer token rejected", "error", err)
unauthorized(w, "invalid token")
return
}
if len(b.audiences) > 0 && !slices.ContainsFunc(tok.Audience, func(a string) bool { return slices.Contains(b.audiences, a) }) {
slog.Info("bearer token rejected", "error", "audience not allowed", "aud", tok.Audience)
unauthorized(w, "invalid token")
return
}
slog.Info("bearer token accepted", "sub", tok.Subject, "method", r.Method, "path", r.URL.Path)
next.ServeHTTP(w, r)
})
}
// RequireWrites gates only mutating methods.
func (b *BearerOIDC) RequireWrites(next http.Handler) http.Handler {
gated := b.Require(next)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet, http.MethodHead, http.MethodOptions:
next.ServeHTTP(w, r)
default:
gated.ServeHTTP(w, r)
}
})
}
func unauthorized(w http.ResponseWriter, msg string) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("WWW-Authenticate", "Bearer")
w.WriteHeader(http.StatusUnauthorized)
_, _ = w.Write([]byte(`{"error":"` + msg + `"}`))
}
+255
View File
@@ -0,0 +1,255 @@
package auth
import (
"context"
"crypto/rand"
"crypto/rsa"
"encoding/base64"
"net/http"
"net/http/httptest"
"strconv"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/coreos/go-oidc/v3/oidc"
"github.com/go-jose/go-jose/v4"
"github.com/go-jose/go-jose/v4/jwt"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
func serve(h http.Handler, method, token string) int {
if token == "" {
return serveAuth(h, method, "")
}
return serveAuth(h, method, "Bearer "+token)
}
func serveAuth(h http.Handler, method, authz string) int {
r := httptest.NewRequest(method, "/x", nil)
if authz != "" {
r.Header.Set("Authorization", authz)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, r)
return w.Code
}
func claims(iss string) jwt.Claims {
now := time.Now()
return jwt.Claims{
Issuer: iss,
Subject: "ci",
Audience: jwt.Audience{"artifactapi"},
IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
Expiry: jwt.NewNumericDate(now.Add(time.Hour)),
}
}
func hs256Token(t *testing.T, c jwt.Claims) string {
sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: []byte("0123456789abcdef0123456789abcdef")},
(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
if err != nil {
t.Fatal(err)
}
tok, err := jwt.Signed(sig).Claims(c).Serialize()
if err != nil {
t.Fatal(err)
}
return tok
}
func noneToken(iss string) string {
enc := base64.RawURLEncoding.EncodeToString
return enc([]byte(`{"alg":"none","typ":"JWT","kid":"k1"}`)) + "." +
enc([]byte(`{"iss":"`+iss+`","sub":"ci","aud":"artifactapi","exp":`+strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)+`}`)) + "."
}
var ok = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) })
func TestBearerOIDC(t *testing.T) {
iss := testsupport.NewOIDCIssuer()
defer iss.Close()
h := NewBearerOIDC(iss.URL, []string{"other", "artifactapi"}).Require(ok)
otherKey, _ := rsa.GenerateKey(rand.Reader, 2048)
cases := []struct {
name string
token string
want int
}{
{"valid", iss.Token(nil, "artifactapi", "ci", time.Hour), http.StatusNoContent},
{"missing", "", http.StatusUnauthorized},
{"garbage", "not.a.jwt", http.StatusUnauthorized},
{"bad signature", iss.Token(otherKey, "artifactapi", "ci", time.Hour), http.StatusUnauthorized},
{"wrong aud", iss.Token(nil, "nope", "ci", time.Hour), http.StatusUnauthorized},
{"expired", iss.Token(nil, "artifactapi", "ci", -time.Hour), http.StatusUnauthorized},
{"wrong iss", iss.Sign(nil, claims("https://evil.example")), http.StatusUnauthorized},
{"hs256", hs256Token(t, claims(iss.URL)), http.StatusUnauthorized},
{"alg none", noneToken(iss.URL), http.StatusUnauthorized},
}
for _, c := range cases {
if got := serve(h, http.MethodPost, c.token); got != c.want {
t.Errorf("%s: status %d, want %d", c.name, got, c.want)
}
}
valid := iss.Token(nil, "artifactapi", "ci", time.Hour)
headers := []struct {
name string
authz string
want int
}{
{"lowercase scheme", "bearer " + valid, http.StatusNoContent},
{"extra whitespace", " Bearer " + valid + " ", http.StatusNoContent},
{"empty value", "Bearer ", http.StatusUnauthorized},
{"scheme only", "Bearer", http.StatusUnauthorized},
{"basic scheme", "Basic " + valid, http.StatusUnauthorized},
}
for _, c := range headers {
if got := serveAuth(h, http.MethodPost, c.authz); got != c.want {
t.Errorf("%s: status %d, want %d", c.name, got, c.want)
}
}
}
func TestBearerOIDCHangingIssuerBacksOff(t *testing.T) {
var hits atomic.Int32
release := make(chan struct{})
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hits.Add(1)
select {
case <-r.Context().Done():
case <-release:
}
}))
defer srv.Close()
defer close(release)
b := NewBearerOIDC(srv.URL, nil)
b.timeout = 100 * time.Millisecond
b.backoff = time.Hour
h := b.Require(ok)
start := time.Now()
if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
t.Fatalf("hanging issuer: status %d, want 503", got)
}
if d := time.Since(start); d > 2*time.Second {
t.Fatalf("hanging issuer: took %s, timeout not applied", d)
}
if got := serve(h, http.MethodPost, "x.y.z"); got != http.StatusServiceUnavailable {
t.Fatalf("during backoff: status %d, want 503", got)
}
if n := hits.Load(); n != 1 {
t.Fatalf("issuer hit %d times, want 1", n)
}
}
func TestBearerOIDCPanickingDiscoveryBacksOff(t *testing.T) {
b := NewBearerOIDC("http://issuer.invalid", nil)
b.backoff = time.Hour
b.discover = func(context.Context, string) (*oidc.Provider, error) { panic("boom") }
h := b.Require(ok)
func() {
defer func() { _ = recover() }()
serve(h, http.MethodPost, "x.y.z")
}()
got := make(chan int, 1)
go func() { got <- serve(h, http.MethodPost, "x.y.z") }()
select {
case code := <-got:
if code != http.StatusServiceUnavailable {
t.Fatalf("after panic: status %d, want 503", code)
}
case <-time.After(2 * time.Second):
t.Fatal("after panic: request hung on stale inflight discovery")
}
}
func TestBearerOIDCConcurrentFirstRequestsWaitForDiscovery(t *testing.T) {
iss := testsupport.NewOIDCIssuer()
defer iss.Close()
var hits atomic.Int32
inner := iss.Config.Handler
iss.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/.well-known/openid-configuration" {
hits.Add(1)
time.Sleep(200 * time.Millisecond)
}
inner.ServeHTTP(w, r)
})
h := NewBearerOIDC(iss.URL, nil).Require(ok)
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
var wg sync.WaitGroup
codes := make([]int, 20)
for i := range codes {
wg.Add(1)
go func() {
defer wg.Done()
codes[i] = serve(h, http.MethodPost, tok)
}()
}
wg.Wait()
for i, c := range codes {
if c != http.StatusNoContent {
t.Errorf("request %d: status %d, want 204", i, c)
}
}
if n := hits.Load(); n != 1 {
t.Fatalf("discovery hit %d times, want 1", n)
}
}
func TestBearerOIDCRetriesAfterBackoff(t *testing.T) {
iss := testsupport.NewOIDCIssuer()
defer iss.Close()
b := NewBearerOIDC(iss.URL, nil)
b.failedAt = time.Now()
b.backoff = 50 * time.Millisecond
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
t.Fatalf("within backoff: status %d, want 503", got)
}
time.Sleep(60 * time.Millisecond)
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusNoContent {
t.Fatalf("after backoff: status %d, want 204", got)
}
}
func TestBearerOIDCIssuerDown(t *testing.T) {
iss := testsupport.NewOIDCIssuer()
b := NewBearerOIDC(iss.URL, []string{"artifactapi"})
tok := iss.Token(nil, "artifactapi", "ci", time.Hour)
iss.Close()
if got := serve(b.Require(ok), http.MethodPost, tok); got != http.StatusServiceUnavailable {
t.Errorf("issuer down: status %d, want 503", got)
}
}
func TestBearerOIDCDisabled(t *testing.T) {
if got := serve(NewBearerOIDC("", nil).RequireWrites(ok), http.MethodDelete, ""); got != http.StatusNoContent {
t.Errorf("disabled: status %d, want 204", got)
}
}
func TestRequireWritesLeavesReadsOpen(t *testing.T) {
iss := testsupport.NewOIDCIssuer()
defer iss.Close()
h := NewBearerOIDC(iss.URL, []string{"artifactapi"}).RequireWrites(ok)
for _, m := range []string{http.MethodGet, http.MethodHead, http.MethodOptions} {
if got := serve(h, m, ""); got != http.StatusNoContent {
t.Errorf("%s: status %d, want 204", m, got)
}
}
for _, m := range []string{http.MethodPost, http.MethodPut, http.MethodDelete} {
if got := serve(h, m, ""); got != http.StatusUnauthorized {
t.Errorf("%s: status %d, want 401", m, got)
}
}
}
+19
View File
@@ -4,6 +4,7 @@ import (
"fmt"
"os"
"strconv"
"strings"
)
type Config struct {
@@ -56,6 +57,10 @@ type Config struct {
GitHubAppInstallationID string
GitHubAppPrivateKey string
GitHubAppPrivateKeyPath string
// Bearer-JWT auth on configuration writes; empty issuer disables it.
OIDCIssuer string
OIDCAudiences []string
}
func (c *Config) DatabaseDSN() string {
@@ -90,6 +95,17 @@ func Load() (*Config, error) {
return nil, fmt.Errorf("invalid GITHUB_SYNC_POLL_INTERVAL: %w", err)
}
oidcIssuer := getenv("ARTIFACTAPI_OIDC_ISSUER", "")
var oidcAudiences []string
for _, a := range strings.Split(getenv("ARTIFACTAPI_OIDC_AUDIENCE", ""), ",") {
if a = strings.TrimSpace(a); a != "" {
oidcAudiences = append(oidcAudiences, a)
}
}
if oidcIssuer != "" && len(oidcAudiences) == 0 {
return nil, fmt.Errorf("ARTIFACTAPI_OIDC_AUDIENCE is required when ARTIFACTAPI_OIDC_ISSUER is set")
}
cfg := &Config{
ListenAddr: getenv("LISTEN_ADDR", ":8000"),
@@ -123,6 +139,9 @@ func Load() (*Config, error) {
GitHubAppInstallationID: getenv("GITHUB_APP_INSTALLATION_ID", ""),
GitHubAppPrivateKey: getenv("GITHUB_APP_PRIVATE_KEY", ""),
GitHubAppPrivateKeyPath: getenv("GITHUB_APP_PRIVATE_KEY_PATH", ""),
OIDCIssuer: oidcIssuer,
OIDCAudiences: oidcAudiences,
}
return cfg, nil
+17
View File
@@ -64,3 +64,20 @@ func TestLoadInvalidPort(t *testing.T) {
t.Error("expected error for invalid DBPORT")
}
}
func TestLoadOIDC(t *testing.T) {
t.Setenv("ARTIFACTAPI_OIDC_ISSUER", "https://vault.example/v1/identity/oidc")
t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", " a, ,b ")
cfg, err := Load()
if err != nil {
t.Fatalf("load: %v", err)
}
if len(cfg.OIDCAudiences) != 2 || cfg.OIDCAudiences[0] != "a" || cfg.OIDCAudiences[1] != "b" {
t.Errorf("audiences = %q", cfg.OIDCAudiences)
}
t.Setenv("ARTIFACTAPI_OIDC_AUDIENCE", "")
if _, err := Load(); err == nil {
t.Error("expected error for issuer without audience")
}
}
+8 -38
View File
@@ -2,11 +2,9 @@ package database
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -30,42 +28,14 @@ func (db *DB) ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, err
return remotes, rows.Err()
}
// ClaimGitHubAlpineSyncLease atomically claims the per-remote sync lease. It
// succeeds only when the remote is due (never synced, or synced longer than
// freshness ago) and no live lease is held by another replica. A zero freshness
// (prime scans) ignores the recency gate. The returned etag is the stored
// releases-list ETag, shared across replicas.
// ClaimGitHubAlpineSyncLease atomically claims the per-remote github_alpine sync
// lease. See claimSyncLease.
func (db *DB) ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
row := db.Pool.QueryRow(ctx, `
INSERT INTO github_alpine_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
VALUES ($1, $2, now() + make_interval(secs => $4))
ON CONFLICT (remote_name) DO UPDATE
SET sync_lease_owner = $2,
sync_lease_expires = now() + make_interval(secs => $4)
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
RETURNING s.etag
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
var etag string
if err := row.Scan(&etag); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return false, "", nil
}
return false, "", err
}
return true, etag, nil
return db.claimSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, freshness, lease)
}
// ReleaseGitHubAlpineSyncLease records the completed scan and frees the lease.
// Only the owning replica may release; last_synced_at advances so the next poll
// waits a full freshness window, and etag is persisted for the next conditional
// request.
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
_, err := db.Pool.Exec(ctx, `
UPDATE github_alpine_sync_state
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
WHERE remote_name = $1 AND sync_lease_owner = $2
`, remoteName, owner, syncedAt, etag)
return err
// ReleaseGitHubAlpineSyncLease records a github_alpine scan outcome and frees the
// lease. See releaseSyncLease.
func (db *DB) ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
return db.releaseSyncLease(ctx, "github_alpine_sync_state", remoteName, owner, res)
}
+8 -37
View File
@@ -2,11 +2,9 @@ package database
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -30,41 +28,14 @@ func (db *DB) ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
return remotes, rows.Err()
}
// ClaimGitHubDebSyncLease atomically claims the per-remote sync lease. It
// succeeds only when the remote is due (never synced, or synced longer than
// freshness ago) and no live lease is held by another replica. A zero freshness
// (prime scans) ignores the recency gate. The returned etag is the stored
// releases-list ETag, shared across replicas.
// ClaimGitHubDebSyncLease atomically claims the per-remote github_deb sync
// lease. See claimSyncLease.
func (db *DB) ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
row := db.Pool.QueryRow(ctx, `
INSERT INTO github_deb_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
VALUES ($1, $2, now() + make_interval(secs => $4))
ON CONFLICT (remote_name) DO UPDATE
SET sync_lease_owner = $2,
sync_lease_expires = now() + make_interval(secs => $4)
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
RETURNING s.etag
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
var etag string
if err := row.Scan(&etag); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return false, "", nil
}
return false, "", err
}
return true, etag, nil
return db.claimSyncLease(ctx, "github_deb_sync_state", remoteName, owner, freshness, lease)
}
// ReleaseGitHubDebSyncLease records the completed scan and frees the lease. Only
// the owning replica may release; last_synced_at advances so the next poll waits
// a full freshness window, and etag is persisted for the next conditional request.
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
_, err := db.Pool.Exec(ctx, `
UPDATE github_deb_sync_state
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
WHERE remote_name = $1 AND sync_lease_owner = $2
`, remoteName, owner, syncedAt, etag)
return err
// ReleaseGitHubDebSyncLease records a github_deb scan outcome and frees the
// lease. See releaseSyncLease.
func (db *DB) ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
return db.releaseSyncLease(ctx, "github_deb_sync_state", remoteName, owner, res)
}
+2 -1
View File
@@ -4,6 +4,7 @@ import (
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -44,7 +45,7 @@ func TestGitHubDebSyncLease(t *testing.T) {
t.Fatal("replica-2 claimed while replica-1 holds the lease")
}
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
if err := testDB.ReleaseGitHubDebSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
t.Fatalf("release: %v", err)
}
+45 -17
View File
@@ -7,6 +7,8 @@ import (
"github.com/jackc/pgx/v5"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -30,22 +32,35 @@ func (db *DB) ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
return remotes, rows.Err()
}
// ClaimGitHubSyncLease atomically claims the per-remote sync lease. It succeeds
// (claimed=true) only when the remote is due — never synced, or synced longer
// than freshness ago — and no live lease is held by another replica. This bounds
// total GitHub load to roughly one scan per freshness window regardless of how
// many replicas poll. The returned etag is the stored releases-list ETag, shared
// across replicas so a conditional request can short-circuit an unchanged repo.
// A zero freshness (used for prime scans) ignores the recency gate and claims
// whenever no live lease is held.
// ClaimGitHubSyncLease atomically claims the per-remote github_rpm sync lease.
// See claimSyncLease.
func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
return db.claimSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, freshness, lease)
}
// ReleaseGitHubSyncLease records a github_rpm scan outcome and frees the lease.
// See releaseSyncLease.
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error {
return db.releaseSyncLease(ctx, "github_rpm_sync_state", remoteName, owner, res)
}
// claimSyncLease atomically claims a per-remote sync lease in table. It succeeds
// (claimed=true) only when the remote is due and no live lease is held by
// another replica. Due means: a pending retry after a failed scan has reached
// next_retry_at, or, with no retry pending, the remote was never synced or was
// synced longer than freshness ago. A zero freshness (prime scans) ignores the
// recency gate but still honours a pending retry's backoff. The returned etag is
// the stored releases-list ETag, shared across replicas so a conditional request
// can short-circuit an unchanged repo.
func (db *DB) claimSyncLease(ctx context.Context, table, remoteName, owner string, freshness, lease time.Duration) (bool, string, error) {
row := db.Pool.QueryRow(ctx, `
INSERT INTO github_rpm_sync_state AS s (remote_name, sync_lease_owner, sync_lease_expires)
INSERT INTO `+table+` AS s (remote_name, sync_lease_owner, sync_lease_expires)
VALUES ($1, $2, now() + make_interval(secs => $4))
ON CONFLICT (remote_name) DO UPDATE
SET sync_lease_owner = $2,
sync_lease_expires = now() + make_interval(secs => $4)
WHERE (s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3))
WHERE (CASE WHEN s.next_retry_at IS NOT NULL THEN s.next_retry_at <= now()
ELSE s.last_synced_at IS NULL OR s.last_synced_at < now() - make_interval(secs => $3) END)
AND (s.sync_lease_expires IS NULL OR s.sync_lease_expires < now())
RETURNING s.etag
`, remoteName, owner, freshness.Seconds(), lease.Seconds())
@@ -60,14 +75,27 @@ func (db *DB) ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string
return true, etag, nil
}
// ReleaseGitHubSyncLease records the completed scan and frees the lease. Only the
// owning replica may release; last_synced_at advances so the next poll waits a
// full freshness window, and etag is persisted for the next conditional request.
func (db *DB) ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error {
// releaseSyncLease records a scan outcome and frees the lease; only the owning
// replica may release. Success advances last_synced_at, persists the etag and
// clears any retry. Failure leaves last_synced_at and etag untouched (the last
// good metadata keeps serving) and schedules next_retry_at with exponential
// backoff from the consecutive-failure count, never before res.RetryAt.
func (db *DB) releaseSyncLease(ctx context.Context, table, remoteName, owner string, res provider.SyncResult) error {
var retryAt *time.Time
if !res.RetryAt.IsZero() {
retryAt = &res.RetryAt
}
_, err := db.Pool.Exec(ctx, `
UPDATE github_rpm_sync_state
SET last_synced_at = $3, etag = $4, sync_lease_owner = '', sync_lease_expires = NULL
UPDATE `+table+`
SET last_synced_at = CASE WHEN $3 THEN last_synced_at ELSE now() END,
etag = CASE WHEN $3 THEN etag ELSE $4 END,
sync_failures = CASE WHEN $3 THEN sync_failures + 1 ELSE 0 END,
next_retry_at = CASE WHEN $3 THEN GREATEST(
now() + make_interval(secs => LEAST($5 * power(2, LEAST(sync_failures, 20)), $6)),
$7::timestamptz)
END,
sync_lease_owner = '', sync_lease_expires = NULL
WHERE remote_name = $1 AND sync_lease_owner = $2
`, remoteName, owner, syncedAt, etag)
`, remoteName, owner, res.Failed, res.Etag, res.Backoff.Seconds(), res.MaxBackoff.Seconds(), retryAt)
return err
}
+115 -1
View File
@@ -4,6 +4,7 @@ import (
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -47,7 +48,7 @@ func TestGitHubSyncLease(t *testing.T) {
}
// Replica 1 finishes: record the sync and persist an etag.
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", `"etag-1"`, time.Now()); err != nil {
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "replica-1", provider.SyncResult{Etag: `"etag-1"`}); err != nil {
t.Fatalf("release: %v", err)
}
@@ -93,3 +94,116 @@ func TestListGitHubRPMRemotes(t *testing.T) {
t.Fatalf("seeded remote %q not returned", name)
}
}
// A failed scan must not count as a sync: the etag and last_synced_at stay put,
// every claim (prime included) is held off until the backoff elapses, the delay
// doubles per consecutive failure up to the cap, an upstream retry hint pushes
// it later, and a success clears the retry state.
func TestGitHubSyncLeaseFailureBackoff(t *testing.T) {
requireDB(t)
name := "gh-retry-" + time.Now().Format("150405.000000")
seedGitHubRPMRemote(t, name)
const lease = 15 * time.Minute
freshness := time.Hour
claim := func(f time.Duration) (bool, string) {
t.Helper()
ok, etag, err := testDB.ClaimGitHubSyncLease(ctx(), name, "r1", f, lease)
if err != nil {
t.Fatalf("claim: %v", err)
}
return ok, etag
}
release := func(res provider.SyncResult) {
t.Helper()
if err := testDB.ReleaseGitHubSyncLease(ctx(), name, "r1", res); err != nil {
t.Fatalf("release: %v", err)
}
}
state := func() (failures int, retryIn time.Duration, synced *time.Time, etag string) {
t.Helper()
var next *time.Time
var dbNow time.Time
if err := testDB.Pool.QueryRow(ctx(), `SELECT sync_failures, next_retry_at, last_synced_at, etag, now() FROM github_rpm_sync_state WHERE remote_name = $1`, name).
Scan(&failures, &next, &synced, &etag, &dbNow); err != nil {
t.Fatalf("read state: %v", err)
}
if next != nil {
retryIn = next.Sub(dbNow)
}
return
}
elapse := func() {
t.Helper()
if _, err := testDB.Pool.Exec(ctx(), `UPDATE github_rpm_sync_state SET next_retry_at = now() - interval '1 second' WHERE remote_name = $1`, name); err != nil {
t.Fatalf("elapse backoff: %v", err)
}
}
failed := provider.SyncResult{Failed: true, Etag: `"ignored"`, Backoff: time.Minute, MaxBackoff: 3 * time.Minute}
// Last good sync.
if ok, _ := claim(freshness); !ok {
t.Fatal("first claim failed")
}
release(provider.SyncResult{Etag: `"good"`})
_, _, goodSynced, _ := state()
// First failure: 60s backoff, sync time and etag untouched.
if ok, _ := claim(0); !ok {
t.Fatal("prime claim failed")
}
release(failed)
n, in, synced, etag := state()
if n != 1 || in < 55*time.Second || in > 65*time.Second {
t.Fatalf("after 1st failure: failures=%d retry in %v, want 1 and ~60s", n, in)
}
if etag != `"good"` || synced == nil || !synced.Equal(*goodSynced) {
t.Fatalf("failed scan overwrote sync state: etag=%q synced=%v", etag, synced)
}
if ok, _ := claim(0); ok {
t.Fatal("prime claimed inside the retry backoff")
}
// Backoff elapsed: claimable despite being inside the freshness window,
// and the second failure doubles the delay.
elapse()
if ok, _ := claim(freshness); !ok {
t.Fatal("periodic claim refused after backoff elapsed")
}
release(failed)
if n, in, _, _ := state(); n != 2 || in < 115*time.Second || in > 125*time.Second {
t.Fatalf("after 2nd failure: failures=%d retry in %v, want 2 and ~120s", n, in)
}
// Third failure caps at MaxBackoff (3m, not 4m).
elapse()
claim(freshness)
release(failed)
if n, in, _, _ := state(); n != 3 || in < 175*time.Second || in > 185*time.Second {
t.Fatalf("after 3rd failure: failures=%d retry in %v, want 3 and ~180s", n, in)
}
// An upstream rate-limit reset later than the backoff wins.
elapse()
claim(freshness)
hinted := failed
hinted.RetryAt = time.Now().Add(20 * time.Minute)
release(hinted)
if _, in, _, _ := state(); in < 19*time.Minute || in > 21*time.Minute {
t.Fatalf("retry hint ignored: retry in %v, want ~20m", in)
}
// Recovery: success persists the new etag and clears the retry, so the
// normal freshness window applies again.
elapse()
if ok, etag := claim(freshness); !ok || etag != `"good"` {
t.Fatalf("recovery claim ok=%v etag=%q, want last good etag", ok, etag)
}
release(provider.SyncResult{Etag: `"new"`})
if n, in, _, etag := state(); n != 0 || in != 0 || etag != `"new"` {
t.Fatalf("after success: failures=%d retry in %v etag=%q", n, in, etag)
}
if ok, _ := claim(freshness); ok {
t.Fatal("claimed inside freshness window after a successful sync")
}
}
+44
View File
@@ -0,0 +1,44 @@
package database
import (
"context"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func (db *DB) InsertHelmMetadata(ctx context.Context, meta *provider.HelmMetadata) error {
_, err := db.Pool.Exec(ctx, `
INSERT INTO helm_metadata (repo_name, file_path, content_hash, name, version, chart_yaml)
VALUES ($1,$2,$3,$4,$5,$6)
ON CONFLICT (repo_name, file_path) DO NOTHING
`, meta.RepoName, meta.FilePath, meta.ContentHash, meta.Name, meta.Version, meta.ChartYAML)
return err
}
func (db *DB) DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error {
_, err := db.Pool.Exec(ctx, `DELETE FROM helm_metadata WHERE repo_name = $1 AND file_path = $2`, repoName, filePath)
return err
}
func (db *DB) ListHelmMetadataEntries(ctx context.Context, repoName string) ([]provider.HelmMetadata, error) {
rows, err := db.Pool.Query(ctx, `
SELECT repo_name, file_path, content_hash, name, version, chart_yaml, created_at
FROM helm_metadata
WHERE repo_name = $1
ORDER BY name, version, file_path
`, repoName)
if err != nil {
return nil, err
}
defer rows.Close()
var result []provider.HelmMetadata
for rows.Next() {
var m provider.HelmMetadata
if err := rows.Scan(&m.RepoName, &m.FilePath, &m.ContentHash, &m.Name, &m.Version, &m.ChartYAML, &m.CreatedAt); err != nil {
return nil, err
}
result = append(result, m)
}
return result, rows.Err()
}
+1 -1
View File
@@ -470,7 +470,7 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote
return nil, "", false, err
}
if resp.StatusCode != http.StatusOK {
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
return nil, "", false, provider.NewUpstreamStatusError("github releases API "+u, resp)
}
if page == 1 {
newEtag = respEtag
+7 -9
View File
@@ -28,7 +28,7 @@ type SyncStore interface {
provider.RemoteMetadataStore
ListGitHubAlpineRemotes(ctx context.Context) ([]models.Remote, error)
ClaimGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
ReleaseGitHubAlpineSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
}
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
@@ -189,10 +189,11 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
s.mu.Unlock()
}()
freshness := time.Duration(job.remote.MutableTTL) * time.Second
if freshness <= 0 {
freshness = defaultSyncFreshness
ttl := time.Duration(job.remote.MutableTTL) * time.Second
if ttl <= 0 {
ttl = defaultSyncFreshness
}
freshness := ttl
if job.prime {
freshness = 0
}
@@ -210,16 +211,13 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
defer cancel()
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
releaseEtag := etag
if scanErr == nil {
releaseEtag = newEtag
} else {
if scanErr != nil {
slog.Error("github_alpine syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
}
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer relCancel()
if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
if err := s.store.ReleaseGitHubAlpineSyncLease(relCtx, job.remote.Name, s.owner, provider.NewSyncResult(newEtag, scanErr, ttl)); err != nil {
slog.Warn("github_alpine syncer: release lease", "remote", job.remote.Name, "error", err)
}
+13 -3
View File
@@ -27,6 +27,7 @@ type fakeSyncStore struct {
leaseExp map[string]time.Time
lastSynced map[string]time.Time
etags map[string]string
retryAt map[string]time.Time
}
func newFakeSyncStore() *fakeSyncStore {
@@ -36,6 +37,7 @@ func newFakeSyncStore() *fakeSyncStore {
leaseExp: map[string]time.Time{},
lastSynced: map[string]time.Time{},
etags: map[string]string{},
retryAt: map[string]time.Time{},
}
}
@@ -52,6 +54,9 @@ func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owne
ls, hasLS := f.lastSynced[name]
exp, hasExp := f.leaseExp[name]
freshOK := !hasLS || now.Sub(ls) >= freshness
if ra, pending := f.retryAt[name]; pending {
freshOK = !now.Before(ra)
}
leaseOK := !hasExp || exp.Before(now)
if freshOK && leaseOK {
f.leaseOwner[name] = owner
@@ -61,14 +66,19 @@ func (f *fakeSyncStore) ClaimGitHubAlpineSyncLease(_ context.Context, name, owne
return false, "", nil
}
func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
func (f *fakeSyncStore) ReleaseGitHubAlpineSyncLease(_ context.Context, name, owner string, res provider.SyncResult) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.leaseOwner[name] != owner {
return nil
}
f.lastSynced[name] = syncedAt
f.etags[name] = etag
if res.Failed {
f.retryAt[name] = time.Now().Add(res.Backoff)
} else {
f.lastSynced[name] = time.Now()
f.etags[name] = res.Etag
delete(f.retryAt, name)
}
delete(f.leaseOwner, name)
delete(f.leaseExp, name)
return nil
+1 -1
View File
@@ -431,7 +431,7 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote
return nil, "", false, err
}
if resp.StatusCode != http.StatusOK {
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
return nil, "", false, provider.NewUpstreamStatusError("github releases API "+u, resp)
}
if page == 1 {
newEtag = respEtag
+7 -9
View File
@@ -28,7 +28,7 @@ type SyncStore interface {
provider.RemoteMetadataStore
ListGitHubDebRemotes(ctx context.Context) ([]models.Remote, error)
ClaimGitHubDebSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
ReleaseGitHubDebSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
}
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
@@ -189,10 +189,11 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
s.mu.Unlock()
}()
freshness := time.Duration(job.remote.MutableTTL) * time.Second
if freshness <= 0 {
freshness = defaultSyncFreshness
ttl := time.Duration(job.remote.MutableTTL) * time.Second
if ttl <= 0 {
ttl = defaultSyncFreshness
}
freshness := ttl
if job.prime {
freshness = 0
}
@@ -210,16 +211,13 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
defer cancel()
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
releaseEtag := etag
if scanErr == nil {
releaseEtag = newEtag
} else {
if scanErr != nil {
slog.Error("github_deb syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
}
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer relCancel()
if err := s.store.ReleaseGitHubDebSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
if err := s.store.ReleaseGitHubDebSyncLease(relCtx, job.remote.Name, s.owner, provider.NewSyncResult(newEtag, scanErr, ttl)); err != nil {
slog.Warn("github_deb syncer: release lease", "remote", job.remote.Name, "error", err)
}
+13 -3
View File
@@ -27,6 +27,7 @@ type fakeSyncStore struct {
leaseExp map[string]time.Time
lastSynced map[string]time.Time
etags map[string]string
retryAt map[string]time.Time
}
func newFakeSyncStore() *fakeSyncStore {
@@ -36,6 +37,7 @@ func newFakeSyncStore() *fakeSyncStore {
leaseExp: map[string]time.Time{},
lastSynced: map[string]time.Time{},
etags: map[string]string{},
retryAt: map[string]time.Time{},
}
}
@@ -52,6 +54,9 @@ func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner s
ls, hasLS := f.lastSynced[name]
exp, hasExp := f.leaseExp[name]
freshOK := !hasLS || now.Sub(ls) >= freshness
if ra, pending := f.retryAt[name]; pending {
freshOK = !now.Before(ra)
}
leaseOK := !hasExp || exp.Before(now)
if freshOK && leaseOK {
f.leaseOwner[name] = owner
@@ -61,14 +66,19 @@ func (f *fakeSyncStore) ClaimGitHubDebSyncLease(_ context.Context, name, owner s
return false, "", nil
}
func (f *fakeSyncStore) ReleaseGitHubDebSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
func (f *fakeSyncStore) ReleaseGitHubDebSyncLease(_ context.Context, name, owner string, res provider.SyncResult) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.leaseOwner[name] != owner {
return nil
}
f.lastSynced[name] = syncedAt
f.etags[name] = etag
if res.Failed {
f.retryAt[name] = time.Now().Add(res.Backoff)
} else {
f.lastSynced[name] = time.Now()
f.etags[name] = res.Etag
delete(f.retryAt, name)
}
delete(f.leaseOwner, name)
delete(f.leaseExp, name)
return nil
+175
View File
@@ -1,12 +1,23 @@
package helm
import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"path"
"strings"
"time"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/provider"
"git.unkin.net/unkin/artifactapi/internal/storage"
"git.unkin.net/unkin/artifactapi/pkg/models"
)
@@ -56,3 +67,167 @@ func (p *Provider) RewriteResponse(body []byte, remote models.Remote, proxyBaseU
func (p *Provider) AuthHeaders(_ context.Context, remote models.Remote) (http.Header, error) {
return auth.BasicHeaders(remote), nil
}
// ValidateUpload stores charts flat by filename so index.yaml can reference
// them with relative urls, as `helm repo index` does.
func (p *Provider) ValidateUpload(filePath string) (storagePath, contentType string, err error) {
filename := path.Base(path.Clean("/" + filePath))
if !strings.HasSuffix(filename, ".tgz") || filename == ".tgz" {
return "", "", fmt.Errorf("file must be a .tgz chart package")
}
return filename, "application/gzip", nil
}
func (p *Provider) UploadResponse(storagePath, contentHash string, sizeBytes int64) map[string]any {
return map[string]any{
"filename": storagePath,
"content_hash": contentHash,
"size_bytes": sizeBytes,
}
}
func (p *Provider) AfterUpload(ctx context.Context, repoName, storagePath, contentHash string, blobs provider.BlobReader, db provider.MetadataStore) {
reader, _, err := blobs.Download(ctx, storage.BlobKey(strings.TrimPrefix(contentHash, "sha256:")))
if err != nil {
slog.Error("helm metadata: download failed", "repo", repoName, "path", storagePath, "error", err)
return
}
defer func() { _ = reader.Close() }()
meta, err := parseChart(reader)
if err != nil {
slog.Error("helm metadata: parse failed", "repo", repoName, "path", storagePath, "error", err)
return
}
meta.RepoName = repoName
meta.FilePath = storagePath
meta.ContentHash = contentHash
store, ok := db.(provider.HelmMetadataStore)
if !ok {
slog.Error("helm metadata: store does not support helm metadata", "repo", repoName)
return
}
if err := store.InsertHelmMetadata(ctx, meta); err != nil {
slog.Error("helm metadata: insert failed", "repo", repoName, "path", storagePath, "error", err)
return
}
slog.Info("helm metadata: parsed", "repo", repoName, "name", meta.Name, "version", meta.Version)
}
func (p *Provider) AfterDelete(ctx context.Context, repoName, storagePath string, db provider.MetadataDeleter) error {
deleter, ok := db.(provider.HelmMetadataDeleter)
if !ok {
return nil
}
return deleter.DeleteHelmMetadata(ctx, repoName, storagePath)
}
func isIndexPath(p string) bool {
return strings.TrimPrefix(path.Clean("/"+p), "/") == "index.yaml"
}
func (p *Provider) ServeLocalIndex(w http.ResponseWriter, r *http.Request, files provider.FileStore, repoName, reqPath string) bool {
if !isIndexPath(reqPath) {
return false
}
body, err := p.GenerateLocalIndex(r.Context(), files, repoName, reqPath)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return true
}
w.Header().Set("Content-Type", "text/yaml")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(body)
return true
}
func (p *Provider) GenerateLocalIndex(ctx context.Context, files provider.FileStore, repoName, reqPath string) ([]byte, error) {
if !isIndexPath(reqPath) {
return nil, fmt.Errorf("unsupported index path: %q", reqPath)
}
reader, ok := files.(provider.HelmMetadataReader)
if !ok {
return nil, errors.New("helm metadata not available")
}
metas, err := reader.ListHelmMetadataEntries(ctx, repoName)
if err != nil {
return nil, err
}
return generateIndex(metas)
}
// generateIndex builds index.yaml from stored Chart.yaml documents; it is
// stamped from the newest upload so every replica serves identical bytes.
func generateIndex(metas []provider.HelmMetadata) ([]byte, error) {
entries := map[string][]map[string]yaml.Node{}
var newest time.Time
for _, m := range metas {
var chart map[string]yaml.Node
if err := yaml.Unmarshal([]byte(m.ChartYAML), &chart); err != nil {
return nil, fmt.Errorf("chart %s: %w", m.FilePath, err)
}
if chart == nil {
chart = map[string]yaml.Node{}
}
chart["name"] = node(m.Name)
chart["version"] = node(m.Version)
chart["urls"] = node([]string{m.FilePath})
chart["digest"] = node(strings.TrimPrefix(m.ContentHash, "sha256:"))
chart["created"] = node(m.CreatedAt.UTC().Format(time.RFC3339Nano))
entries[m.Name] = append(entries[m.Name], chart)
if m.CreatedAt.After(newest) {
newest = m.CreatedAt
}
}
return yaml.Marshal(map[string]any{
"apiVersion": "v1",
"entries": entries,
"generated": newest.UTC().Format(time.RFC3339Nano),
})
}
func node(v any) yaml.Node {
var n yaml.Node
_ = n.Encode(v)
return n
}
const maxChartYAML = 1 << 20
// parseChart reads the top-level <chart>/Chart.yaml from a packaged chart.
func parseChart(r io.Reader) (*provider.HelmMetadata, error) {
zr, err := gzip.NewReader(r)
if err != nil {
return nil, err
}
tr := tar.NewReader(zr)
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil, errors.New("no Chart.yaml in chart archive")
}
if err != nil {
return nil, err
}
dir, file := path.Split(path.Clean(hdr.Name))
if file != "Chart.yaml" || strings.Count(dir, "/") != 1 {
continue
}
raw, err := io.ReadAll(io.LimitReader(tr, maxChartYAML))
if err != nil {
return nil, err
}
var chart struct {
Name string `yaml:"name"`
Version string `yaml:"version"`
}
if err := yaml.Unmarshal(raw, &chart); err != nil {
return nil, fmt.Errorf("parse Chart.yaml: %w", err)
}
if chart.Name == "" || chart.Version == "" {
return nil, errors.New("chart metadata missing name or version")
}
return &provider.HelmMetadata{Name: chart.Name, Version: chart.Version, ChartYAML: string(raw)}, nil
}
}
+162
View File
@@ -0,0 +1,162 @@
package helm
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"io"
"net/http/httptest"
"testing"
"time"
"gopkg.in/yaml.v3"
"git.unkin.net/unkin/artifactapi/internal/provider"
)
func chartTgz(t *testing.T, files map[string]string) []byte {
t.Helper()
var buf bytes.Buffer
zw := gzip.NewWriter(&buf)
tw := tar.NewWriter(zw)
for name, body := range files {
if err := tw.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(body))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write([]byte(body)); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := zw.Close(); err != nil {
t.Fatal(err)
}
return buf.Bytes()
}
type fakeBlobs struct{ data []byte }
func (f fakeBlobs) Download(context.Context, string) (io.ReadCloser, int64, error) {
return io.NopCloser(bytes.NewReader(f.data)), int64(len(f.data)), nil
}
type fakeStore struct {
inserted *provider.HelmMetadata
metas []provider.HelmMetadata
}
func (f *fakeStore) InsertRPMMetadata(context.Context, *provider.RPMMetadata) error { return nil }
func (f *fakeStore) InsertDebMetadata(context.Context, *provider.DebMetadata) error { return nil }
func (f *fakeStore) InsertHelmMetadata(_ context.Context, m *provider.HelmMetadata) error {
f.inserted = m
return nil
}
func (f *fakeStore) ListHelmMetadataEntries(context.Context, string) ([]provider.HelmMetadata, error) {
return f.metas, nil
}
func (f *fakeStore) ListFilesByPrefix(context.Context, string, string) ([]provider.FileEntry, error) {
return nil, nil
}
func (f *fakeStore) ListPackages(context.Context, string) ([]string, error) { return nil, nil }
func TestValidateUpload(t *testing.T) {
p := &Provider{}
for in, want := range map[string]string{
"mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
"charts/mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
"../../mychart-0.1.0.tgz": "mychart-0.1.0.tgz",
} {
sp, ct, err := p.ValidateUpload(in)
if err != nil || sp != want || ct != "application/gzip" {
t.Errorf("ValidateUpload(%q) = %q %q %v", in, sp, ct, err)
}
}
for _, bad := range []string{"index.yaml", "chart.tar.gz", ".tgz", ""} {
if _, _, err := p.ValidateUpload(bad); err == nil {
t.Errorf("ValidateUpload(%q) accepted", bad)
}
}
}
func TestAfterUploadParsesTopLevelChart(t *testing.T) {
data := chartTgz(t, map[string]string{
"mychart/charts/sub/Chart.yaml": "apiVersion: v2\nname: sub\nversion: 9.9.9\n",
"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\nversion: 1.0\ndescription: hi\n",
"mychart/values.yaml": "a: 1\n",
})
store := &fakeStore{}
(&Provider{}).AfterUpload(context.Background(), "r", "mychart-1.0.tgz", "sha256:ab", fakeBlobs{data}, store)
m := store.inserted
if m == nil || m.Name != "mychart" || m.Version != "1.0" || m.FilePath != "mychart-1.0.tgz" || m.ContentHash != "sha256:ab" {
t.Fatalf("inserted %+v", m)
}
}
func TestAfterUploadRejectsChartWithoutMetadata(t *testing.T) {
store := &fakeStore{}
data := chartTgz(t, map[string]string{"mychart/Chart.yaml": "apiVersion: v2\nname: mychart\n"})
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{data}, store)
if store.inserted != nil {
t.Fatal("chart without version was recorded")
}
(&Provider{}).AfterUpload(context.Background(), "r", "x.tgz", "sha256:ab", fakeBlobs{[]byte("not gzip")}, store)
if store.inserted != nil {
t.Fatal("non-gzip upload was recorded")
}
}
func TestServeLocalIndex(t *testing.T) {
t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
t2 := t1.Add(time.Hour)
store := &fakeStore{metas: []provider.HelmMetadata{
{FilePath: "mychart-1.0.tgz", ContentHash: "sha256:aa", Name: "mychart", Version: "1.0", CreatedAt: t1,
ChartYAML: "apiVersion: v2\nname: mychart\nversion: 1.0\nappVersion: \"2\"\ndescription: hi\n"},
{FilePath: "other-0.2.0.tgz", ContentHash: "sha256:bb", Name: "other", Version: "0.2.0", CreatedAt: t2,
ChartYAML: "apiVersion: v2\nname: other\nversion: 0.2.0\n"},
}}
p := &Provider{}
if p.ServeLocalIndex(httptest.NewRecorder(), httptest.NewRequest("GET", "/", nil), store, "r", "mychart-1.0.tgz") {
t.Fatal("chart path handled as index")
}
rec := httptest.NewRecorder()
if !p.ServeLocalIndex(rec, httptest.NewRequest("GET", "/", nil), store, "r", "./index.yaml") {
t.Fatal("index.yaml not handled")
}
if rec.Code != 200 || rec.Header().Get("Content-Type") != "text/yaml" {
t.Fatalf("code=%d ct=%q", rec.Code, rec.Header().Get("Content-Type"))
}
var idx struct {
APIVersion string `yaml:"apiVersion"`
Generated string `yaml:"generated"`
Entries map[string][]map[string]any
}
if err := yaml.Unmarshal(rec.Body.Bytes(), &idx); err != nil {
t.Fatal(err)
}
if idx.APIVersion != "v1" || idx.Generated != t2.Format(time.RFC3339Nano) {
t.Errorf("header %+v", idx)
}
e := idx.Entries["mychart"][0]
if e["digest"] != "aa" || e["description"] != "hi" || e["appVersion"] != "2" || e["apiVersion"] != "v2" {
t.Errorf("entry %v", e)
}
if urls, _ := e["urls"].([]any); len(urls) != 1 || urls[0] != "mychart-1.0.tgz" {
t.Errorf("urls %v", e["urls"])
}
if e["version"] != "1.0" || e["name"] != "mychart" {
t.Errorf("version must stay a string: %#v", e["version"])
}
if len(idx.Entries["other"]) != 1 {
t.Errorf("other entries %v", idx.Entries["other"])
}
again, err := p.GenerateLocalIndex(context.Background(), store, "r", "index.yaml")
if err != nil || !bytes.Equal(again, rec.Body.Bytes()) {
t.Errorf("index not deterministic: %v", err)
}
}
+24
View File
@@ -240,3 +240,27 @@ func Get(t models.PackageType) (Provider, error) {
func All() map[models.PackageType]Provider {
return registry
}
type HelmMetadataStore interface {
InsertHelmMetadata(ctx context.Context, meta *HelmMetadata) error
}
type HelmMetadataDeleter interface {
DeleteHelmMetadata(ctx context.Context, repoName, filePath string) error
}
type HelmMetadataReader interface {
ListHelmMetadataEntries(ctx context.Context, repoName string) ([]HelmMetadata, error)
}
// HelmMetadata is a local chart's Chart.yaml, kept verbatim so index.yaml
// entries carry every field helm reads.
type HelmMetadata struct {
RepoName string
FilePath string
ContentHash string
Name string
Version string
ChartYAML string
CreatedAt time.Time
}
+1 -1
View File
@@ -447,7 +447,7 @@ func (p *GitHubProvider) fetchReleases(ctx context.Context, remote models.Remote
return nil, "", false, err
}
if resp.StatusCode != http.StatusOK {
return nil, "", false, fmt.Errorf("github releases API %s: status %d", u, resp.StatusCode)
return nil, "", false, provider.NewUpstreamStatusError("github releases API "+u, resp)
}
if page == 1 {
newEtag = respEtag
+9
View File
@@ -78,6 +78,7 @@ type githubFixture struct {
notModHit int // releases-list requests answered 304
releaseAuth string // Authorization header seen on the last releases request
assetAuth string // Authorization header seen on the last asset request
failStatus int // when set, the releases list answers this status (rate-limit style)
mu sync.Mutex
}
@@ -100,6 +101,14 @@ func newGitHubFixture(t *testing.T, withDigest bool) *githubFixture {
f.mu.Lock()
f.releasesHit++
f.releaseAuth = r.Header.Get("Authorization")
if f.failStatus != 0 {
status := f.failStatus
f.mu.Unlock()
w.Header().Set("X-RateLimit-Remaining", "0")
w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(30*time.Second).Unix(), 10))
http.Error(w, "API rate limit exceeded", status)
return
}
etag := f.etag
if etag != "" && r.Header.Get("If-None-Match") == etag {
f.notModHit++
+8 -10
View File
@@ -35,7 +35,7 @@ type SyncStore interface {
provider.RemoteMetadataStore
ListGitHubRPMRemotes(ctx context.Context) ([]models.Remote, error)
ClaimGitHubSyncLease(ctx context.Context, remoteName, owner string, freshness, lease time.Duration) (claimed bool, etag string, err error)
ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner, etag string, syncedAt time.Time) error
ReleaseGitHubSyncLease(ctx context.Context, remoteName, owner string, res provider.SyncResult) error
}
// SyncConfig tunes the shared syncer. Zero values fall back to safe defaults.
@@ -205,10 +205,11 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
s.mu.Unlock()
}()
freshness := time.Duration(job.remote.MutableTTL) * time.Second
if freshness <= 0 {
freshness = defaultSyncFreshness
ttl := time.Duration(job.remote.MutableTTL) * time.Second
if ttl <= 0 {
ttl = defaultSyncFreshness
}
freshness := ttl
if job.prime {
freshness = 0 // prime ignores the recency gate but still respects a live lease
}
@@ -226,18 +227,15 @@ func (s *Syncer) process(ctx context.Context, job syncJob) {
defer cancel()
newEtag, changed, scanErr := s.prov.scanWithState(scanCtx, job.remote, s.store, etag)
releaseEtag := etag
if scanErr == nil {
releaseEtag = newEtag
} else {
if scanErr != nil {
slog.Error("github_rpm syncer: scan failed", "remote", job.remote.Name, "error", scanErr)
}
// Release on a detached context so a clean shutdown mid-scan still frees the
// lease and advances last_synced_at (otherwise it simply expires).
// lease and records the outcome (otherwise the lease simply expires).
relCtx, relCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer relCancel()
if err := s.store.ReleaseGitHubSyncLease(relCtx, job.remote.Name, s.owner, releaseEtag, time.Now()); err != nil {
if err := s.store.ReleaseGitHubSyncLease(relCtx, job.remote.Name, s.owner, provider.NewSyncResult(newEtag, scanErr, ttl)); err != nil {
slog.Warn("github_rpm syncer: release lease", "remote", job.remote.Name, "error", err)
}
+92 -3
View File
@@ -27,6 +27,8 @@ type fakeSyncStore struct {
leaseExp map[string]time.Time
lastSynced map[string]time.Time
etags map[string]string
retryAt map[string]time.Time
lastResult provider.SyncResult
}
func newFakeSyncStore() *fakeSyncStore {
@@ -36,6 +38,7 @@ func newFakeSyncStore() *fakeSyncStore {
leaseExp: map[string]time.Time{},
lastSynced: map[string]time.Time{},
etags: map[string]string{},
retryAt: map[string]time.Time{},
}
}
@@ -52,6 +55,9 @@ func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner stri
ls, hasLS := f.lastSynced[name]
exp, hasExp := f.leaseExp[name]
freshOK := !hasLS || now.Sub(ls) >= freshness
if ra, pending := f.retryAt[name]; pending {
freshOK = !now.Before(ra)
}
leaseOK := !hasExp || exp.Before(now)
if freshOK && leaseOK {
f.leaseOwner[name] = owner
@@ -61,14 +67,20 @@ func (f *fakeSyncStore) ClaimGitHubSyncLease(_ context.Context, name, owner stri
return false, "", nil
}
func (f *fakeSyncStore) ReleaseGitHubSyncLease(_ context.Context, name, owner, etag string, syncedAt time.Time) error {
func (f *fakeSyncStore) ReleaseGitHubSyncLease(_ context.Context, name, owner string, res provider.SyncResult) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.leaseOwner[name] != owner {
return nil
}
f.lastSynced[name] = syncedAt
f.etags[name] = etag
f.lastResult = res
if res.Failed {
f.retryAt[name] = time.Now().Add(res.Backoff)
} else {
f.lastSynced[name] = time.Now()
f.etags[name] = res.Etag
delete(f.retryAt, name)
}
delete(f.leaseOwner, name)
delete(f.leaseExp, name)
return nil
@@ -310,3 +322,80 @@ func TestSyncerPrimeBypassesRecencyPeriodicDoesNot(t *testing.T) {
t.Fatalf("periodic scan ran inside recency window: %d -> %d releases calls", releasesAfterPrime, fx.releasesHit)
}
}
// A rate-limited scan is not recorded as a sync: the last good repodata keeps
// serving, the retry honours the upstream reset hint and backoff, and once the
// upstream recovers the next poll after the backoff re-syncs without waiting
// out mutable_ttl.
func TestSyncerFailedScanRetriesAfterBackoff(t *testing.T) {
fx := newGitHubFixture(t, true)
fx.etag = `"v1"`
store := newFakeSyncStore()
p := newTestProvider()
s := newSyncer(store, p, testSyncConfig())
remote := fx.remote()
bg := context.Background()
s.process(bg, syncJob{remote: remote, prime: true})
if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
t.Fatalf("prime did not derive: %d rows", len(rows))
}
// mutable_ttl elapses while GitHub is rate-limiting.
fx.mu.Lock()
fx.failStatus = http.StatusForbidden
fx.etag = `"v2"`
fx.mu.Unlock()
store.mu.Lock()
store.lastSynced[remote.Name] = time.Now().Add(-2 * time.Hour)
store.mu.Unlock()
s.process(bg, syncJob{remote: remote})
res := store.lastResult
if !res.Failed {
t.Fatal("403 scan was recorded as a successful sync")
}
if until := time.Until(res.RetryAt); until < 20*time.Second || until > 40*time.Second {
t.Fatalf("retry hint from X-RateLimit-Reset not carried: retry in %v", until)
}
if res.Backoff != time.Minute || res.MaxBackoff >= time.Duration(remote.MutableTTL)*time.Second {
t.Fatalf("backoff %v cap %v, want 1m first retry capped below mutable_ttl", res.Backoff, res.MaxBackoff)
}
if store.etags[remote.Name] != `"v1"` {
t.Fatalf("failed scan replaced the etag: %q", store.etags[remote.Name])
}
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/v1/remote/acme-rpm/repodata/repomd.xml", nil)
p.ServeRemote(rec, req, remote, "repodata/repomd.xml", "https://x", store)
if rec.Code != http.StatusOK {
t.Fatalf("last good repodata not served during failure: %d %s", rec.Code, rec.Body.String())
}
if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 1 {
t.Fatalf("failed scan dropped cached metadata: %d rows", len(rows))
}
// Inside the backoff nothing is retried.
hits := fx.releasesHit
s.process(bg, syncJob{remote: remote})
if fx.releasesHit != hits {
t.Fatal("scan retried inside the backoff window")
}
// Upstream recovers and the backoff elapses: the next poll re-syncs.
fx.mu.Lock()
fx.failStatus = 0
fx.rpmBytes["other-9-9.aarch64.rpm"] = testsupport.MinimalRPM("other", "9", "9", "aarch64")
fx.mu.Unlock()
store.mu.Lock()
store.retryAt[remote.Name] = time.Now().Add(-time.Second)
store.mu.Unlock()
s.process(bg, syncJob{remote: remote})
if store.lastResult.Failed || store.etags[remote.Name] != `"v2"` {
t.Fatalf("recovery scan not recorded: failed=%v etag=%q", store.lastResult.Failed, store.etags[remote.Name])
}
if rows, _ := store.ListRPMMetadataEntries(bg, remote.Name); len(rows) != 2 {
t.Fatalf("recovery did not refresh metadata: %d rows", len(rows))
}
}
+75
View File
@@ -0,0 +1,75 @@
package provider
import (
"errors"
"fmt"
"net/http"
"strconv"
"time"
)
const (
syncRetryBase = time.Minute
syncRetryMax = 10 * time.Minute
)
// UpstreamStatusError is a non-success upstream response. RetryAt is the
// upstream's own retry hint (Retry-After, or X-RateLimit-Reset once the quota
// is exhausted); zero when it gave none.
type UpstreamStatusError struct {
URL string
Status int
RetryAt time.Time
}
func (e *UpstreamStatusError) Error() string {
return fmt.Sprintf("%s: status %d", e.URL, e.Status)
}
// NewUpstreamStatusError wraps a non-success response, capturing its retry hint.
func NewUpstreamStatusError(url string, resp *http.Response) *UpstreamStatusError {
e := &UpstreamStatusError{URL: url, Status: resp.StatusCode}
if ra := resp.Header.Get("Retry-After"); ra != "" {
if secs, err := strconv.Atoi(ra); err == nil {
e.RetryAt = time.Now().Add(time.Duration(secs) * time.Second)
} else if t, err := http.ParseTime(ra); err == nil {
e.RetryAt = t
}
} else if resp.Header.Get("X-RateLimit-Remaining") == "0" {
if reset, err := strconv.ParseInt(resp.Header.Get("X-RateLimit-Reset"), 10, 64); err == nil {
e.RetryAt = time.Unix(reset, 0)
}
}
return e
}
// SyncResult is a background scan's outcome, recorded when its sync lease is
// released. A failed scan keeps the prior sync time and ETag and schedules a
// retry after Backoff, doubled per consecutive failure up to MaxBackoff, and
// never earlier than RetryAt.
type SyncResult struct {
Etag string
Failed bool
RetryAt time.Time
Backoff time.Duration
MaxBackoff time.Duration
}
// NewSyncResult builds the result for a scan against a remote with the given
// mutable_ttl. The retry cap stays well below ttl, and an upstream hint is
// clamped to ttl so a bogus reset can never stall the remote longer than a
// normal sync interval would.
func NewSyncResult(etag string, scanErr error, ttl time.Duration) SyncResult {
if scanErr == nil {
return SyncResult{Etag: etag}
}
res := SyncResult{Failed: true, Backoff: syncRetryBase, MaxBackoff: min(syncRetryMax, max(syncRetryBase, ttl/4))}
var se *UpstreamStatusError
if errors.As(scanErr, &se) && !se.RetryAt.IsZero() {
res.RetryAt = se.RetryAt
if limit := time.Now().Add(ttl); res.RetryAt.After(limit) {
res.RetryAt = limit
}
}
return res
}
+63
View File
@@ -0,0 +1,63 @@
package provider
import (
"errors"
"fmt"
"net/http"
"strconv"
"testing"
"time"
)
func TestNewUpstreamStatusErrorRetryHint(t *testing.T) {
reset := time.Now().Add(15 * time.Minute).Truncate(time.Second)
cases := []struct {
name string
header map[string]string
want time.Duration // 0 = no hint
}{
{"retry-after seconds", map[string]string{"Retry-After": "90"}, 90 * time.Second},
{"retry-after date", map[string]string{"Retry-After": reset.UTC().Format(http.TimeFormat)}, 15 * time.Minute},
{"quota exhausted", map[string]string{"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": strconv.FormatInt(reset.Unix(), 10)}, 15 * time.Minute},
{"quota left is not a hint", map[string]string{"X-RateLimit-Remaining": "12", "X-RateLimit-Reset": strconv.FormatInt(reset.Unix(), 10)}, 0},
{"no headers", nil, 0},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
resp := &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}}
for k, v := range c.header {
resp.Header.Set(k, v)
}
e := NewUpstreamStatusError("u", resp)
if c.want == 0 {
if !e.RetryAt.IsZero() {
t.Fatalf("unexpected hint %v", e.RetryAt)
}
return
}
if d := time.Until(e.RetryAt) - c.want; d < -2*time.Second || d > 2*time.Second {
t.Fatalf("retry in %v, want ~%v", time.Until(e.RetryAt), c.want)
}
})
}
}
func TestNewSyncResult(t *testing.T) {
if r := NewSyncResult(`"e"`, nil, time.Hour); r.Failed || r.Etag != `"e"` {
t.Fatalf("success result = %+v", r)
}
r := NewSyncResult(`"e"`, errors.New("boom"), time.Hour)
if !r.Failed || r.Backoff != time.Minute || r.MaxBackoff != 10*time.Minute || !r.RetryAt.IsZero() {
t.Fatalf("plain failure = %+v, want 1m backoff capped at 10m, no hint", r)
}
if r := NewSyncResult("", errors.New("boom"), 5*time.Minute); r.MaxBackoff != time.Minute+15*time.Second {
t.Fatalf("short ttl cap = %v, want ttl/4", r.MaxBackoff)
}
far := &UpstreamStatusError{Status: 403, RetryAt: time.Now().Add(3 * time.Hour)}
r = NewSyncResult("", fmt.Errorf("scan: %w", far), time.Hour)
if until := time.Until(r.RetryAt); until > time.Hour || until < 59*time.Minute {
t.Fatalf("wrapped hint not clamped to ttl: retry in %v", until)
}
}
+1 -1
View File
@@ -12,7 +12,7 @@ func cors(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization")
if r.Method == http.MethodOptions {
w.WriteHeader(http.StatusNoContent)
return
+80
View File
@@ -0,0 +1,80 @@
package server
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"git.unkin.net/unkin/artifactapi/internal/testsupport"
)
func TestServerOIDCWriteAuth(t *testing.T) {
requireStack(t)
iss := testsupport.NewOIDCIssuer()
defer iss.Close()
cfg := *testSrv.cfg
cfg.OIDCIssuer = iss.URL
cfg.OIDCAudiences = []string{"artifactapi"}
s := *testSrv
s.cfg = &cfg
ts := httptest.NewServer(s.routes())
defer ts.Close()
good := iss.Token(nil, "artifactapi", "ci", time.Hour)
do := func(method, path, body, token string) int {
t.Helper()
rq, _ := http.NewRequest(method, ts.URL+path, strings.NewReader(body))
rq.Header.Set("Content-Type", "application/json")
if token != "" {
rq.Header.Set("Authorization", "Bearer "+token)
}
resp, err := http.DefaultClient.Do(rq)
if err != nil {
t.Fatalf("%s %s: %v", method, path, err)
}
_, _ = io.Copy(io.Discard, resp.Body)
_ = resp.Body.Close()
return resp.StatusCode
}
expect := func(want int, method, path, body, token string) {
t.Helper()
if got := do(method, path, body, token); got != want {
t.Errorf("%s %s: status %d, want %d", method, path, got, want)
}
}
remote := `{"name":"srv-oidc","package_type":"generic","repo_type":"local"}`
virtual := `{"name":"srv-oidc-v","package_type":"generic","members":["srv-oidc"]}`
expect(401, "POST", "/api/v2/remotes", remote, "")
expect(401, "POST", "/api/v2/remotes", remote, iss.Token(nil, "other", "ci", time.Hour))
expect(201, "POST", "/api/v2/remotes", remote, good)
expect(401, "PUT", "/api/v2/remotes/srv-oidc", `{"package_type":"generic"}`, "")
expect(401, "DELETE", "/api/v2/remotes/srv-oidc", "", "")
expect(401, "POST", "/api/v2/virtuals", virtual, "")
expect(201, "POST", "/api/v2/virtuals", virtual, good)
expect(401, "PUT", "/api/v2/virtuals/srv-oidc-v", virtual, "")
expect(401, "DELETE", "/api/v2/virtuals/srv-oidc-v", "", "")
expect(200, "GET", "/api/v2/remotes", "", "")
expect(200, "GET", "/api/v2/remotes/srv-oidc", "", "")
expect(200, "GET", "/api/v2/virtuals/srv-oidc-v", "", "")
expect(201, "PUT", "/api/v2/remotes/srv-oidc/files/a.bin", "payload", "")
expect(200, "GET", "/api/v2/remotes/srv-oidc/files/a.bin", "", "")
if got := do("DELETE", "/api/v2/remotes/srv-oidc/files/a.bin", "", ""); got >= 400 {
t.Errorf("delete file: status %d", got)
}
if got := do("POST", "/api/v2/probe", `{}`, ""); got == 401 {
t.Error("probe gated")
}
if got := do("GET", "/v2/", "", ""); got == 401 {
t.Error("docker /v2 gated")
}
expect(204, "DELETE", "/api/v2/virtuals/srv-oidc-v", "", good)
expect(204, "DELETE", "/api/v2/remotes/srv-oidc", "", good)
}
+4 -2
View File
@@ -15,6 +15,7 @@ import (
tfregistry "git.unkin.net/unkin/artifactapi/internal/api/terraform"
v1 "git.unkin.net/unkin/artifactapi/internal/api/v1"
v2 "git.unkin.net/unkin/artifactapi/internal/api/v2"
"git.unkin.net/unkin/artifactapi/internal/auth"
"git.unkin.net/unkin/artifactapi/internal/cache"
"git.unkin.net/unkin/artifactapi/internal/config"
"git.unkin.net/unkin/artifactapi/internal/database"
@@ -186,10 +187,11 @@ func (s *Server) routes() chi.Router {
statsHandler := v2.NewStatsHandler(s.db)
eventsHandler := v2.NewEventsHandler()
probeHandler := v2.NewProbeHandler(s.engine, s.db)
writeAuth := auth.NewBearerOIDC(s.cfg.OIDCIssuer, s.cfg.OIDCAudiences)
r.Route("/api/v2", func(r chi.Router) {
r.Mount("/remotes", remotesHandler.Routes())
r.Mount("/virtuals", virtualsHandler.Routes())
r.Mount("/remotes", writeAuth.RequireWrites(remotesHandler.Routes()))
r.Mount("/virtuals", writeAuth.RequireWrites(virtualsHandler.Routes()))
r.Mount("/health", healthHandler.Routes())
r.Mount("/stats", statsHandler.Routes())
r.Mount("/events", eventsHandler.Routes())
+73
View File
@@ -0,0 +1,73 @@
package testsupport
import (
"crypto/rand"
"crypto/rsa"
"encoding/json"
"net/http"
"net/http/httptest"
"time"
"github.com/go-jose/go-jose/v4"
"github.com/go-jose/go-jose/v4/jwt"
)
// OIDCIssuer is a minimal OIDC issuer serving discovery and JWKS, signing
// RS256 tokens with a generated key.
type OIDCIssuer struct {
*httptest.Server
key *rsa.PrivateKey
}
func NewOIDCIssuer() *OIDCIssuer {
key, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
panic(err)
}
iss := &OIDCIssuer{key: key}
mux := http.NewServeMux()
mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(map[string]any{
"issuer": iss.URL,
"jwks_uri": iss.URL + "/keys",
"id_token_signing_alg_values_supported": []string{"RS256"},
})
})
mux.HandleFunc("/keys", func(w http.ResponseWriter, r *http.Request) {
_ = json.NewEncoder(w).Encode(jose.JSONWebKeySet{Keys: []jose.JSONWebKey{
{Key: &key.PublicKey, KeyID: "k1", Algorithm: "RS256", Use: "sig"},
}})
})
iss.Server = httptest.NewServer(mux)
return iss
}
// Token signs a token for aud and sub. Pass a nil key to use the issuer's own;
// expiry is relative to now.
func (o *OIDCIssuer) Token(key *rsa.PrivateKey, aud, sub string, expiry time.Duration) string {
now := time.Now()
return o.Sign(key, jwt.Claims{
Issuer: o.URL,
Subject: sub,
Audience: jwt.Audience{aud},
IssuedAt: jwt.NewNumericDate(now.Add(-time.Minute)),
Expiry: jwt.NewNumericDate(now.Add(expiry)),
})
}
// Sign signs arbitrary claims with RS256. Pass a nil key to use the issuer's own.
func (o *OIDCIssuer) Sign(key *rsa.PrivateKey, claims jwt.Claims) string {
if key == nil {
key = o.key
}
sig, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: key},
(&jose.SignerOptions{}).WithType("JWT").WithHeader("kid", "k1"))
if err != nil {
panic(err)
}
tok, err := jwt.Signed(sig).Claims(claims).Serialize()
if err != nil {
panic(err)
}
return tok
}
+46 -39
View File
@@ -16,22 +16,17 @@ func init() {
type HelmMerger struct{}
type helmIndex struct {
APIVersion string `yaml:"apiVersion"`
Entries map[string][]helmChartVersion `yaml:"entries"`
Generated string `yaml:"generated,omitempty"`
}
type helmChartVersion struct {
Name string `yaml:"name"`
Version string `yaml:"version"`
URLs []string `yaml:"urls"`
rest map[string]any
APIVersion string `yaml:"apiVersion"`
Entries map[string][]map[string]yaml.Node `yaml:"entries"`
Generated string `yaml:"generated,omitempty"`
}
// MergeIndexes keeps each chart version's fields as raw nodes so digest,
// apiVersion and the rest survive the merge untouched; only urls are rewritten.
func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([]byte, error) {
merged := &helmIndex{
APIVersion: "v1",
Entries: make(map[string][]helmChartVersion),
Entries: make(map[string][]map[string]yaml.Node),
}
seen := map[string]map[string]bool{}
@@ -47,42 +42,26 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
seen[chart] = map[string]bool{}
}
for _, ver := range versions {
key := chart + ":" + ver.Version
if seen[chart][ver.Version] {
version := ver["version"].Value
if seen[chart][version] {
continue
}
seen[chart][ver.Version] = true
seen[chart][version] = true
if proxyBaseURL != "" {
routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
if urlsNode, ok := ver["urls"]; ok && proxyBaseURL != "" {
var urls []string
if err := urlsNode.Decode(&urls); err != nil {
continue
}
baseHost := extractHost(member.BaseURL)
for i, u := range ver.URLs {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
if baseHost != "" && extractHost(u) != baseHost {
continue
}
relPath := extractPathRelativeToBase(u, member.BaseURL)
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
relPath)
} else {
ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
u)
}
rewriteHelmURLs(urls, member, proxyBaseURL)
var n yaml.Node
if err := n.Encode(urls); err != nil {
return nil, err
}
ver["urls"] = n
}
merged.Entries[chart] = append(merged.Entries[chart], ver)
_ = key
}
}
}
@@ -90,6 +69,34 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
return yaml.Marshal(merged)
}
func rewriteHelmURLs(urls []string, member MemberIndex, proxyBaseURL string) {
routePrefix := "remote"
if member.RepoType == "local" {
routePrefix = "local"
}
baseHost := extractHost(member.BaseURL)
for i, u := range urls {
if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
if baseHost != "" && extractHost(u) != baseHost {
continue
}
relPath := extractPathRelativeToBase(u, member.BaseURL)
urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
relPath)
} else {
urls[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
strings.TrimRight(proxyBaseURL, "/"),
routePrefix,
member.RemoteName,
u)
}
}
}
func extractHost(rawURL string) string {
idx := strings.Index(rawURL, "://")
if idx == -1 {
+33
View File
@@ -122,3 +122,36 @@ entries:
t.Error("expected fallback repo to be excluded for duplicate")
}
}
func TestHelmMerger_PreservesChartFields(t *testing.T) {
m := &virtual.HelmMerger{}
body := []byte(`apiVersion: v1
entries:
app:
- apiVersion: v2
name: app
version: "1.0"
appVersion: "3.1"
description: in-house
digest: deadbeef
urls:
- app-1.0.tgz
`)
result, err := m.MergeIndexes([]virtual.MemberIndex{{RemoteName: "charts", RepoType: "local", Body: body}}, "https://proxy.example.com")
if err != nil {
t.Fatal(err)
}
out := string(result)
for _, want := range []string{
"digest: deadbeef",
"apiVersion: v2",
`appVersion: "3.1"`,
"description: in-house",
`version: "1.0"`,
"https://proxy.example.com/api/v1/local/charts/app-1.0.tgz",
} {
if !strings.Contains(out, want) {
t.Errorf("merged index missing %q:\n%s", want, out)
}
}
}
+9
View File
@@ -0,0 +1,9 @@
-- Failed GitHub release scans retry with backoff instead of waiting out a full
-- mutable_ttl: sync_failures counts consecutive failures, next_retry_at gates
-- the next claim while a retry is pending.
ALTER TABLE github_rpm_sync_state ADD COLUMN IF NOT EXISTS sync_failures INT NOT NULL DEFAULT 0;
ALTER TABLE github_rpm_sync_state ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ;
ALTER TABLE github_deb_sync_state ADD COLUMN IF NOT EXISTS sync_failures INT NOT NULL DEFAULT 0;
ALTER TABLE github_deb_sync_state ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ;
ALTER TABLE github_alpine_sync_state ADD COLUMN IF NOT EXISTS sync_failures INT NOT NULL DEFAULT 0;
ALTER TABLE github_alpine_sync_state ADD COLUMN IF NOT EXISTS next_retry_at TIMESTAMPTZ;
+13
View File
@@ -0,0 +1,13 @@
CREATE TABLE IF NOT EXISTS helm_metadata (
id BIGSERIAL PRIMARY KEY,
repo_name TEXT NOT NULL,
file_path TEXT NOT NULL,
content_hash TEXT NOT NULL,
name TEXT NOT NULL,
version TEXT NOT NULL,
chart_yaml TEXT NOT NULL,
created_at TIMESTAMPTZ DEFAULT NOW(),
UNIQUE(repo_name, file_path)
);
CREATE INDEX IF NOT EXISTS idx_helm_metadata_repo ON helm_metadata(repo_name);
+13 -2
View File
@@ -167,12 +167,23 @@ docker push ${host}/${name}/myapp:latest`,
case 'helm':
return [
{
title: 'Add the Helm repo',
title: isLocal ? 'Add the Helm repo (index.yaml auto-generated)' : 'Add the Helm repo',
language: 'bash',
code: `helm repo add ${name} ${proxy}/
code: `helm repo add ${name} ${isLocal ? `${url}/api/v1/local/${name}` : proxy}/
helm repo update
helm install <release> ${name}/<chart>`,
},
...(isLocal
? [
{
title: 'Publish a chart',
language: 'bash',
code: `helm package ./mychart
curl -fsSL --upload-file mychart-0.1.0.tgz \\
${url}/api/v2/remotes/${name}/files/mychart-0.1.0.tgz`,
},
]
: []),
];
case 'alpine':