perf: memoise regex compilation in the classifier

Classify runs on every proxied request and recompiled each remote's
pattern lists every time. Cache compiled regexes keyed by pattern text so
each distinct pattern compiles once and is reused thereafter.

Refs #73

.woodpecker/docker.yaml

@@ -8,6 +8,8 @@ steps:
     settings:
       registry: git.unkin.net
       repo: git.unkin.net/unkin/artifactapi
+      build_args:
+        VERSION: ${CI_COMMIT_TAG}
       username: droneci
       password:
         from_secret: DRONECI_PASSWORD
@@ -22,6 +24,8 @@ steps:
       repo: git.unkin.net/unkin/artifactapi-ui
       dockerfile: ui/Dockerfile.ui
       context: ui
+      build_args:
+        BASE_PATH: /ui
       username: droneci
       password:
         from_secret: DRONECI_PASSWORD

Dockerfile

@@ -9,7 +9,8 @@ RUN go mod download
 
 COPY . .
 
-RUN CGO_ENABLED=0 go build -ldflags="-s -w" -o artifactapi ./cmd/artifactapi
+ARG VERSION=dev
+RUN CGO_ENABLED=0 go build -ldflags="-s -w -X main.version=${VERSION}" -o artifactapi ./cmd/artifactapi
 
 FROM gcr.io/distroless/static-debian12:nonroot
 

Makefile

@@ -12,7 +12,7 @@ check-go:
 	fi
 
 build: check-go tidy
-	go build -ldflags="-s -w" -o $(BINARY) ./cmd/artifactapi
+	go build -ldflags="-s -w -X main.version=$(VERSION)" -o $(BINARY) ./cmd/artifactapi
 
 test: check-go
 	go test -race -count=1 ./pkg/... ./internal/...

cmd/artifactapi/main.go

@@ -13,6 +13,8 @@ import (
 	"git.unkin.net/unkin/artifactapi/internal/tui"
 )
 
+var version = "dev"
+
 func main() {
 	if len(os.Args) > 1 && os.Args[1] == "tui" {
 		endpoint := os.Getenv("ARTIFACTAPI_ENDPOINT")
@@ -42,7 +44,7 @@ func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
 	defer cancel()
 
-	srv, err := server.New(cfg)
+	srv, err := server.New(cfg, version)
 	if err != nil {
 		slog.Error("failed to create server", "error", err)
 		os.Exit(1)

internal/api/v1/proxy.go

@@ -37,6 +37,20 @@ func (h *ProxyHandler) Routes() chi.Router {
 	return r
 }
 
+func (h *ProxyHandler) DockerV2Routes() chi.Router {
+	r := chi.NewRouter()
+	r.Get("/", h.handleDockerPing)
+	r.Head("/", h.handleDockerPing)
+	r.Get("/{remoteName}/*", h.handleProxy)
+	r.Head("/{remoteName}/*", h.handleProxy)
+	return r
+}
+
+func (h *ProxyHandler) handleDockerPing(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Docker-Distribution-Api-Version", "registry/2.0")
+	w.WriteHeader(http.StatusOK)
+}
+
 func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 	remoteName := chi.URLParam(r, "remoteName")
 	path := chi.URLParam(r, "*")
@@ -53,7 +67,7 @@ func (h *ProxyHandler) handleProxy(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	result, err := h.engine.Fetch(r.Context(), *remote, path, prov)
+	result, err := h.engine.Fetch(r.Context(), *remote, path, prov, r.Header)
 	if err != nil {
 		var proxyErr *proxy.ProxyError
 		if errors.As(err, &proxyErr) {

internal/database/stats.go

@@ -30,6 +30,15 @@ func (db *DB) GetOverviewStats(ctx context.Context) (*models.OverviewStats, erro
 		return nil, err
 	}
 
+	err = db.Pool.QueryRow(ctx, `
+		SELECT COALESCE(SUM(size_bytes), 0)
+		FROM access_log
+		WHERE cache_hit = TRUE AND created_at > NOW() - INTERVAL '30 days'
+	`).Scan(&stats.BandwidthSaved30d)
+	if err != nil {
+		return nil, err
+	}
+
 	return &stats, nil
 }
 

internal/proxy/classifier.go

@@ -2,6 +2,7 @@ package proxy
 
 import (
 	"regexp"
+	"sync"
 
 	"git.unkin.net/unkin/artifactapi/internal/provider"
 	"git.unkin.net/unkin/artifactapi/pkg/models"
@@ -60,10 +61,29 @@ func (c *Classifier) Classify(remote models.Remote, path string) Classification
 	return ClassImmutable
 }
 
+// patternCache memoises regex compilation. Classify runs on every proxied
+// request and previously recompiled each remote's pattern lists every time;
+// keying by the pattern string lets each distinct pattern compile once and
+// then be reused, with no invalidation needed (the pattern text is the key).
+// A pattern that fails to compile is cached as a typed nil so we don't retry.
+var patternCache sync.Map // map[string]*regexp.Regexp
+
+func compileCached(pattern string) *regexp.Regexp {
+	if v, ok := patternCache.Load(pattern); ok {
+		return v.(*regexp.Regexp)
+	}
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		re = nil
+	}
+	patternCache.Store(pattern, re)
+	return re
+}
+
 func compilePatterns(patterns []string) []*regexp.Regexp {
 	compiled := make([]*regexp.Regexp, 0, len(patterns))
 	for _, p := range patterns {
-		if re, err := regexp.Compile(p); err == nil {
+		if re := compileCached(p); re != nil {
 			compiled = append(compiled, re)
 		}
 	}

internal/proxy/engine.go

@@ -4,10 +4,12 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
+	"strings"
 	"time"
 
 	"git.unkin.net/unkin/artifactapi/internal/cache"
@@ -42,7 +44,7 @@ type FetchResult struct {
 	Source      string // "cache" or "remote"
 }
 
-func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, prov provider.Provider) (*FetchResult, error) {
+func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, prov provider.Provider, clientHeaders ...http.Header) (*FetchResult, error) {
 	classifier := NewClassifier(prov)
 	class := classifier.Classify(remote, path)
 
@@ -103,8 +105,13 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
 		}
 	}
 
+	var fwdHeaders http.Header
+	if len(clientHeaders) > 0 && clientHeaders[0] != nil {
+		fwdHeaders = clientHeaders[0]
+	}
+
 	start := time.Now()
-	result, err := e.fetchFromUpstream(ctx, remote, path, prov, class, ttl)
+	result, err := e.fetchFromUpstream(ctx, remote, path, prov, class, ttl, fwdHeaders)
 	upstreamMS := int(time.Since(start).Milliseconds())
 	if err != nil {
 		if remote.StaleOnError && isNetworkError(err) {
@@ -124,7 +131,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
 	return result, nil
 }
 
-func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration) (*FetchResult, error) {
+func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, path string, prov provider.Provider, class Classification, ttl time.Duration, clientHeaders http.Header) (*FetchResult, error) {
 	url := prov.UpstreamURL(remote, path)
 
 	authHeaders, err := prov.AuthHeaders(ctx, remote)
@@ -141,12 +148,37 @@ func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, pa
 			req.Header.Add(k, v)
 		}
 	}
+	if clientHeaders != nil {
+		if accept := clientHeaders.Get("Accept"); accept != "" {
+			req.Header.Set("Accept", accept)
+		}
+	}
 
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
 		return nil, &UpstreamError{Err: err}
 	}
 
+	if resp.StatusCode == http.StatusUnauthorized {
+		resp.Body.Close()
+		token, err := fetchBearerToken(ctx, resp.Header.Get("Www-Authenticate"), remote)
+		if err == nil && token != "" {
+			req2, _ := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+			req2.Header.Set("Authorization", "Bearer "+token)
+			if clientHeaders != nil {
+				if accept := clientHeaders.Get("Accept"); accept != "" {
+					req2.Header.Set("Accept", accept)
+				}
+			}
+			resp, err = http.DefaultClient.Do(req2)
+			if err != nil {
+				return nil, &UpstreamError{Err: err}
+			}
+		} else {
+			return nil, &ProxyError{Status: http.StatusUnauthorized, Message: "upstream returned 401"}
+		}
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		resp.Body.Close()
 		return nil, &ProxyError{Status: resp.StatusCode, Message: fmt.Sprintf("upstream returned %d", resp.StatusCode)}
@@ -167,7 +199,7 @@ func (e *Engine) fetchFromUpstream(ctx context.Context, remote models.Remote, pa
 	}
 
 	contentType := prov.ContentType(path)
-	if ct := resp.Header.Get("Content-Type"); ct != "" && contentType == "application/octet-stream" {
+	if ct := resp.Header.Get("Content-Type"); ct != "" {
 		contentType = ct
 	}
 
@@ -319,6 +351,71 @@ func (r readerAt) ReadAt(p []byte, off int64) (n int, err error) {
 	return
 }
 
+func fetchBearerToken(ctx context.Context, wwwAuth string, remote models.Remote) (string, error) {
+	if !strings.HasPrefix(wwwAuth, "Bearer ") {
+		return "", fmt.Errorf("not a Bearer challenge")
+	}
+
+	params := map[string]string{}
+	for _, part := range strings.Split(wwwAuth[7:], ",") {
+		part = strings.TrimSpace(part)
+		eq := strings.Index(part, "=")
+		if eq < 0 {
+			continue
+		}
+		key := part[:eq]
+		val := strings.Trim(part[eq+1:], `"`)
+		params[key] = val
+	}
+
+	realm := params["realm"]
+	if realm == "" {
+		return "", fmt.Errorf("no realm in Bearer challenge")
+	}
+
+	tokenURL := realm
+	sep := "?"
+	if s, ok := params["service"]; ok {
+		tokenURL += sep + "service=" + s
+		sep = "&"
+	}
+	if s, ok := params["scope"]; ok {
+		tokenURL += sep + "scope=" + s
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenURL, nil)
+	if err != nil {
+		return "", err
+	}
+
+	if remote.Username != "" && remote.Password != "" {
+		req.SetBasicAuth(remote.Username, remote.Password)
+	}
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("token endpoint returned %d", resp.StatusCode)
+	}
+
+	var tokenResp struct {
+		Token       string `json:"token"`
+		AccessToken string `json:"access_token"`
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil {
+		return "", err
+	}
+
+	if tokenResp.Token != "" {
+		return tokenResp.Token, nil
+	}
+	return tokenResp.AccessToken, nil
+}
+
 type ProxyError struct {
 	Status  int
 	Message string

internal/server/server.go

@@ -35,6 +35,7 @@ import (
 
 type Server struct {
 	cfg          *config.Config
+	version      string
 	router       chi.Router
 	db           *database.DB
 	cache        *cache.Redis
@@ -45,7 +46,7 @@ type Server struct {
 	gc           *gc.Collector
 }
 
-func New(cfg *config.Config) (*Server, error) {
+func New(cfg *config.Config, version string) (*Server, error) {
 	db, err := database.New(cfg.DatabaseDSN())
 	if err != nil {
 		return nil, fmt.Errorf("database: %w", err)
@@ -68,6 +69,7 @@ func New(cfg *config.Config) (*Server, error) {
 
 	s := &Server{
 		cfg:          cfg,
+		version:      version,
 		db:           db,
 		cache:        redis,
 		store:        s3,
@@ -96,6 +98,7 @@ func (s *Server) routes() chi.Router {
 
 	proxyHandler := v1.NewProxyHandler(s.engine, s.virtEngine, s.db, s.store, s.localHandler)
 	r.Mount("/api/v1", proxyHandler.Routes())
+	r.Mount("/v2", proxyHandler.DockerV2Routes())
 
 	remotesHandler := v2.NewRemotesHandler(s.db)
 	virtualsHandler := v2.NewVirtualsHandler(s.db)
@@ -137,7 +140,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
 func (s *Server) handleRoot(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(http.StatusOK)
-	fmt.Fprint(w, `{"name":"artifactapi","version":"3.0.0-dev"}`)
+	fmt.Fprintf(w, `{"name":"artifactapi","version":"%s"}`, s.version)
 }
 
 func (s *Server) newHTTPServer() *http.Server {

internal/virtual/engine.go

@@ -79,7 +79,7 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
 					results[idx] = result{err: fmt.Errorf("local index %q: %w", name, err)}
 					return
 				}
-				results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, Body: body}}
+				results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}}
 				return
 			}
 
@@ -102,7 +102,7 @@ func (e *Engine) fetchMemberIndexes(ctx context.Context, virt models.Virtual, pa
 				return
 			}
 
-			results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, Body: body}}
+			results[idx] = result{index: MemberIndex{RemoteName: name, RepoType: remote.RepoType, BaseURL: remote.BaseURL, Body: body}}
 		}(i, memberName)
 	}
 

internal/virtual/helm_merger.go

@@ -54,15 +54,27 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
 				seen[chart][ver.Version] = true
 
 				if proxyBaseURL != "" {
+					routePrefix := "remote"
+					if member.RepoType == "local" {
+						routePrefix = "local"
+					}
+					baseHost := extractHost(member.BaseURL)
+
 					for i, u := range ver.URLs {
 						if strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://") {
-							ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
+							if baseHost != "" && extractHost(u) != baseHost {
+								continue
+							}
+							relPath := extractPathRelativeToBase(u, member.BaseURL)
+							ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
 								strings.TrimRight(proxyBaseURL, "/"),
+								routePrefix,
 								member.RemoteName,
-								extractPath(u))
+								relPath)
 						} else {
-							ver.URLs[i] = fmt.Sprintf("%s/api/v1/remote/%s/%s",
+							ver.URLs[i] = fmt.Sprintf("%s/api/v1/%s/%s/%s",
 								strings.TrimRight(proxyBaseURL, "/"),
+								routePrefix,
 								member.RemoteName,
 								u)
 						}
@@ -78,6 +90,31 @@ func (m *HelmMerger) MergeIndexes(members []MemberIndex, proxyBaseURL string) ([
 	return yaml.Marshal(merged)
 }
 
+func extractHost(rawURL string) string {
+	idx := strings.Index(rawURL, "://")
+	if idx == -1 {
+		return ""
+	}
+	rest := rawURL[idx+3:]
+	slashIdx := strings.Index(rest, "/")
+	if slashIdx == -1 {
+		return rest
+	}
+	return rest[:slashIdx]
+}
+
+func extractPathRelativeToBase(rawURL, baseURL string) string {
+	fullPath := extractPath(rawURL)
+	basePath := extractPath(baseURL)
+	if basePath != "" {
+		basePath = strings.TrimRight(basePath, "/") + "/"
+		if strings.HasPrefix(fullPath, basePath) {
+			return fullPath[len(basePath):]
+		}
+	}
+	return fullPath
+}
+
 func extractPath(rawURL string) string {
 	idx := strings.Index(rawURL, "://")
 	if idx == -1 {

internal/virtual/merger.go

@@ -9,6 +9,7 @@ import (
 type MemberIndex struct {
 	RemoteName string
 	RepoType   models.RepoType
+	BaseURL    string
 	Body       []byte
 }
 

ui/Dockerfile.ui

@@ -6,13 +6,20 @@ COPY package.json package-lock.json* ./
 RUN npm ci
 
 COPY . .
+
+ARG BASE_PATH=/
+ENV BASE_PATH=${BASE_PATH}
 RUN npm run build
 
 FROM nginx:alpine
 
+ARG BASE_PATH=/
+
 COPY --from=builder /app/dist /usr/share/nginx/html
 COPY nginx.conf /etc/nginx/conf.d/default.conf
 
+RUN sed -i "s|\${BASE_PATH}|${BASE_PATH}|g" /etc/nginx/conf.d/default.conf
+
 EXPOSE 80
 
 CMD ["nginx", "-g", "daemon off;"]

ui/nginx.conf

@@ -5,33 +5,12 @@ server {
     root /usr/share/nginx/html;
     index index.html;
 
-    location /api/ {
+    location ${BASE_PATH}/ {
-        proxy_pass http://artifactapi:8000;
+        rewrite ^${BASE_PATH}(/.*)$ $1 break;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_buffering off;
-    }
-
-    location /v2/ {
-        proxy_pass http://artifactapi:8000;
-        proxy_set_header Host $host;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_buffering off;
-    }
-
-    location /health {
-        proxy_pass http://artifactapi:8000;
-    }
-
-    location /metrics {
-        proxy_pass http://artifactapi:8000;
-    }
-
-    location / {
         try_files $uri $uri/ /index.html;
     }
+
+    location = ${BASE_PATH} {
+        return 301 ${BASE_PATH}/;
+    }
 }

ui/src/main.tsx

@@ -4,9 +4,13 @@ import { BrowserRouter } from 'react-router-dom';
 import { App } from './App';
 import './index.css';
 
+declare const __BASE_PATH__: string;
+
+const basename = __BASE_PATH__.replace(/\/+$/, '') || '/';
+
 createRoot(document.getElementById('root')!).render(
   <StrictMode>
-    <BrowserRouter>
+    <BrowserRouter basename={basename}>
       <App />
     </BrowserRouter>
   </StrictMode>,

ui/src/pages/Dashboard.tsx

@@ -50,6 +50,11 @@ export function Dashboard() {
           value={formatNumber(stats.total_blobs_deduped)}
           sub="shared blobs"
         />
+        <StatsCard
+          label="Bandwidth Saved"
+          value={formatBytes(stats.bandwidth_saved_30d)}
+          sub="last 30 days"
+        />
       </div>
 
       {health && (

ui/vite.config.ts

@@ -1,7 +1,10 @@
 import { defineConfig } from 'vite'
 import react from '@vitejs/plugin-react'
 
+const basePath = process.env.BASE_PATH || '/'
+
 export default defineConfig({
+  base: basePath,
   plugins: [react()],
   server: {
     proxy: {
@@ -11,4 +14,7 @@ export default defineConfig({
       '/metrics': 'http://localhost:8000',
     },
   },
+  define: {
+    '__BASE_PATH__': JSON.stringify(basePath),
+  },
 })