perf: batch access-log writes instead of goroutine+insert per request (#91)
Fixes #76 ## Why Every proxied request spawned a goroutine running a 5s-timeout single-row INSERT. Under load this is unbounded goroutines and connection-pool pressure. ## Changes - Add `database.AccessLogEntry` + `InsertAccessLogBatch` (bulk `COPY`). - The engine starts one background writer that drains a buffered channel and flushes every 128 entries or 2s. - `logAccess` is now a non-blocking channel send (drops on full buffer), so the request path never blocks on the DB. Best-effort telemetry: a small tail may be lost on abrupt shutdown. ## Validation - `make e2e` passes. Reviewed-on: #91 Co-authored-by: Ben Vincent <ben@unkin.net> Co-committed-by: Ben Vincent <ben@unkin.net>
This commit was merged in pull request #91.
This commit is contained in:
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"git.unkin.net/unkin/artifactapi/pkg/models"
|
||||
)
|
||||
|
||||
@@ -109,6 +111,33 @@ func (db *DB) InsertAccessLog(ctx context.Context, remoteName, path string, cach
|
||||
return err
|
||||
}
|
||||
|
||||
// AccessLogEntry is one buffered access-log record.
|
||||
type AccessLogEntry struct {
|
||||
RemoteName string
|
||||
Path string
|
||||
CacheHit bool
|
||||
SizeBytes int64
|
||||
UpstreamMS int
|
||||
ClientIP string
|
||||
}
|
||||
|
||||
// InsertAccessLogBatch bulk-inserts access-log rows with a single COPY.
|
||||
func (db *DB) InsertAccessLogBatch(ctx context.Context, entries []AccessLogEntry) error {
|
||||
if len(entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
rows := make([][]any, len(entries))
|
||||
for i, e := range entries {
|
||||
rows[i] = []any{e.RemoteName, e.Path, e.CacheHit, e.SizeBytes, e.UpstreamMS, e.ClientIP}
|
||||
}
|
||||
_, err := db.Pool.CopyFrom(ctx,
|
||||
pgx.Identifier{"access_log"},
|
||||
[]string{"remote_name", "path", "cache_hit", "size_bytes", "upstream_ms", "client_ip"},
|
||||
pgx.CopyFromRows(rows),
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) FindOrphanedBlobs(ctx context.Context) ([]models.Blob, error) {
|
||||
rows, err := db.Pool.Query(ctx, `
|
||||
SELECT b.content_hash, b.s3_key, b.size_bytes, b.content_type, b.created_at
|
||||
|
||||
Reference in New Issue
Block a user