Merge master into benvin/remote-query-forwarder

internal/proxy/engine.go

@@ -134,10 +134,11 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
 		slog.Warn("cache hit but S3 miss, re-fetching", "remote", remote.Name, "path", path)
 	}
 
-	locked, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
+	token, err := e.cache.AcquireLock(ctx, remote.Name, path, fetchLockTTL)
 	if err != nil {
 		slog.Warn("lock acquire failed", "error", err)
 	}
+	locked := token != ""
 
 	if !locked {
 		// Another request holds the fetch lock. Poll the store until the leader
@@ -151,7 +152,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
 	}
 
 	if locked {
-		defer e.cache.ReleaseLock(ctx, remote.Name, path)
+		defer func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remote.Name, path, token) }()
 	}
 
 	if class == ClassMutable && remote.CheckMutable {
@@ -160,7 +161,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p
 			notModified, err := e.checkUpstream(ctx, remote, path, etag, prov)
 			if err == nil && notModified {
 				_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
-				_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
+				_ = e.cache.SetETag(ctx, remote.Name, path, etag, etagTTL(ttl))
 				result, err := e.serveFromStore(ctx, remote, path)
 				if err == nil {
 					result.Source = "cache"
@@ -229,10 +230,11 @@ func (e *Engine) Evict(ctx context.Context, remoteName, path string) error {
 		return fmt.Errorf("get remote: %w", err)
 	}
 	if !wildcard {
-		if err := e.waitForLock(ctx, remoteName, path); err != nil {
+		release, err := e.Lock(ctx, remoteName, path, e.evictLockWait, fetchLockTTL)
+		if err != nil {
 			return err
 		}
-		defer func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remoteName, path) }()
+		defer release()
 		if err := e.db.DeleteArtifact(ctx, remoteName, path); err != nil {
 			return fmt.Errorf("delete artifact: %w", err)
 		}
@@ -265,28 +267,36 @@ func (e *Engine) evictPrefix(ctx context.Context, remoteName, prefix string) err
 	return e.cache.ForgetPrefix(ctx, remoteName, prefix)
 }
 
-// waitForLock takes the per-path fetch lock so an in-flight Fetch cannot
-// re-set TTL/ETag keys after an evict. It fails with a 503 when the lock
-// cannot be taken within evictLockWait or Redis errors.
-func (e *Engine) waitForLock(ctx context.Context, remoteName, path string) error {
-	deadline := time.Now().Add(e.evictLockWait)
+// Lock takes the per-path Redis lock (the one Fetch holds) and returns its
+// release. It fails with a 503 when the lock cannot be taken within wait or
+// Redis errors.
+func (e *Engine) Lock(ctx context.Context, remoteName, path string, wait, ttl time.Duration) (func(), error) {
+	token, err := e.waitForLock(ctx, remoteName, path, wait, ttl)
+	if err != nil {
+		return nil, err
+	}
+	return func() { _ = e.cache.ReleaseLock(context.WithoutCancel(ctx), remoteName, path, token) }, nil
+}
+
+func (e *Engine) waitForLock(ctx context.Context, remoteName, path string, wait, ttl time.Duration) (string, error) {
+	deadline := time.Now().Add(wait)
 	for {
-		ok, err := e.cache.AcquireLock(ctx, remoteName, path, fetchLockTTL)
-		if ok {
-			return nil
+		token, err := e.cache.AcquireLock(ctx, remoteName, path, ttl)
+		if token != "" {
+			return token, nil
 		}
 		if ctx.Err() != nil {
-			return ctx.Err()
+			return "", ctx.Err()
 		}
 		if err != nil {
-			return &ProxyError{Status: http.StatusServiceUnavailable, Message: fmt.Sprintf("fetch lock: %v", err)}
+			return "", &ProxyError{Status: http.StatusServiceUnavailable, Message: fmt.Sprintf("lock: %v", err)}
 		}
 		if time.Now().After(deadline) {
-			return &ProxyError{Status: http.StatusServiceUnavailable, Message: "fetch in progress, retry evict"}
+			return "", &ProxyError{Status: http.StatusServiceUnavailable, Message: "resource busy, retry later"}
 		}
 		select {
 		case <-ctx.Done():
-			return ctx.Err()
+			return "", ctx.Err()
 		case <-time.After(50 * time.Millisecond):
 		}
 	}
@@ -366,7 +376,7 @@ func (e *Engine) headUpstreamOnce(ctx context.Context, remote models.Remote, pat
 				req.Header.Set(k, v)
 			}
 		}
-		return http.DefaultClient.Do(req)
+		return clientForRemote(remote).Do(req)
 	}
 
 	resp, err := doHead(nil)
@@ -504,8 +514,9 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
 		}
 
 		_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
-		if etag := resp.Header.Get("ETag"); etag != "" {
-			_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
+		// A zero TTL never expires, so it never needs revalidating.
+		if remote.CheckMutable && ttl > 0 {
+			_ = e.cache.SetETag(ctx, remote.Name, path, resp.Header.Get("ETag"), etagTTL(ttl))
 		}
 
 		return &FetchResult{
@@ -533,9 +544,6 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote
 	}
 
 	_ = e.cache.SetTTL(ctx, remote.Name, path, ttl)
-	if etag := resp.Header.Get("ETag"); etag != "" {
-		_ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl)
-	}
 
 	reader, info, err := e.store.Download(ctx, casResult.S3Key)
 	if err != nil {
@@ -655,6 +663,12 @@ func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, pa
 	return resp.StatusCode == http.StatusNotModified, nil
 }
 
+// etagTTL keeps a mutable path's ETag well past its freshness key so an expired
+// path revalidates with a conditional HEAD, while idle paths still age out.
+func etagTTL(ttl time.Duration) time.Duration {
+	return max(10*ttl, 7*24*time.Hour)
+}
+
 func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration {
 	switch class {
 	case ClassImmutable: