Release Redis locks only while the caller still owns them (#156)

AcquireLock stored a constant value and ReleaseLock was an unconditional DEL, so a holder that outlived its TTL deleted the next holder's lock. This hits Fetch, Evict and gems push/yank.

- store a random owner token with SET NX PX and return it from AcquireLock
- release via a Lua compare-and-delete keyed on that token
- carry the token through Fetch, Evict and Engine.Lock's release func
- release the Fetch lock with an uncancelled context
- make Engine.Lock's busy 503 message generic

Reviewed-on: #156
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

internal/cache/cache_test.go

@@ -61,18 +61,54 @@ func TestTTL(t *testing.T) {
 func TestLock(t *testing.T) {
 	requireRedis(t)
 	ctx := context.Background()
-	ok, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
-	if err != nil || !ok {
-		t.Fatalf("first acquire should succeed: %v %v", ok, err)
+	token, err := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
+	if err != nil || token == "" {
+		t.Fatalf("first acquire should succeed: %q %v", token, err)
 	}
-	if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); ok {
+	if again, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); again != "" {
 		t.Error("second acquire should fail while held")
 	}
-	if err := testRedis.ReleaseLock(ctx, "r", "lockpath"); err != nil {
+	if err := testRedis.ReleaseLock(ctx, "r", "lockpath", "not-the-owner"); err != nil {
 		t.Fatal(err)
 	}
-	if ok, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); !ok {
-		t.Error("acquire should succeed after release")
+	if again, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute); again != "" {
+		t.Error("release with a foreign token must not drop the lock")
+	}
+	if err := testRedis.ReleaseLock(ctx, "r", "lockpath", token); err != nil {
+		t.Fatal(err)
+	}
+	next, _ := testRedis.AcquireLock(ctx, "r", "lockpath", time.Minute)
+	if next == "" {
+		t.Fatal("acquire should succeed after release")
+	}
+	if next == token {
+		t.Error("tokens must differ between holders")
+	}
+}
+
+func TestStaleReleaseKeepsNextHolder(t *testing.T) {
+	requireRedis(t)
+	ctx := context.Background()
+	stale, err := testRedis.AcquireLock(ctx, "r", "stale", 50*time.Millisecond)
+	if err != nil || stale == "" {
+		t.Fatalf("acquire: %q %v", stale, err)
+	}
+	time.Sleep(100 * time.Millisecond)
+	fresh, err := testRedis.AcquireLock(ctx, "r", "stale", time.Minute)
+	if err != nil || fresh == "" {
+		t.Fatalf("acquire after expiry: %q %v", fresh, err)
+	}
+	if err := testRedis.ReleaseLock(ctx, "r", "stale", stale); err != nil {
+		t.Fatal(err)
+	}
+	if again, _ := testRedis.AcquireLock(ctx, "r", "stale", time.Minute); again != "" {
+		t.Fatal("stale holder's release dropped the second holder's lock")
+	}
+	if err := testRedis.ReleaseLock(ctx, "r", "stale", fresh); err != nil {
+		t.Fatal(err)
+	}
+	if again, _ := testRedis.AcquireLock(ctx, "r", "stale", time.Minute); again == "" {
+		t.Error("owner release should free the lock")
 	}
 }
 

internal/cache/redis.go

@@ -2,6 +2,7 @@ package cache
 
 import (
 	"context"
+	"crypto/rand"
 	"fmt"
 	"strings"
 	"time"
@@ -46,15 +47,23 @@ func (r *Redis) CheckTTL(ctx context.Context, remote, path string) (bool, error)
 	return exists > 0, nil
 }
 
-func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (bool, error) {
+// AcquireLock returns the owner token on success and "" when the lock is held.
+func (r *Redis) AcquireLock(ctx context.Context, remote, path string, ttl time.Duration) (string, error) {
 	key := fmt.Sprintf("lock:%s:%s", remote, path)
-	ok, err := r.client.SetNX(ctx, key, "1", ttl).Result()
-	return ok, err
+	token := rand.Text()
+	ok, err := r.client.SetNX(ctx, key, token, ttl).Result()
+	if err != nil || !ok {
+		return "", err
+	}
+	return token, nil
 }
 
-func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error {
+var releaseScript = redis.NewScript(`if redis.call('get',KEYS[1])==ARGV[1] then return redis.call('del',KEYS[1]) else return 0 end`)
+
+// ReleaseLock deletes the lock only while token still owns it.
+func (r *Redis) ReleaseLock(ctx context.Context, remote, path, token string) error {
 	key := fmt.Sprintf("lock:%s:%s", remote, path)
-	return r.client.Del(ctx, key).Err()
+	return releaseScript.Run(ctx, r.client, []string{key}, token).Err()
 }
 
 // SetETag stores etag for path, or drops the stored one when etag is empty.