Restrict wildcard evict to <dir>/*, return 503 on lock contention, surface S3 list errors

internal/storage/s3.go

@@ -99,15 +99,28 @@ func (s *S3) Stat(ctx context.Context, key string) (*minio.ObjectInfo, error) {
 	return &info, nil
 }
 
-// DeletePrefix removes every object whose key starts with prefix.
+// DeletePrefix removes every object whose key starts with prefix. It lists
+// first so a failed list returns its error instead of deleting nothing.
 func (s *S3) DeletePrefix(ctx context.Context, prefix string) error {
-	objects := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
-	for res := range s.client.RemoveObjects(ctx, s.bucket, objects, minio.RemoveObjectsOptions{}) {
-		if res.Err != nil {
-			return res.Err
+	var objs []minio.ObjectInfo
+	for obj := range s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
+		if obj.Err != nil {
+			return obj.Err
+		}
+		objs = append(objs, obj)
+	}
+	ch := make(chan minio.ObjectInfo, len(objs))
+	for _, obj := range objs {
+		ch <- obj
+	}
+	close(ch)
+	var err error
+	for res := range s.client.RemoveObjects(ctx, s.bucket, ch, minio.RemoveObjectsOptions{}) {
+		if res.Err != nil && err == nil {
+			err = res.Err
 		}
 	}
-	return nil
+	return err
 }
 
 // ListStaleObjects returns keys under prefix last modified before cutoff. Used

internal/storage/storage_test.go

@@ -4,11 +4,15 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/minio/minio-go/v7"
+
 	"git.unkin.net/unkin/artifactapi/internal/testsupport"
 )
 
@@ -158,3 +162,26 @@ func TestCASStore(t *testing.T) {
 		t.Errorf("stored content mismatch: %q", got)
 	}
 }
+
+// The fake endpoint fails every list but accepts every delete, as an S3 that
+// tolerates deleting an empty key would.
+func TestDeletePrefixReturnsListError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodPost {
+			_, _ = io.WriteString(w, `<DeleteResult></DeleteResult>`)
+			return
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = io.WriteString(w, `<Error><Code>InternalError</Code><Message>list failed</Message></Error>`)
+	}))
+	defer srv.Close()
+	client, err := minio.New(strings.TrimPrefix(srv.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := &S3{client: client, bucket: "bucket"}
+	err = s.DeletePrefix(context.Background(), "indexes/r/")
+	if err == nil {
+		t.Fatal("DeletePrefix on a failing list returned nil")
+	}
+}