Evict remote objects from every cache layer (#128)

Evicting a remote object only deleted its Postgres row. The S3 index and the Redis TTL/ETag keys survived, so stale mirror metadata such as EPEL repodata kept being served.

- evict the artifact row, S3 index and Redis keys via `Engine.Evict`
- evict a directory with `<dir>/*`; other wildcards return 400, unknown remotes 404
- wait on the per-path fetch lock for single-path evicts; return 503 on timeout or a Redis error
- wildcard evicts take no lock; a fetch already in flight may re-cache its path
- return S3 list errors from `DeletePrefix` instead of deleting nothing

Reviewed-on: #128
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

internal/storage/s3.go

@@ -99,6 +99,30 @@ func (s *S3) Stat(ctx context.Context, key string) (*minio.ObjectInfo, error) {
 	return &info, nil
 }
 
+// DeletePrefix removes every object whose key starts with prefix. It lists
+// first so a failed list returns its error instead of deleting nothing.
+func (s *S3) DeletePrefix(ctx context.Context, prefix string) error {
+	var objs []minio.ObjectInfo
+	for obj := range s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
+		if obj.Err != nil {
+			return obj.Err
+		}
+		objs = append(objs, obj)
+	}
+	ch := make(chan minio.ObjectInfo, len(objs))
+	for _, obj := range objs {
+		ch <- obj
+	}
+	close(ch)
+	var err error
+	for res := range s.client.RemoveObjects(ctx, s.bucket, ch, minio.RemoveObjectsOptions{}) {
+		if res.Err != nil && err == nil {
+			err = res.Err
+		}
+	}
+	return err
+}
+
 // ListStaleObjects returns keys under prefix last modified before cutoff. Used
 // by the GC to reap abandoned staging objects (e.g. cancelled docker pushes).
 func (s *S3) ListStaleObjects(ctx context.Context, prefix string, cutoff time.Time) ([]string, error) {

internal/storage/storage_test.go

@@ -4,11 +4,15 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/minio/minio-go/v7"
+
 	"git.unkin.net/unkin/artifactapi/internal/testsupport"
 )
 
@@ -158,3 +162,26 @@ func TestCASStore(t *testing.T) {
 		t.Errorf("stored content mismatch: %q", got)
 	}
 }
+
+// The fake endpoint fails every list but accepts every delete, as an S3 that
+// tolerates deleting an empty key would.
+func TestDeletePrefixReturnsListError(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodPost {
+			_, _ = io.WriteString(w, `<DeleteResult></DeleteResult>`)
+			return
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = io.WriteString(w, `<Error><Code>InternalError</Code><Message>list failed</Message></Error>`)
+	}))
+	defer srv.Close()
+	client, err := minio.New(strings.TrimPrefix(srv.URL, "http://"), &minio.Options{Region: "us-east-1", MaxRetries: 1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := &S3{client: client, bucket: "bucket"}
+	err = s.DeletePrefix(context.Background(), "indexes/r/")
+	if err == nil {
+		t.Fatal("DeletePrefix on a failing list returned nil")
+	}
+}