Evict remote objects from every cache layer (#128)

Evicting a remote object only deleted its Postgres row. The S3 index and the Redis TTL/ETag keys survived, so stale mirror metadata such as EPEL repodata kept being served.

- evict the artifact row, S3 index and Redis keys via `Engine.Evict`
- evict a directory with `<dir>/*`; other wildcards return 400, unknown remotes 404
- wait on the per-path fetch lock for single-path evicts; return 503 on timeout or a Redis error
- wildcard evicts take no lock; a fetch already in flight may re-cache its path
- return S3 list errors from `DeletePrefix` instead of deleting nothing

Reviewed-on: #128
Co-authored-by: unkin-agent <unkin-agent@unkin.net>
Co-committed-by: unkin-agent <unkin-agent@unkin.net>

internal/storage/s3.go

@@ -99,6 +99,30 @@ func (s *S3) Stat(ctx context.Context, key string) (*minio.ObjectInfo, error) {
 	return &info, nil
 }
 
+// DeletePrefix removes every object whose key starts with prefix. It lists
+// first so a failed list returns its error instead of deleting nothing.
+func (s *S3) DeletePrefix(ctx context.Context, prefix string) error {
+	var objs []minio.ObjectInfo
+	for obj := range s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
+		if obj.Err != nil {
+			return obj.Err
+		}
+		objs = append(objs, obj)
+	}
+	ch := make(chan minio.ObjectInfo, len(objs))
+	for _, obj := range objs {
+		ch <- obj
+	}
+	close(ch)
+	var err error
+	for res := range s.client.RemoveObjects(ctx, s.bucket, ch, minio.RemoveObjectsOptions{}) {
+		if res.Err != nil && err == nil {
+			err = res.Err
+		}
+	}
+	return err
+}
+
 // ListStaleObjects returns keys under prefix last modified before cutoff. Used
 // by the GC to reap abandoned staging objects (e.g. cancelled docker pushes).
 func (s *S3) ListStaleObjects(ctx context.Context, prefix string, cutoff time.Time) ([]string, error) {