feat: keep stale mutables when upstream is unreachable; update README

When a mutable file's TTL expires and the upstream backend cannot be
contacted (network error or timeout), the cached copy is kept and its
TTL refreshed instead of being evicted. This keeps RPM repodata, Alpine
indexes, branch archives, and other mutable data available during
upstream outages.

Adds UpstreamUnreachable exception and _upstream_reachable() helper.
check_upstream_changed() now raises UpstreamUnreachable on network
errors (was silently returning True). handle_expired_mutable() catches
the exception on the check_mutable_updates path and calls
_upstream_reachable() on the plain-expiry path.

README updated to current immutable/mutable terminology and documents
all new caching features.

.gitignore

@@ -35,7 +35,6 @@ env/
 
 # Environment variables
 .env
-remotes.yaml
 
 # Logs
 *.log

README.md

@@ -6,10 +6,13 @@ A generic FastAPI-based artifact caching system that downloads and stores files
 
 - **Generic Remote Support**: Works with any HTTP-based file server (GitHub, Gitea, HashiCorp, custom servers)
 - **Configuration-Based**: YAML configuration for remotes, patterns, and access control
-- **Direct URL API**: Access cached files via clean URLs like `/api/github/owner/repo/path/file.tar.gz`
-- **Pattern Filtering**: Regex-based inclusion patterns for security and organization
+- **Direct URL API**: Access cached files via clean URLs like `/api/v1/remote/github/owner/repo/path/file.tar.gz`
+- **Immutable/Mutable Pattern Model**: Per-remote regex patterns distinguish forever-cached artifacts from TTL-expiring metadata
 - **Smart Caching**: Automatic download and cache on first access, serve from cache afterward
+- **Conditional Revalidation**: Optional `check_mutable_updates` flag — sends `If-None-Match`/`If-Modified-Since` on expiry; skips re-download on 304
+- **Stale-on-Upstream-Error**: Expired mutable files are kept and their TTL refreshed when the backend cannot be reached, so cached data remains available during upstream outages
 - **S3 Storage**: MinIO/S3 backend with predictable paths
+- **Docker Registry Proxy**: Full Docker Registry HTTP API v2 for transparent container image caching
 - **Content-Type Detection**: Automatic MIME type detection for downloads
 
 ## Architecture
@@ -71,15 +74,18 @@ The system uses `remotes.yaml` to define remote repositories and access patterns
 remotes:
   remote-name:
     base_url: "https://example.com"       # Base URL for the remote
-    type: "remote"                        # Type: "remote" or "local"
-    package: "generic"                    # Package type: "generic", "alpine", "rpm"
+    type: "remote"                        # "remote" or "local"
+    package: "generic"                    # "generic", "alpine", "rpm", or "docker"
     description: "Human readable description"
-    include_patterns:                     # Regex patterns for allowed files
+    immutable_patterns:                   # Files cached forever (release binaries, versioned tags)
       - "pattern1"
       - "pattern2"
-    cache:                               # Cache configuration (optional)
-      file_ttl: 0                        # File cache TTL (0 = indefinite)
-      index_ttl: 300                     # Index file TTL in seconds
+    mutable_patterns:                     # Files that expire after mutable_ttl (optional)
+      - "pattern3"
+    check_mutable_updates: false          # Enable conditional HEAD before re-fetching (optional)
+    cache:
+      immutable_ttl: 0                    # TTL for immutable files (0 = indefinitely)
+      mutable_ttl: 3600                   # TTL in seconds for mutable files
 ```
 
 ### Remote Types
@@ -94,30 +100,30 @@ remotes:
     type: "remote"
     package: "generic"
     description: "GitHub releases and files"
-    include_patterns:
+    immutable_patterns:
       - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
       - "lxc/incus/.*\\.tar\\.gz$"
       - "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
     cache:
-      file_ttl: 0      # Cache files indefinitely
-      index_ttl: 0     # No index files for generic remotes
+      immutable_ttl: 0      # Cache files indefinitely
 
-  hashicorp-releases:
-    base_url: "https://releases.hashicorp.com"
+  github-archive:
+    base_url: "https://github.com"
     type: "remote"
     package: "generic"
-    description: "HashiCorp product releases"
-    include_patterns:
-      - "terraform/.*terraform_.*_linux_amd64\\.zip$"
-      - "vault/.*vault_.*_linux_amd64\\.zip$"
-      - "consul/.*/consul_.*_linux_amd64\\.zip$"
+    description: "GitHub repository archive tarballs"
+    immutable_patterns:
+      - ".*/archive/refs/tags/.*\\.tar\\.gz$"   # tag archives never change
+    mutable_patterns:
+      - ".*/archive/refs/heads/main\\.tar\\.gz$"  # branch archives can change
+    check_mutable_updates: true   # send If-None-Match on expiry; skip re-download on 304
     cache:
-      file_ttl: 0
-      index_ttl: 0
+      immutable_ttl: 0
+      mutable_ttl: 86400          # re-check branch archives after 1 day
 ```
 
 #### Package Repository Remotes
-For Linux package repositories with index files:
+For Linux package repositories:
 
 ```yaml
 remotes:
@@ -126,23 +132,25 @@ remotes:
     type: "remote"
     package: "alpine"
     description: "Alpine Linux APK package repository"
-    include_patterns:
-      - ".*/x86_64/.*\\.apk$"           # Only x86_64 packages
+    immutable_patterns:
+      - ".*/x86_64/.*\\.apk$"     # packages are immutable by content-hash
+    # APKINDEX.tar.gz is a package-type default mutable file — no mutable_patterns needed
     cache:
-      file_ttl: 0                       # Cache packages indefinitely
-      index_ttl: 7200                   # Cache APKINDEX.tar.gz for 2 hours
+      immutable_ttl: 0
+      mutable_ttl: 7200            # re-fetch APKINDEX.tar.gz after 2 hours
 
   almalinux:
-    base_url: "http://mirror.aarnet.edu.au/pub/almalinux"
+    base_url: "https://mirror.example.com/almalinux"
     type: "remote"
     package: "rpm"
     description: "AlmaLinux RPM package repository"
-    include_patterns:
+    immutable_patterns:
       - ".*/x86_64/.*\\.rpm$"
       - ".*/noarch/.*\\.rpm$"
+    # repomd.xml and repodata/* are package-type defaults
     cache:
-      file_ttl: 0
-      index_ttl: 7200                   # Cache metadata files for 2 hours
+      immutable_ttl: 0
+      mutable_ttl: 7200
 ```
 
 #### Local Repositories
@@ -155,62 +163,45 @@ remotes:
     package: "generic"
     description: "Local generic file repository"
     cache:
-      file_ttl: 0
-      index_ttl: 0
+      immutable_ttl: 0
+      mutable_ttl: 0
 ```
 
-### Include Patterns
+### Immutable Patterns
 
-Include patterns are regular expressions that control which files can be accessed. Patterns use Python `re.search`, so they match anywhere in the path unless anchored with `^` or `$`. Only files matching at least one pattern are served; all others return HTTP 403.
+`immutable_patterns` are regular expressions that control which files can be accessed. Patterns use Python `re.search`, so they match anywhere in the path unless anchored with `^` or `$`. Only files matching at least one pattern are served; all others return HTTP 403.
+
+Matched files are cached with `immutable_ttl` (default 0 = forever). Use these for versioned release artifacts that never change once published.
 
 ```yaml
-include_patterns:
-  # Exact project + architecture — most restrictive
+immutable_patterns:
   - "^gruntwork-io/terragrunt/releases/download/.*/terragrunt_linux_amd64$"
-
-  # Any release asset for a project, any version
   - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
-
-  # File extension only — allow all files of a given type from any path
   - ".*\\.tar\\.gz$"
-  - ".*\\.rpm$"
-  - ".*\\.zip$"
-
-  # Architecture subtree — allow everything under x86_64/
-  - ".*/x86_64/.*"
-
-  # Combined: architecture + extension
   - ".*/x86_64/.*\\.rpm$"
   - ".*/noarch/.*\\.rpm$"
-
-  # Docker image names (used with package: docker remotes)
-  - "^library/nginx"           # nginx official images only
-  - "^rancher/"                # all rancher/* images
-  - "^rancher/rke2-runtime"    # specific image
-
-  # Repodata directories — allow all metadata for an RPM repo
   - ".*/repodata/.*$"
 ```
 
-**Security note**: Omitting `include_patterns` entirely allows all files from that remote. Index files (e.g. `APKINDEX.tar.gz`, `repomd.xml`, tag manifests) always bypass pattern enforcement — they are served unconditionally so clients can discover available packages.
+**Security note**: Omitting `immutable_patterns` entirely allows all files from that remote.
 
-### Index Patterns
+### Mutable Patterns
 
-Index patterns identify repository metadata files. Index files get special treatment:
-- **Always served** regardless of `include_patterns`
-- **Cached with `index_ttl`** instead of `file_ttl`
-- **Automatically refreshed** when the TTL expires — the cached copy is evicted and re-fetched on next request
+`mutable_patterns` identify files that change over time (index files, branch archives, metadata). Mutable files:
+- **Always served** regardless of `immutable_patterns`
+- **Cached with `mutable_ttl`** and re-fetched from upstream when the TTL expires
+- **Kept stale** when the upstream backend is unreachable — TTL is refreshed automatically so the cached copy remains available until the backend recovers (see below)
 
-Built-in defaults per package type:
+Built-in defaults per package type (no configuration needed):
 
-| Package type | Built-in index patterns |
+| Package type | Built-in mutable patterns |
 |---|---|
 | `alpine` | `APKINDEX\.tar\.gz$` |
 | `rpm` | `repomd\.xml$`, `repodata/` metadata (xml, sqlite, yaml, asc, txt variants), `Packages\.gz$` |
 | `docker` | Tag manifests (non-digest refs), `/tags/list` |
 | `generic` | *(none)* |
 
-Use `index_patterns` to add extra patterns on top of the defaults. Duplicates are ignored automatically.
+Use `mutable_patterns` to add extra patterns on top of the defaults. Duplicates are ignored automatically.
 
 ```yaml
 remotes:
@@ -218,60 +209,74 @@ remotes:
     base_url: "https://charts.example.com"
     type: "remote"
     package: "generic"
-    include_patterns:
-      - ".*\\.tgz$"            # chart archives
-    index_patterns:
-      - "index\\.yaml$"        # Helm repo index — re-fetched on every TTL expiry
+    immutable_patterns:
+      - ".*\\.tgz$"
+    mutable_patterns:
+      - "index\\.yaml$"        # Helm repo index
     cache:
-      file_ttl: 0
-      index_ttl: 600           # re-check the index every 10 minutes
+      immutable_ttl: 0
+      mutable_ttl: 600         # re-check the index every 10 minutes
 
   apt-mirror:
     base_url: "https://apt.example.com"
     type: "remote"
     package: "generic"
-    include_patterns:
+    immutable_patterns:
       - ".*\\.deb$"
-    index_patterns:
-      - "InRelease$"           # signed APT release file
-      - "Release$"             # unsigned APT release file
-      - "Packages\\.gz$"       # compressed package list
+    mutable_patterns:
+      - "InRelease$"
+      - "Release$"
+      - "Packages\\.gz$"
       - "Packages\\.xz$"
     cache:
-      file_ttl: 0
-      index_ttl: 3600          # hourly index refresh
-
-  almalinux-with-extras:
-    base_url: "https://mirror.example.com/almalinux"
-    type: "remote"
-    package: "rpm"             # inherits repomd.xml + repodata/* defaults
-    include_patterns:
-      - ".*/x86_64/.*\\.rpm$"
-      - ".*/noarch/.*\\.rpm$"
-    index_patterns:
-      - "comps\\.xml$"         # optional group metadata (adds to rpm defaults)
-    cache:
-      file_ttl: 0
-      index_ttl: 7200
+      immutable_ttl: 0
+      mutable_ttl: 3600
 ```
 
-Pattern matching uses `re.search`, so `"index\\.yaml$"` matches `/stable/index.yaml` and `/index.yaml`. Anchor with `^` to restrict to the path root.
+### Conditional Revalidation (`check_mutable_updates`)
+
+By default, when a mutable file's TTL expires the cached copy is evicted and the full file is re-downloaded on the next request. Setting `check_mutable_updates: true` on a remote enables a cheaper conditional check first:
+
+1. On TTL expiry, a `HEAD` request is sent to the upstream with `If-None-Match` / `If-Modified-Since` headers (populated from the original download).
+2. If the upstream replies **304 Not Modified**, the TTL is refreshed in place — no re-download, no S3 traffic.
+3. If the upstream replies **200**, the cached copy is evicted and re-downloaded normally.
+
+This only applies to user-defined `mutable_patterns`. Package-type built-in patterns (APKINDEX, repomd.xml, Docker manifests) are always re-fetched unconditionally.
+
+```yaml
+remotes:
+  github-archive:
+    base_url: "https://github.com"
+    type: "remote"
+    package: "generic"
+    immutable_patterns:
+      - ".*/archive/refs/tags/.*\\.tar\\.gz$"
+    mutable_patterns:
+      - ".*/archive/refs/heads/main\\.tar\\.gz$"
+    check_mutable_updates: true
+    cache:
+      immutable_ttl: 0
+      mutable_ttl: 86400
+```
+
+### Stale-on-Upstream-Error
+
+When a mutable file's TTL expires and the upstream backend **cannot be reached** (connection refused, DNS failure, timeout), the cached copy is **kept and its TTL refreshed** rather than evicted. This means:
+
+- RPM repodata, Alpine indexes, branch archives, and other mutable files remain available during upstream outages.
+- Clients continue to receive the last-known-good copy without errors.
+- Once the backend recovers and the refreshed TTL next expires, normal eviction resumes.
+
+This behaviour is automatic and requires no configuration. Only network-level failures trigger it — HTTP error responses (404, 503, etc.) are treated as the backend being reachable and proceed with normal expiry.
 
 ### Cache Configuration
 
-Control how long different file types are cached:
-
 ```yaml
 cache:
-  file_ttl: 0        # Regular files (0 = cache indefinitely)
-  index_ttl: 300     # Index files like APKINDEX.tar.gz (seconds)
+  immutable_ttl: 0       # Immutable files (0 = cache indefinitely, rarely changed)
+  mutable_ttl: 3600      # Mutable files — TTL in seconds before re-fetch is attempted
 ```
 
-**Index Files**: Repository metadata files that change frequently:
-- Alpine: `APKINDEX.tar.gz`
-- RPM: `repomd.xml`, `*-primary.xml.gz`, etc.
-- These are automatically detected and use `index_ttl`
-
 ### Environment Variables
 
 All runtime configuration comes from environment variables:
@@ -351,26 +356,26 @@ data:
         type: "remote"
         package: "generic"
         description: "GitHub releases and files"
-        include_patterns:
+        immutable_patterns:
           - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
           - "lxc/incus/.*\\.tar\\.gz$"
           - "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
         cache:
-          file_ttl: 0
-          index_ttl: 0
+          immutable_ttl: 0
+          mutable_ttl: 0
 
       hashicorp-releases:
         base_url: "https://releases.hashicorp.com"
         type: "remote"
         package: "generic"
         description: "HashiCorp product releases"
-        include_patterns:
+        immutable_patterns:
           - "terraform/.*terraform_.*_linux_amd64\\.zip$"
           - "vault/.*vault_.*_linux_amd64\\.zip$"
           - "consul/.*/consul_.*_linux_amd64\\.zip$"
         cache:
-          file_ttl: 0
-          index_ttl: 0
+          immutable_ttl: 0
+          mutable_ttl: 0
 ```
 
 ### 3. Secret for Environment Variables
@@ -778,8 +783,8 @@ remotes:
     username: "your-dockerhub-username"
     password: "your-dockerhub-token"   # PAT with read scope
     cache:
-      file_ttl: 0
-      index_ttl: 300
+      immutable_ttl: 0
+      mutable_ttl: 300
 ```
 
 A pull of `nginx:latest` becomes `/v2/dockerhub/library/nginx/manifests/latest` on the artifact API.
@@ -804,8 +809,8 @@ remotes:
     username: "your-github-username"
     password: "ghp_your_github_pat"   # read:packages scope required
     cache:
-      file_ttl: 0
-      index_ttl: 300
+      immutable_ttl: 0
+      mutable_ttl: 300
 ```
 
 A pull of `ghcr.io/rancher/rke2-runtime:v1.30.0-rke2r1` becomes `/v2/ghcr/rancher/rke2-runtime/manifests/v1.30.0-rke2r1`.
@@ -844,7 +849,7 @@ Each entry needs a matching remote in `remotes.yaml` using the name from the rew
 
 #### Restricting which images are cached
 
-Use `include_patterns` on the remote to allow only specific images through the proxy. Requests for images not matching any pattern return HTTP 403 to the node.
+Use `immutable_patterns` on the remote to allow only specific images through the proxy. Requests for images not matching any pattern return HTTP 403 to the node.
 
 ```yaml
 remotes:
@@ -852,17 +857,17 @@ remotes:
     base_url: "https://registry-1.docker.io"
     type: "remote"
     package: "docker"
-    include_patterns:
+    immutable_patterns:
       - "^library/nginx"           # official nginx only
       - "^library/redis"           # official redis only
       - "^rancher/"                # all rancher images
       - "^grafana/grafana"         # specific image
     cache:
-      file_ttl: 0
-      index_ttl: 300
+      immutable_ttl: 0
+      mutable_ttl: 300
 ```
 
-Omit `include_patterns` to allow all images from that registry.
+Omit `immutable_patterns` to allow all images from that registry.
 
 #### TLS configuration
 

remotes.yaml

@@ -0,0 +1,203 @@
+# Example remotes configuration — copy and adapt for your environment.
+#
+# immutable_patterns: artifacts cached forever (e.g. release binaries, versioned tags).
+# mutable_patterns:   artifacts that expire after cache.mutable_ttl seconds and are
+#                     re-fetched from upstream on next request (e.g. index files,
+#                     branch archives). Defaults to the package-type built-ins when
+#                     not set (APKINDEX, repomd.xml, Docker manifests, etc.).
+# cache:
+#   immutable_ttl: TTL for immutable files (0 = forever, rarely needed to change).
+#   mutable_ttl:   TTL in seconds for mutable files. Omit to use the default (3600).
+#
+# WARNING: this file may contain credentials — do not commit real values.
+#
+# Global configuration
+#s3:
+#  endpoint: "localhost:9000"
+#  access_key: "minioadmin"
+#  secret_key: "minioadmin"
+#  bucket: "artifacts"
+#  secure: false
+#
+#redis:
+#  url: "redis://localhost:6379/0"
+#
+#database:
+#  url: "postgresql://artifacts:artifacts123@localhost:5432/artifacts"
+#
+remotes:
+  github:
+    base_url: "https://github.com"
+    type: "remote"
+    package: "generic"
+    description: "GitHub releases and files"
+    immutable_patterns:
+      - "gruntwork-io/terragrunt/.*terragrunt_linux_amd64.*"
+      - "lxc/incus/.*\\.tar\\.gz$"
+      - "prometheus/node_exporter/.*/node_exporter-.*\\.linux-amd64\\.tar\\.gz$"
+      - "VictoriaMetrics/VictoriaMetrics/.*/vmutils-linux-amd64-.*\\.tar\\.gz$"
+      - "VictoriaMetrics/VictoriaMetrics/.*/victoria-metrics-linux-amd64-.*-cluster\\.tar\\.gz$"
+      - "VictoriaMetrics/VictoriaMetrics/.*/victoria-logs-linux-amd64-.*\\.tar\\.gz$"
+      - "VictoriaMetrics/VictoriaMetrics/.*/vlutils-linux-amd64-.*\\.tar\\.gz$"
+      - "prometheus-community/bind_exporter/.*/bind_exporter-.*\\.linux-amd64\\.tar\\.gz$"
+      - "prometheus-community/pgbouncer_exporter/.*/pgbouncer_exporter-.*\\.linux-amd64\\.tar\\.gz$"
+      - "prometheus-community/postgres_exporter/.*/postgres_exporter-.*\\.linux-amd64\\.tar\\.gz$"
+      - "onedr0p/exportarr/.*/exportarr_.*_linux_amd64\\.tar\\.gz$"
+      - "tynany/frr_exporter/.*/frr_exporter-.*\\.linux-amd64\\.tar\\.gz$"
+      - "camptocamp/prometheus-puppetdb-exporter/.*/prometheus-puppetdb-exporter-.*\\.linux-amd64\\.tar\\.gz$"
+      - "grafana/jsonnet-language-server/.*/jsonnet-language-server_.*_linux_amd64$"
+      - "helmfile/helmfile/.*/helmfile_.*_linux_amd64\\.tar\\.gz$"
+      - "helmfile/vals/.*/vals_.*_linux_amd64\\.tar\\.gz$"
+      - "openbao/openbao-plugins/.*/openbao-plugin-secrets-consul_linux_amd64_.*\\.tar\\.gz$"
+      - "openbao/openbao-plugins/.*/openbao-plugin-secrets-nomad_linux_amd64_.*\\.tar\\.gz$"
+      - "apple/foundationdb/.*/libfdb_c\\.x86_64\\.so$"
+      - "stalwartlabs/stalwart/.*/stalwart-cli-x86_64-unknown-linux-gnu\\.tar\\.gz$"
+      - "stalwartlabs/stalwart/.*/stalwart-foundationdb-x86_64-unknown-linux-gnu\\.tar\\.gz$"
+      - "stalwartlabs/stalwart/.*/stalwart-x86_64-unknown-linux-gnu\\.tar\\.gz$"
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 0
+
+  github-archive:
+    base_url: "https://github.com"
+    type: "remote"
+    package: "generic"
+    description: "GitHub repository archive tarballs"
+    immutable_patterns:
+      # Tag archives are immutable — a tag never changes
+      - ".*/archive/refs/tags/.*\\.tar\\.gz$"
+    mutable_patterns:
+      # Branch archives can change on every push
+      - ".*/archive/refs/heads/main\\.tar\\.gz$"
+      - ".*/archive/refs/heads/master\\.tar\\.gz$"
+    # Before re-downloading an expired branch archive, check whether it has
+    # actually changed (304 Not Modified → just refresh the TTL, no transfer).
+    # Only applies to user-defined mutable_patterns, not package-type defaults.
+    check_mutable_updates: true
+    cache:
+      immutable_ttl: 0       # Tag archives cached indefinitely
+      mutable_ttl: 86400     # Branch archives refreshed after 1 day
+
+  gitea-dl:
+    base_url: "https://dl.gitea.com"
+    type: "remote"
+    package: "generic"
+    description: "Gitea download site"
+    immutable_patterns:
+      - "act_runner/.*/act_runner-.*-linux-amd64$"
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 0
+
+  hashicorp-releases:
+    base_url: "https://releases.hashicorp.com"
+    type: "remote"
+    package: "generic"
+    description: "HashiCorp product releases"
+    immutable_patterns:
+      - "terraform/.*terraform_.*_linux_amd64\\.zip$"
+      - "terraform/.*terraform_.*_windows_amd64\\.zip$"
+      - "terraform/.*terraform_.*_darwin_amd64\\.zip$"
+      - "vault/.*vault_.*_linux_amd64\\.zip$"
+      - "vault/.*vault_.*_windows_amd64\\.zip$"
+      - "vault/.*vault_.*_darwin_amd64\\.zip$"
+      - "consul-cni/.*/consul-cni_.*_linux_amd64\\.zip$"
+      - "consul/.*/consul_.*_linux_amd64\\.zip$"
+      - "nomad-autoscaler/.*/nomad-autoscaler_.*_linux_amd64\\.zip$"
+      - "nomad/.*/nomad_.*_linux_amd64\\.zip$"
+      - "packer/.*/packer_.*_linux_amd64\\.zip$"
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 0
+
+  alpine:
+    base_url: "https://dl-cdn.alpinelinux.org"
+    type: "remote"
+    package: "alpine"
+    description: "Alpine Linux APK package repository"
+    immutable_patterns:
+      - ".*/x86_64/.*\\.apk$"
+    # check_mutable_updates not set: APKINDEX.tar.gz is a package-type default
+    # and is always re-fetched on expiry — conditional checks are skipped for
+    # built-in mutable patterns regardless of this flag.
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 7200  # Index files (APKINDEX.tar.gz) cached for 2 hours
+
+  almalinux:
+    base_url: "https://gsl-syd.mm.fcix.net/almalinux"
+    type: "remote"
+    package: "rpm"
+    description: "AlmaLinux RPM package repository"
+    immutable_patterns:
+      - ".*/x86_64/.*\\.rpm$"
+      - ".*/noarch/.*\\.rpm$"
+      - ".*/repodata/.*$"
+      - ".*\\.rpm$"  # Allow all RPM files
+    # repomd.xml / repodata are package-type defaults — always re-fetched on
+    # expiry. check_mutable_updates would only apply to any custom
+    # mutable_patterns added here.
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 7200  # Metadata files cached for 2 hours
+
+  epel:
+    base_url: "http://mirror.aarnet.edu.au/pub/epel"
+    type: "remote"
+    package: "rpm"
+    description: "EPEL (Extra Packages for Enterprise Linux)"
+    immutable_patterns:
+      - "8/Everything/x86_64/.*\\.rpm$"
+      - "9/Everything/x86_64/.*\\.rpm$"
+      - "10/Everything/x86_64/.*\\.rpm$"
+      - ".*/noarch/.*\\.rpm$"
+      - ".*/repodata/.*$"
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 7200  # Metadata files cached for 2 hours
+
+  fedora:
+    base_url: "https://gsl-syd.mm.fcix.net/fedora/linux"
+    type: "remote"
+    package: "rpm"
+    description: "Fedora Linux RPM package repository"
+    immutable_patterns:
+      - "releases/.*/Everything/x86_64/.*\\.rpm$"
+      - "updates/.*/Everything/x86_64/.*\\.rpm$"
+      - "development/.*/Everything/x86_64/.*\\.rpm$"
+      - ".*/noarch/.*\\.rpm$"
+      - "updates/.*/Everything/x86_64/repodata/.*$"
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 300  # Metadata files cached for 5 minutes
+
+  ghcr:
+    base_url: "https://ghcr.io"
+    type: "remote"
+    package: "docker"
+    description: "GitHub Container Registry"
+    # username: "your-github-username"
+    # password: "your-github-pat"  # needs read:packages scope
+    # Docker manifest/tag-list patterns are package-type defaults — always
+    # re-fetched on expiry. check_mutable_updates only applies to any custom
+    # mutable_patterns you add (e.g. a metadata endpoint).
+    cache:
+      immutable_ttl: 0
+      mutable_ttl: 300
+
+  dockerhub:
+    base_url: "https://registry-1.docker.io"
+    type: "remote"
+    package: "docker"
+    description: "Docker Hub registry"
+    cache:
+      immutable_ttl: 0
+      mutable_ttl: 300
+
+  local-generic:
+    type: "local"
+    package: "generic"
+    description: "Local generic file repository"
+    cache:
+      immutable_ttl: 0  # Files cached indefinitely
+      mutable_ttl: 0

src/artifactapi/cache.py

@@ -26,11 +26,13 @@ class RedisCache:
         return any(re.search(p, file_path) for p in patterns)
 
     def get_index_cache_key(self, remote_name: str, path: str) -> str:
-        """Generate cache key for index files"""
         return f"index:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}"
 
-    def is_index_valid(self, remote_name: str, path: str, ttl_override: int = None) -> bool:
-        """Check if index file is still valid (not expired)"""
+    def get_mutable_meta_key(self, remote_name: str, path: str) -> str:
+        return f"mutable:meta:{remote_name}:{hashlib.sha256(path.encode()).hexdigest()[:16]}"
+
+    def is_index_valid(self, remote_name: str, path: str) -> bool:
+        """Check if mutable file is still within its TTL window."""
         if not self.available:
             return False
 
@@ -41,7 +43,7 @@ class RedisCache:
             return False
 
     def mark_index_cached(self, remote_name: str, path: str, ttl: int = 300) -> None:
-        """Mark index file as cached with TTL"""
+        """Set or refresh the TTL key for a mutable file."""
         if not self.available:
             return
 
@@ -51,13 +53,45 @@ class RedisCache:
         except Exception:
             pass
 
+    def store_mutable_meta(self, remote_name: str, path: str, etag: str | None, last_modified: str | None) -> None:
+        """Persist ETag and Last-Modified for future conditional requests."""
+        if not self.available:
+            return
+        data = {}
+        if etag:
+            data["etag"] = etag
+        if last_modified:
+            data["last_modified"] = last_modified
+        if not data:
+            return
+        try:
+            self.client.hset(self.get_mutable_meta_key(remote_name, path), mapping=data)
+        except Exception:
+            pass
+
+    def get_mutable_meta(self, remote_name: str, path: str) -> dict:
+        """Return stored ETag/Last-Modified for a mutable file, or {}."""
+        if not self.available:
+            return {}
+        try:
+            return self.client.hgetall(self.get_mutable_meta_key(remote_name, path)) or {}
+        except Exception:
+            return {}
+
+    def delete_mutable_meta(self, remote_name: str, path: str) -> None:
+        if not self.available:
+            return
+        try:
+            self.client.delete(self.get_mutable_meta_key(remote_name, path))
+        except Exception:
+            pass
+
     def cleanup_expired_index(self, storage, remote_name: str, path: str) -> None:
-        """Remove expired index from S3 storage"""
+        """Remove an expired mutable file from S3 and clear its Redis meta."""
         if not self.available:
             return
 
         try:
-            # Construct the URL the same way as in the main flow
             import os
 
             from .config import ConfigManager
@@ -69,9 +103,10 @@ class RedisCache:
                 if remote_config:
                     base_url = remote_config.get("base_url")
                     if base_url:
-                        # Use hierarchical path-based key (same as cache_single_artifact)
                         s3_key = storage.get_object_key(remote_name, path)
                         if storage.exists(s3_key):
                             storage.client.delete_object(Bucket=storage.bucket, Key=s3_key)
         except Exception:
             pass
+
+        self.delete_mutable_meta(remote_name, path)

src/artifactapi/config.py

@@ -124,6 +124,13 @@ class ConfigManager:
         db_url = f"postgresql://{db_user}:{db_pass}@{db_host}:{db_port}/{db_name}"
         return {"url": db_url}
 
+    def get_user_mutable_patterns(self, remote_name: str) -> list[str]:
+        """Return only user-configured mutable_patterns, excluding package-type defaults."""
+        remote_config = self.get_remote_config(remote_name)
+        if not remote_config:
+            return []
+        return remote_config.get("mutable_patterns", [])
+
     def get_mutable_patterns(self, remote_name: str) -> list[str]:
         """Return mutable-file patterns for a remote (TTL is configured per-remote in cache.index_ttl).
 

src/artifactapi/main.py

@@ -32,6 +32,10 @@ class ArtifactRequest(BaseModel):
     include_pattern: str
 
 
+class UpstreamUnreachable(Exception):
+    """Raised when the upstream backend cannot be contacted (network or timeout error)."""
+
+
 # Configure logging
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 logger = logging.getLogger(__name__)
@@ -87,8 +91,10 @@ def flush_cache(
             if cache_type in ["all", "index"]:
                 if remote:
                     patterns.append(f"index:{remote}:*")
+                    patterns.append(f"mutable:meta:{remote}:*")
                 else:
                     patterns.append("index:*")
+                    patterns.append("mutable:meta:*")
 
             if cache_type in ["all", "metrics"]:
                 if remote:
@@ -240,12 +246,80 @@ async def cache_single_artifact(url: str, remote_name: str, path: str) -> dict:
                 "storage_path": storage_path,
                 "size": len(response.content),
                 "status": "cached",
+                "etag": response.headers.get("ETag"),
+                "last_modified": response.headers.get("Last-Modified"),
             }
 
     except Exception as e:
         return {"url": url, "status": "error", "error": str(e)}
 
 
+async def _upstream_reachable(url: str) -> bool:
+    """HEAD with a short timeout. Returns False only on network/timeout errors."""
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            await client.head(url, timeout=10.0)
+        return True
+    except (httpx.NetworkError, httpx.TimeoutException):
+        return False
+    except Exception:
+        return True  # 4xx/5xx means backend is up
+
+
+async def check_upstream_changed(remote_url: str, remote_name: str, path: str) -> bool:
+    """Conditional HEAD against upstream. Returns False only on a definitive 304.
+    Raises UpstreamUnreachable if the backend cannot be contacted."""
+    meta = cache.get_mutable_meta(remote_name, path)
+    if not meta:
+        return True
+
+    headers = {}
+    if meta.get("etag"):
+        headers["If-None-Match"] = meta["etag"]
+    if meta.get("last_modified"):
+        headers["If-Modified-Since"] = meta["last_modified"]
+    if not headers:
+        return True
+
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            response = await client.head(remote_url, headers=headers)
+        return response.status_code != 304
+    except (httpx.NetworkError, httpx.TimeoutException) as exc:
+        raise UpstreamUnreachable(str(exc)) from exc
+
+
+async def handle_expired_mutable(remote_name: str, path: str, remote_url: str) -> bool:
+    """Handle an expired mutable file. Returns True if the cached copy is still valid."""
+    mutable_ttl = config.get_cache_config(remote_name).get("mutable_ttl", 3600)
+
+    remote_cfg = config.get_remote_config(remote_name) or {}
+    check_updates = remote_cfg.get("check_mutable_updates", False)
+    user_mutable = check_updates and cache.is_mutable_file(path, config.get_user_mutable_patterns(remote_name))
+
+    if user_mutable:
+        try:
+            changed = await check_upstream_changed(remote_url, remote_name, path)
+        except UpstreamUnreachable:
+            cache.mark_index_cached(remote_name, path, mutable_ttl)
+            logger.warning(f"Mutable STALE (backend unreachable): {remote_name}/{path} - TTL extended ({mutable_ttl}s)")
+            return True
+        if not changed:
+            cache.mark_index_cached(remote_name, path, mutable_ttl)
+            logger.info(f"Mutable file UNCHANGED: {remote_name}/{path} - TTL refreshed ({mutable_ttl}s)")
+            return True
+        logger.info(f"Mutable file CHANGED: {remote_name}/{path} - re-downloading")
+    else:
+        if not await _upstream_reachable(remote_url):
+            cache.mark_index_cached(remote_name, path, mutable_ttl)
+            logger.warning(f"Mutable STALE (backend unreachable): {remote_name}/{path} - TTL extended ({mutable_ttl}s)")
+            return True
+        logger.info(f"Mutable file EXPIRED: {remote_name}/{path} - removing from cache")
+
+    cache.cleanup_expired_index(storage, remote_name, path)
+    return False
+
+
 @app.get("/api/v1/remote/{remote_name}/{path:path}")
 async def get_artifact(remote_name: str, path: str):
     # Check if remote is configured
@@ -302,9 +376,8 @@ async def get_artifact(remote_name: str, path: str):
 
     if cached_key and is_mutable:
         if not cache.is_index_valid(remote_name, path):
-            logger.info(f"Mutable file EXPIRED: {remote_name}/{path} - removing from cache")
-            cache.cleanup_expired_index(storage, remote_name, path)
-            cached_key = None  # Force re-download
+            if not await handle_expired_mutable(remote_name, path, remote_url):
+                cached_key = None
 
     if cached_key:
         # Return cached artifact
@@ -362,6 +435,8 @@ async def get_artifact(remote_name: str, path: str):
         mutable_ttl = cache_config.get("mutable_ttl", 3600)
         cache.mark_index_cached(remote_name, path, mutable_ttl)
         logger.info(f"Mutable file cached with TTL: {remote_name}/{path} (ttl: {mutable_ttl}s)")
+        if result.get("etag") or result.get("last_modified"):
+            cache.store_mutable_meta(remote_name, path, result.get("etag"), result.get("last_modified"))
 
     # Now return the cached artifact
     try:
@@ -439,8 +514,7 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
 
     if cached_key and is_mutable:
         if not cache.is_index_valid(remote_name, path):
-            logger.info(f"Mutable file EXPIRED: {remote_name}/{path} - removing from cache")
-            cache.cleanup_expired_index(storage, remote_name, path)
+            if not await handle_expired_mutable(remote_name, path, remote_url):
                 cached_key = None
 
     if not cached_key:
@@ -453,6 +527,8 @@ async def docker_v2_proxy(request: Request, remote_name: str, path: str):
             mutable_ttl = cache_config.get("mutable_ttl", 3600)
             cache.mark_index_cached(remote_name, path, mutable_ttl)
             logger.info(f"Mutable file cached with TTL: {remote_name}/{path} (ttl: {mutable_ttl}s)")
+            if result.get("etag") or result.get("last_modified"):
+                cache.store_mutable_meta(remote_name, path, result.get("etag"), result.get("last_modified"))
 
     artifact_data = storage.download_object(storage.get_object_key(remote_name, path))
 

tests/conftest.py

@@ -59,6 +59,14 @@ TEST_REMOTES = {
             "mutable_patterns": ["metadata\\.json$"],
             "cache": {"immutable_ttl": 0, "mutable_ttl": 600},
         },
+        "check-mutable-test": {
+            "base_url": "https://example.com",
+            "type": "remote",
+            "package": "generic",
+            "mutable_patterns": ["metadata\\.json$"],
+            "check_mutable_updates": True,
+            "cache": {"immutable_ttl": 0, "mutable_ttl": 600},
+        },
         "local-test": {
             "type": "local",
             "package": "generic",

tests/test_cache.py

@@ -235,3 +235,51 @@ class TestIndexValidity:
         # client is None when Redis is unavailable — setex cannot be called
         assert unavailable_cache.client is None
         unavailable_cache.mark_index_cached("remote", "some/path", 300)  # must not raise
+
+
+# ---------------------------------------------------------------------------
+# mutable meta (ETag / Last-Modified storage)
+# ---------------------------------------------------------------------------
+
+
+class TestMutableMeta:
+    def test_meta_key_format(self, bare_cache):
+        path = "repo/metadata.json"
+        expected_hash = hashlib.sha256(path.encode()).hexdigest()[:16]
+        assert bare_cache.get_mutable_meta_key("myremote", path) == f"mutable:meta:myremote:{expected_hash}"
+
+    def test_meta_key_hash_is_16_chars(self, bare_cache):
+        key = bare_cache.get_mutable_meta_key("remote", "some/path/file.json")
+        assert len(key.split(":")[-1]) == 16
+
+    def test_store_and_retrieve_etag(self, cache_with_redis, mock_redis_client):
+        mock_redis_client.hgetall.return_value = {"etag": '"abc123"'}
+        cache_with_redis.store_mutable_meta("remote", "path/meta.json", '"abc123"', None)
+        mock_redis_client.hset.assert_called_once()
+        meta = cache_with_redis.get_mutable_meta("remote", "path/meta.json")
+        assert meta["etag"] == '"abc123"'
+
+    def test_store_and_retrieve_last_modified(self, cache_with_redis, mock_redis_client):
+        lm = "Mon, 01 Jan 2024 00:00:00 GMT"
+        mock_redis_client.hgetall.return_value = {"last_modified": lm}
+        cache_with_redis.store_mutable_meta("remote", "path/meta.json", None, lm)
+        meta = cache_with_redis.get_mutable_meta("remote", "path/meta.json")
+        assert meta["last_modified"] == lm
+
+    def test_store_no_op_when_both_none(self, cache_with_redis, mock_redis_client):
+        cache_with_redis.store_mutable_meta("remote", "path/meta.json", None, None)
+        mock_redis_client.hset.assert_not_called()
+
+    def test_store_no_op_when_unavailable(self, unavailable_cache):
+        unavailable_cache.store_mutable_meta("remote", "path", "etag", None)  # must not raise
+
+    def test_get_returns_empty_when_unavailable(self, unavailable_cache):
+        assert unavailable_cache.get_mutable_meta("remote", "path") == {}
+
+    def test_delete_removes_meta_key(self, cache_with_redis, mock_redis_client):
+        expected_key = cache_with_redis.get_mutable_meta_key("remote", "path/meta.json")
+        cache_with_redis.delete_mutable_meta("remote", "path/meta.json")
+        mock_redis_client.delete.assert_called_once_with(expected_key)
+
+    def test_delete_no_op_when_unavailable(self, unavailable_cache):
+        unavailable_cache.delete_mutable_meta("remote", "path")  # must not raise

tests/test_config.py

@@ -208,6 +208,39 @@ class TestGetImmutablePatterns:
         assert cfg.get_immutable_patterns("r", "/unknown/path") == [r".*\.tar\.gz$"]
 
 
+# ---------------------------------------------------------------------------
+# get_user_mutable_patterns
+# ---------------------------------------------------------------------------
+
+
+class TestGetUserMutablePatterns:
+    def test_returns_only_user_patterns(self, make_config):
+        cfg = make_config(
+            {
+                "r": {
+                    "type": "remote",
+                    "package": "alpine",
+                    "base_url": "https://x.com",
+                    "mutable_patterns": [r"custom\.json$"],
+                }
+            }
+        )
+        assert cfg.get_user_mutable_patterns("r") == [r"custom\.json$"]
+
+    def test_excludes_package_defaults(self, make_config):
+        # Package defaults (APKINDEX etc.) must NOT appear here
+        cfg = make_config({"r": {"type": "remote", "package": "alpine", "base_url": "https://x.com"}})
+        assert cfg.get_user_mutable_patterns("r") == []
+
+    def test_returns_empty_for_missing_remote(self, make_config):
+        cfg = make_config({})
+        assert cfg.get_user_mutable_patterns("nonexistent") == []
+
+    def test_returns_empty_when_key_absent(self, make_config):
+        cfg = make_config({"r": {"type": "remote", "package": "generic", "base_url": "https://x.com"}})
+        assert cfg.get_user_mutable_patterns("r") == []
+
+
 # ---------------------------------------------------------------------------
 # get_cache_config
 # ---------------------------------------------------------------------------

tests/test_routes.py

@@ -248,6 +248,7 @@ class TestDockerProxy:
         deps["cache"].is_index_valid.return_value = False  # but TTL expired
         deps["storage"].download_object.return_value = manifest
 
+        with patch("artifactapi.main._upstream_reachable", new_callable=AsyncMock, return_value=True):
             with patch(
                 "artifactapi.main.cache_single_artifact",
                 new_callable=AsyncMock,
@@ -419,6 +420,103 @@ class TestGenericArtifactRoute:
         response = client.get("/api/v1/remote/alpine-test/alpine/v3.18/x86_64/APKINDEX.tar.gz")
         assert response.status_code == 200
 
+    def test_mutable_unchanged_refreshes_ttl_without_redownload(self, client, patched_deps):
+        """When check_mutable_updates=True and upstream says 304, TTL is refreshed in place."""
+        deps = patched_deps
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = b"metadata content"
+        # File is mutable and its TTL has expired
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = False
+        deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
+
+        with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock, return_value=False):
+            response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
+
+        assert response.status_code == 200
+        deps["cache"].mark_index_cached.assert_called()
+        # S3 object must NOT have been deleted (no re-download)
+        deps["storage"].client.delete_object.assert_not_called()
+
+    def test_mutable_changed_triggers_redownload(self, client, patched_deps):
+        """When check_mutable_updates=True and upstream says 200, cache is invalidated."""
+        deps = patched_deps
+        deps["storage"].exists.return_value = False
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = False
+        deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
+
+        with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock, return_value=True):
+            with patch("artifactapi.main.cache_single_artifact", new_callable=AsyncMock) as mock_cache:
+                mock_cache.return_value = {"status": "error", "error": "upstream gone"}
+                response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
+
+        assert response.status_code == 502
+
+    def test_mutable_changed_redownloads_successfully(self, client, patched_deps):
+        """When check_mutable_updates=True and upstream says 200, fresh copy is fetched and served."""
+        deps = patched_deps
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = b"fresh metadata"
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = False
+        deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
+
+        with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock, return_value=True):
+            with patch("artifactapi.main.cache_single_artifact", new_callable=AsyncMock) as mock_cache:
+                mock_cache.return_value = {"status": "cached", "etag": '"def"', "last_modified": None}
+                response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
+
+        assert response.status_code == 200
+        mock_cache.assert_called_once()
+
+    def test_mutable_backend_unreachable_on_check_updates_keeps_stale(self, client, patched_deps):
+        """When check_mutable_updates=True and backend is unreachable, stale copy is kept and TTL refreshed."""
+        from artifactapi.main import UpstreamUnreachable
+
+        deps = patched_deps
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = b"stale metadata"
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = False
+        deps["cache"].get_mutable_meta.return_value = {"etag": '"abc"'}
+
+        with patch("artifactapi.main.check_upstream_changed", side_effect=UpstreamUnreachable("connection refused")):
+            response = client.get("/api/v1/remote/check-mutable-test/metadata.json")
+
+        assert response.status_code == 200
+        deps["cache"].mark_index_cached.assert_called()
+        deps["storage"].client.delete_object.assert_not_called()
+
+    def test_mutable_backend_unreachable_on_expiry_keeps_stale(self, client, patched_deps):
+        """When a regular mutable file expires and backend is unreachable, stale copy is kept and TTL refreshed."""
+        deps = patched_deps
+        deps["storage"].exists.return_value = True
+        deps["storage"].download_object.return_value = b"stale APKINDEX"
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = False
+
+        with patch("artifactapi.main._upstream_reachable", new_callable=AsyncMock, return_value=False):
+            response = client.get("/api/v1/remote/alpine-test/alpine/v3.18/x86_64/APKINDEX.tar.gz")
+
+        assert response.status_code == 200
+        deps["cache"].mark_index_cached.assert_called()
+        deps["storage"].client.delete_object.assert_not_called()
+
+    def test_mutable_flag_off_skips_conditional_check(self, client, patched_deps):
+        """When check_mutable_updates is not set, expired mutable files are always re-fetched."""
+        deps = patched_deps
+        deps["storage"].exists.return_value = False
+        deps["cache"].is_mutable_file.return_value = True
+        deps["cache"].is_index_valid.return_value = False
+
+        with patch("artifactapi.main.check_upstream_changed", new_callable=AsyncMock) as mock_check:
+            with patch("artifactapi.main.cache_single_artifact", new_callable=AsyncMock) as mock_cache:
+                mock_cache.return_value = {"status": "error", "error": "upstream gone"}
+                client.get("/api/v1/remote/custom-index-test/metadata.json")
+
+        mock_check.assert_not_called()
+
     def test_local_repo_file_not_found_returns_404(self, client, patched_deps):
         deps = patched_deps
         deps["database"].get_local_file_metadata.return_value = None
@@ -519,8 +617,8 @@ class TestCacheFlushEndpoint:
         deps["cache"].available = True
         redis_mock = MagicMock()
         deps["cache"].client = redis_mock
-        # First pattern (index:*) returns keys; subsequent pattern returns nothing
-        redis_mock.keys.side_effect = [["index:test:abc", "index:test:def"], []]
+        # index:* returns keys; mutable:meta:* and metrics:* return nothing
+        redis_mock.keys.side_effect = [["index:test:abc", "index:test:def"], [], []]
         deps["storage"].client.list_objects_v2.return_value = {}
 
         response = client.put("/cache/flush")