From d69a58e54a8dd7c9bef590955a01afa0ba28a319 Mon Sep 17 00:00:00 2001 From: unkin-agent Date: Sat, 17 Oct 2026 17:17:41 +1100 Subject: [PATCH] Fix mutable path refresh: default mutable_ttl, revalidate after expiry (#151) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A remote created through the API without `mutable_ttl` was stored as 0, so its mutable paths never refreshed. With `check_mutable`, the ETag expired together with the freshness key, so an expired mutable path was always refetched in full and never revalidated. - default an omitted `mutable_ttl` to 3600 on create; keep the stored value on update; explicit 0 still means never expire - keep mutable ETags for `max(10 × mutable_ttl, 7d)`, re-armed on each 304, only when `check_mutable` is set and `mutable_ttl` is non-zero - drop the stored ETag when a refetch returns none; never store one for immutable paths - flush a remote's Redis keys on delete - update docs/common/caching.md Reviewed-on: https://git.unkin.net/unkin/artifactapi/pulls/151 Co-authored-by: unkin-agent Co-committed-by: unkin-agent --- docs/common/caching.md | 7 +- internal/api/v2/remotes.go | 30 +++-- internal/api/v2/remotes_flush_test.go | 25 ++++ internal/api/v2/remotes_ttl_test.go | 57 +++++++++ internal/cache/cache_test.go | 6 + internal/cache/redis.go | 4 + internal/proxy/engine.go | 16 ++- internal/proxy/engine_test.go | 176 +++++++++++++++++++++++--- internal/proxy/evict_test.go | 2 +- 9 files changed, 286 insertions(+), 37 deletions(-) create mode 100644 internal/api/v2/remotes_ttl_test.go diff --git a/docs/common/caching.md b/docs/common/caching.md index 40d70e3..f74bd75 100644 --- a/docs/common/caching.md +++ b/docs/common/caching.md @@ -12,8 +12,8 @@ Every proxied request on a remote is classified as **denied**, **immutable** or | `immutable_patterns` | Regexes forcing a path immutable. | | `mutable_patterns` | Regexes forcing a path mutable. | | `immutable_ttl` | Seconds an immutable path stays fresh. `0` = forever. | -| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires, and the API stores `0` when the field is omitted. | -| `check_mutable` | Stores the upstream ETag and tries a conditional `HEAD` (`If-None-Match`) when a mutable path is fetched. The ETag expires with the `mutable_ttl` freshness key, so an expired mutable path is refetched in full, not revalidated. | +| `mutable_ttl` | Seconds a mutable path stays fresh. `0` also means never expires. Omitted on create, it defaults to `3600`; omitted on update, the stored value is kept. | +| `check_mutable` | When a mutable path's `mutable_ttl` expires, sends a conditional `HEAD` (`If-None-Match`) with the stored upstream ETag. A `304` re-arms the TTL and serves the stored copy; anything else refetches in full. The ETag is kept for `max(10 × mutable_ttl, 7 days)`, re-armed on each `304`, so idle paths age out. It is only stored when `check_mutable` is set and `mutable_ttl` is non-zero, and never for immutable paths. | | `stale_on_error` | On an upstream network error, serve the stored copy instead of failing. | | `quarantine_enabled` / `quarantine_days` | Stored on the remote but not enforced by the proxy yet. | | `mirrorlist` / `mirror_strategy` | rpm/deb/alpine only. Upstream pool is `[base_url] + mirrorlist`; `round_robin` (default) rotates the starting mirror, `least_conn` starts at the one with the fewest in-flight requests. Network errors and 5xx fail over to the next mirror; other statuses return as-is. | @@ -46,7 +46,7 @@ So `blocklist` beats the allowlist, and `immutable_patterns` beats |---|---|---| | Stored as | content-addressed blob (sha256), streamed | per-path index object, buffered and rewritten by the provider | | Fresh for | `immutable_ttl` (`0` = forever) | `mutable_ttl` | -| After expiry | refetched from upstream | refetched from upstream (the ETag expires with the TTL, so `check_mutable` does not revalidate it) | +| After expiry | refetched from upstream | revalidated with `If-None-Match` when `check_mutable` is set and an ETag is stored, else refetched | | Upstream down | `stale_on_error` serves the stored copy and re-arms the TTL; an open circuit breaker always serves stale if a copy exists, else 503 | same | Concurrent misses on one path share a fetch lock. Waiters poll the store for @@ -60,6 +60,7 @@ curl -X DELETE https://artifactapi.example.com/api/v2/remotes//objects//*` and takes no lock. - Returns 204, or 404 for an unknown remote. diff --git a/internal/api/v2/remotes.go b/internal/api/v2/remotes.go index a8f679e..b5c3a82 100644 --- a/internal/api/v2/remotes.go +++ b/internal/api/v2/remotes.go @@ -67,8 +67,12 @@ func (h *RemotesHandler) get(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, remote) } +// defaultMutableTTL matches the remotes.mutable_ttl column default; a create +// that omits mutable_ttl gets it, an explicit 0 still means never expire. +const defaultMutableTTL = 3600 + func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) { - var remote models.Remote + remote := models.Remote{MutableTTL: defaultMutableTTL} if err := json.NewDecoder(r.Body).Decode(&remote); err != nil { http.Error(w, "invalid json", http.StatusBadRequest) return @@ -114,7 +118,15 @@ func (h *RemotesHandler) create(w http.ResponseWriter, r *http.Request) { func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) { name := chi.URLParam(r, "name") + // Capture the current backend before the update so we can tell whether the + // remote's base_url (its upstream) changed. A read failure just means we + // skip the freshness flush; it must not block the update. + existing, err := h.db.GetRemote(r.Context(), name) + oldKnown := err == nil var remote models.Remote + if oldKnown { + remote.MutableTTL = existing.MutableTTL + } if err := json.NewDecoder(r.Body).Decode(&remote); err != nil { http.Error(w, "invalid json", http.StatusBadRequest) return @@ -132,13 +144,6 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - // Capture the current backend before the update so we can tell whether the - // remote's base_url (its upstream) changed. A read failure just means we - // skip the freshness flush; it must not block the update. - oldBaseURL, oldKnown := "", false - if existing, err := h.db.GetRemote(r.Context(), name); err == nil { - oldBaseURL, oldKnown = existing.BaseURL, true - } if err := h.db.UpdateRemote(r.Context(), &remote); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -147,13 +152,13 @@ func (h *RemotesHandler) update(w http.ResponseWriter, r *http.Request) { // Release / APKINDEX): purge it so the next request re-fetches from the new // upstream instead of serving stale data until TTL expiry. A flush failure // is logged but does not fail the request — the DB update already landed. - if oldKnown && oldBaseURL != remote.BaseURL && h.cache != nil { + if oldKnown && existing.BaseURL != remote.BaseURL && h.cache != nil { if err := h.cache.FlushRemote(r.Context(), name); err != nil { slog.Warn("flush cached metadata after base_url change failed", "remote", name, "error", err) } else { slog.Info("flushed cached metadata after base_url change", - "remote", name, "old_base_url", oldBaseURL, "new_base_url", remote.BaseURL) + "remote", name, "old_base_url", existing.BaseURL, "new_base_url", remote.BaseURL) } } writeJSON(w, http.StatusOK, remote) @@ -165,6 +170,11 @@ func (h *RemotesHandler) del(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } + if h.cache != nil { + if err := h.cache.FlushRemote(r.Context(), name); err != nil { + slog.Warn("flush cached metadata after delete failed", "remote", name, "error", err) + } + } w.WriteHeader(http.StatusNoContent) } diff --git a/internal/api/v2/remotes_flush_test.go b/internal/api/v2/remotes_flush_test.go index 09a25b5..4f8036a 100644 --- a/internal/api/v2/remotes_flush_test.go +++ b/internal/api/v2/remotes_flush_test.go @@ -94,3 +94,28 @@ func TestUpdateFlushFailureStillSucceeds(t *testing.T) { t.Fatalf("flush calls = %v, want exactly one attempted flush", ff.calls) } } + +// Deleting a remote flushes its cached state so a re-created remote of the +// same name starts cold; a flush error does not fail the delete. +func TestDeleteFlushesCache(t *testing.T) { + if testDSN == "" { + t.Skip("Docker unavailable") + } + db, err := database.New(testDSN) + if err != nil { + t.Fatal(err) + } + defer db.Close() + + for _, ff := range []*fakeFlusher{{}, {err: errors.New("redis down")}} { + const name = "rpm-flush-delete" + seedRemote(t, db, name, "https://example.com/repo") + h := NewRemotesHandler(db, ff, nil).Routes() + if c := do(t, h, "DELETE", "/"+name, ""); c != 204 { + t.Fatalf("delete = %d, want 204", c) + } + if len(ff.calls) != 1 || ff.calls[0] != name { + t.Fatalf("flush calls = %v, want exactly one flush of %q", ff.calls, name) + } + } +} diff --git a/internal/api/v2/remotes_ttl_test.go b/internal/api/v2/remotes_ttl_test.go new file mode 100644 index 0000000..7beb859 --- /dev/null +++ b/internal/api/v2/remotes_ttl_test.go @@ -0,0 +1,57 @@ +package v2 + +import ( + "context" + "testing" + + "git.unkin.net/unkin/artifactapi/internal/database" +) + +// An omitted mutable_ttl defaults to 3600 on create and keeps the stored value +// on update; an explicit 0 is stored as 0 in both. +func TestMutableTTLOmittedVsZero(t *testing.T) { + if testDSN == "" { + t.Skip("Docker unavailable") + } + db, err := database.New(testDSN) + if err != nil { + t.Fatal(err) + } + defer db.Close() + h := NewRemotesHandler(db, nil, nil).Routes() + ttl := func(name string) int { + t.Helper() + r, err := db.GetRemote(context.Background(), name) + if err != nil { + t.Fatalf("get %s: %v", name, err) + } + return r.MutableTTL + } + const body = `"package_type":"rpm","base_url":"https://example.com/repo"` + + steps := []struct { + method, path, json string + want int + }{ + {"POST", "/", `{"name":"ttl-omit",` + body + `}`, 3600}, + {"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":120}`, 120}, + {"PUT", "/ttl-omit", `{` + body + `}`, 120}, + {"PUT", "/ttl-omit", `{` + body + `,"mutable_ttl":0}`, 0}, + {"PUT", "/ttl-omit", `{` + body + `}`, 0}, + } + for i, s := range steps { + if c := do(t, h, s.method, s.path, s.json); c/100 != 2 { + t.Fatalf("step %d %s %s = %d", i, s.method, s.path, c) + } + if got := ttl("ttl-omit"); got != s.want { + t.Errorf("step %d: mutable_ttl = %d, want %d", i, got, s.want) + } + } + + if c := do(t, h, "POST", "/", `{"name":"ttl-zero",`+body+`,"mutable_ttl":0}`); c != 201 { + t.Fatalf("create explicit 0 = %d", c) + } + if got := ttl("ttl-zero"); got != 0 { + t.Errorf("explicit 0 stored as %d", got) + } +} diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 4093a60..8bcb380 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -86,6 +86,12 @@ func TestETagAndToken(t *testing.T) { if v, _ := testRedis.GetETag(ctx, "r", "p"); v != `"abc"` { t.Errorf("etag = %q", v) } + if err := testRedis.SetETag(ctx, "r", "p", "", 0); err != nil { + t.Fatal(err) + } + if v, _ := testRedis.GetETag(ctx, "r", "p"); v != "" { + t.Errorf("empty etag should drop the stored one, got %q", v) + } if v, _ := testRedis.GetToken(ctx, "missing"); v != "" { t.Error("missing token should be empty") diff --git a/internal/cache/redis.go b/internal/cache/redis.go index b33e0cc..eeca6c5 100644 --- a/internal/cache/redis.go +++ b/internal/cache/redis.go @@ -57,8 +57,12 @@ func (r *Redis) ReleaseLock(ctx context.Context, remote, path string) error { return r.client.Del(ctx, key).Err() } +// SetETag stores etag for path, or drops the stored one when etag is empty. func (r *Redis) SetETag(ctx context.Context, remote, path, etag string, ttl time.Duration) error { key := fmt.Sprintf("etag:%s:%s", remote, path) + if etag == "" { + return r.client.Del(ctx, key).Err() + } return r.client.Set(ctx, key, etag, ttl).Err() } diff --git a/internal/proxy/engine.go b/internal/proxy/engine.go index 45a1b87..2c8d497 100644 --- a/internal/proxy/engine.go +++ b/internal/proxy/engine.go @@ -160,7 +160,7 @@ func (e *Engine) Fetch(ctx context.Context, remote models.Remote, path string, p notModified, err := e.checkUpstream(ctx, remote, path, etag, prov) if err == nil && notModified { _ = e.cache.SetTTL(ctx, remote.Name, path, ttl) - _ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl) + _ = e.cache.SetETag(ctx, remote.Name, path, etag, etagTTL(ttl)) result, err := e.serveFromStore(ctx, remote, path) if err == nil { result.Source = "cache" @@ -491,8 +491,9 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote } _ = e.cache.SetTTL(ctx, remote.Name, path, ttl) - if etag := resp.Header.Get("ETag"); etag != "" { - _ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl) + // A zero TTL never expires, so it never needs revalidating. + if remote.CheckMutable && ttl > 0 { + _ = e.cache.SetETag(ctx, remote.Name, path, resp.Header.Get("ETag"), etagTTL(ttl)) } return &FetchResult{ @@ -520,9 +521,6 @@ func (e *Engine) fetchFromUpstreamOnce(ctx context.Context, remote models.Remote } _ = e.cache.SetTTL(ctx, remote.Name, path, ttl) - if etag := resp.Header.Get("ETag"); etag != "" { - _ = e.cache.SetETag(ctx, remote.Name, path, etag, ttl) - } reader, info, err := e.store.Download(ctx, casResult.S3Key) if err != nil { @@ -642,6 +640,12 @@ func (e *Engine) checkUpstreamOnce(ctx context.Context, remote models.Remote, pa return resp.StatusCode == http.StatusNotModified, nil } +// etagTTL keeps a mutable path's ETag well past its freshness key so an expired +// path revalidates with a conditional HEAD, while idle paths still age out. +func etagTTL(ttl time.Duration) time.Duration { + return max(10*ttl, 7*24*time.Hour) +} + func (e *Engine) ttlFor(remote models.Remote, class Classification) time.Duration { switch class { case ClassImmutable: diff --git a/internal/proxy/engine_test.go b/internal/proxy/engine_test.go index 87ce690..b4cf1ff 100644 --- a/internal/proxy/engine_test.go +++ b/internal/proxy/engine_test.go @@ -17,12 +17,14 @@ import ( "git.unkin.net/unkin/artifactapi/internal/storage" "git.unkin.net/unkin/artifactapi/internal/testsupport" "git.unkin.net/unkin/artifactapi/pkg/models" + goredis "github.com/redis/go-redis/v9" ) var ( testEngine *Engine testCache *cache.Redis testDB *database.DB + testRedis string upstream *httptest.Server ) @@ -64,6 +66,7 @@ func TestMain(m *testing.M) { } testCache = redis + testRedis = redisURL testDB = db testEngine = NewEngine(db, redis, s3) upstream = httptest.NewServer(http.HandlerFunc(mockUpstream)) @@ -260,30 +263,169 @@ func TestCircuitOpenServesStale(t *testing.T) { res.Reader.Close() } +// expire drops path's freshness key so the next fetch revalidates. +func expire(t *testing.T, remote, path string) { + t.Helper() + if err := testCache.SetTTL(context.Background(), remote, path, time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(5 * time.Millisecond) +} + +func mutableFetch(t *testing.T, r models.Remote, path string) (string, string) { + t.Helper() + res, err := testEngine.Fetch(context.Background(), r, path, prov(t, models.PackageRPM)) + if err != nil { + t.Fatalf("fetch: %v", err) + } + return readAll(t, res), res.Source +} + +const repomd = "8/Everything/x86_64/repodata/repomd.xml" + func TestMutableRevalidation(t *testing.T) { + requireStack(t) + srv, rev, conditional := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-reval", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60}) + + if body, _ := mutableFetch(t, r, repomd); body != "rev1:/"+repomd { + t.Fatalf("initial fetch = %q", body) + } + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" { + t.Errorf("after expiry = %q from %s, want cached rev1", body, src) + } + if n := conditional.Load(); n != 1 { + t.Errorf("conditional requests = %d, want 1 revalidation", n) + } + if fresh, _ := testCache.CheckTTL(context.Background(), r.Name, repomd); !fresh { + t.Error("304 should re-arm the freshness key") + } + + rev.Store("rev2") + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev2:/"+repomd || src != "remote" { + t.Errorf("after upstream change = %q from %s, want remote rev2", body, src) + } +} + +func etagKeyTTL(t *testing.T, remote, path string) time.Duration { + t.Helper() + opts, err := goredis.ParseURL(testRedis) + if err != nil { + t.Fatal(err) + } + c := goredis.NewClient(opts) + defer func() { _ = c.Close() }() + d, err := c.PTTL(context.Background(), "etag:"+remote+":"+path).Result() + if err != nil { + t.Fatal(err) + } + return d +} + +// The ETag is only kept when it can be used, and never without an expiry. +func TestMutableETagStorage(t *testing.T) { + requireStack(t) + srv, _, _ := changingUpstream(t) + for _, tc := range []struct { + name string + check bool + ttl int + want time.Duration + }{ + {"eng-etag-off", false, 60, -2}, + {"eng-etag-never", true, 0, -2}, + {"eng-etag-short", true, 60, 7 * 24 * time.Hour}, + {"eng-etag-long", true, 86400, 10 * 24 * time.Hour}, + } { + r := seed(t, models.Remote{Name: tc.name, PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: tc.check, MutableTTL: tc.ttl}) + mutableFetch(t, r, repomd) + got := etagKeyTTL(t, tc.name, repomd) + if tc.want < 0 { + if got != tc.want { + t.Errorf("%s: etag key ttl = %v, want no key", tc.name, got) + } + continue + } + if got <= tc.want-time.Minute || got > tc.want { + t.Errorf("%s: etag key ttl = %v, want ~%v", tc.name, got, tc.want) + } + } +} + +// Immutable paths are never revalidated, so their upstream ETag is not kept. +func TestImmutableFetchStoresNoETag(t *testing.T) { + requireStack(t) + srv, _, _ := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-etag-imm", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60}) + const rpm = "Packages/e2e-1.0-1.noarch.rpm" + if body, src := mutableFetch(t, r, rpm); body != "rev1:/"+rpm || src != "remote" { + t.Fatalf("fetch = %q from %s", body, src) + } + if got := etagKeyTTL(t, r.Name, rpm); got != -2 { + t.Errorf("etag key ttl = %v, want no key", got) + } +} + +// A 304 for a path whose S3 body is gone falls back to a full refetch. +func TestNotModifiedWithEvictedBodyRefetches(t *testing.T) { + requireStack(t) + srv, _, conditional := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-304-gone", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60}) + + mutableFetch(t, r, repomd) + if err := testEngine.store.Delete(context.Background(), storage.IndexKey(r.Name, repomd)); err != nil { + t.Fatal(err) + } + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "remote" { + t.Errorf("got %q from %s, want refetched rev1 from remote", body, src) + } + if n := conditional.Load(); n != 1 { + t.Errorf("conditional requests = %d, want the 304 HEAD first", n) + } +} + +// A failing revalidation HEAD still serves the stored copy with stale_on_error. +func TestStaleOnErrorWhenRevalidationFails(t *testing.T) { + requireStack(t) + srv, _, _ := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-stale-head", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: srv.URL, CheckMutable: true, MutableTTL: 60, StaleOnError: true}) + + mutableFetch(t, r, repomd) + srv.Close() + expire(t, r.Name, repomd) + if body, src := mutableFetch(t, r, repomd); body != "rev1:/"+repomd || src != "cache" { + t.Errorf("got %q from %s, want stale rev1 from cache", body, src) + } +} + +// After a remote is deleted and flushed, a re-created remote of the same name +// fetches from its own upstream instead of revalidating the old ETag. +func TestRecreatedRemoteHasNoStaleState(t *testing.T) { requireStack(t) ctx := context.Background() - r := seed(t, models.Remote{Name: "eng-npm", PackageType: models.PackageNPM, RepoType: models.RepoTypeRemote, BaseURL: upstream.URL, CheckMutable: true, MutableTTL: 3600, StaleOnError: true}) - p := prov(t, models.PackageNPM) + oldSrv, _, _ := changingUpstream(t) + r := seed(t, models.Remote{Name: "eng-recreate", PackageType: models.PackageRPM, RepoType: models.RepoTypeRemote, BaseURL: oldSrv.URL, CheckMutable: true, MutableTTL: 60}) + mutableFetch(t, r, repomd) - res, err := testEngine.Fetch(ctx, r, "pkg", p) - if err != nil { - t.Fatalf("initial mutable fetch: %v", err) + if err := testDB.DeleteRemote(ctx, r.Name); err != nil { + t.Fatal(err) } - res.Reader.Close() - - // Expire only the freshness marker; the ETag persists, forcing a - // conditional revalidation that the upstream answers with 304. - testCache.SetTTL(ctx, "eng-npm", "pkg", time.Millisecond) - time.Sleep(10 * time.Millisecond) - res, err = testEngine.Fetch(ctx, r, "pkg", p) - if err != nil { - t.Fatalf("revalidation fetch: %v", err) + if err := testCache.FlushRemote(ctx, r.Name); err != nil { + t.Fatal(err) } - if res.Source != "cache" { - t.Errorf("revalidated response should come from cache, got %s", res.Source) + newSrv, rev, conditional := changingUpstream(t) + rev.Store("rev9") + r.BaseURL = newSrv.URL + seed(t, r) + if body, src := mutableFetch(t, r, repomd); body != "rev9:/"+repomd || src != "remote" { + t.Errorf("got %q from %s, want rev9 from the new upstream", body, src) + } + if n := conditional.Load(); n != 0 { + t.Errorf("conditional requests = %d, want none after flush", n) } - res.Reader.Close() } func TestBearerTokenFlow(t *testing.T) { diff --git a/internal/proxy/evict_test.go b/internal/proxy/evict_test.go index 15b8def..622cc35 100644 --- a/internal/proxy/evict_test.go +++ b/internal/proxy/evict_test.go @@ -59,7 +59,7 @@ type cached struct{ row, index, ttl, etag bool } // Mutable indexes live in the S3 index; immutable blobs get an artifact row. var ( indexCached = cached{index: true, ttl: true, etag: true} - blobCached = cached{row: true, ttl: true, etag: true} + blobCached = cached{row: true, ttl: true} ) func layers(t *testing.T, remote, path string) cached {