package chlog import ( "context" "time" ) const DefaultPageSize = 10000 type runner interface { Run(ctx context.Context, q Query, fn func(Row) error) error } // Page walks the filter's time range as a series of bounded keyset-paged // queries: each page re-queries from the last-seen timestamp (inclusive, so // nothing on the boundary millisecond is lost) and dedupes the overlap. Each // page requests len(seen) extra rows on top of the wanted count, so known // boundary duplicates can never starve progress. Returns rows emitted. func Page(ctx context.Context, c runner, f Filter, pageSize uint64, emit func(Row) error) (uint64, error) { if pageSize == 0 { pageSize = DefaultPageSize } budget := f.Limit cursor := f.Since seen := map[uint64]struct{}{} var emitted uint64 for { want := pageSize if budget > 0 && budget-emitted < want { want = budget - emitted } pf := f pf.Since = cursor pf.Limit = want + uint64(len(seen)) q, err := Build(pf) if err != nil { return emitted, err } var got uint64 var lastTS time.Time pageSeen := map[uint64]struct{}{} err = c.Run(ctx, q, func(r Row) error { got++ if budget > 0 && emitted >= budget { return nil } ts := r.Time() k := r.Key() if ts.Equal(cursor) { if _, dup := seen[k]; dup { return nil } seen[k] = struct{}{} } else { if !ts.Equal(lastTS) { lastTS = ts pageSeen = map[uint64]struct{}{} } pageSeen[k] = struct{}{} } emitted++ return emit(r) }) if err != nil { return emitted, err } if got < pf.Limit || (budget > 0 && emitted >= budget) { return emitted, nil } if lastTS.After(cursor) { cursor = lastTS seen = pageSeen } } }