clickhouse-tools/internal/chlog/pager.go

package chlog

import (
	"context"
	"time"
)

const DefaultPageSize = 10000

type runner interface {
	Run(ctx context.Context, q Query, fn func(Row) error) error
}

// Page walks the filter's time range as a series of bounded keyset-paged
// queries: each page re-queries from the last-seen timestamp (inclusive, so
// nothing on the boundary millisecond is lost) and dedupes the overlap. Each
// page requests len(seen) extra rows on top of the wanted count, so known
// boundary duplicates can never starve progress. Returns rows emitted.
func Page(ctx context.Context, c runner, f Filter, pageSize uint64, emit func(Row) error) (uint64, error) {
	if pageSize == 0 {
		pageSize = DefaultPageSize
	}
	budget := f.Limit

	cursor := f.Since
	seen := map[uint64]struct{}{}
	var emitted uint64

	for {
		want := pageSize
		if budget > 0 && budget-emitted < want {
			want = budget - emitted
		}
		pf := f
		pf.Since = cursor
		pf.Limit = want + uint64(len(seen))
		q, err := Build(pf)
		if err != nil {
			return emitted, err
		}

		var got uint64
		var lastTS time.Time
		pageSeen := map[uint64]struct{}{}
		err = c.Run(ctx, q, func(r Row) error {
			got++
			if budget > 0 && emitted >= budget {
				return nil
			}
			ts := r.Time()
			k := r.Key()
			if ts.Equal(cursor) {
				if _, dup := seen[k]; dup {
					return nil
				}
				seen[k] = struct{}{}
			} else {
				if !ts.Equal(lastTS) {
					lastTS = ts
					pageSeen = map[uint64]struct{}{}
				}
				pageSeen[k] = struct{}{}
			}
			emitted++
			return emit(r)
		})
		if err != nil {
			return emitted, err
		}
		if got < pf.Limit || (budget > 0 && emitted >= budget) {
			return emitted, nil
		}
		if lastTS.After(cursor) {
			cursor = lastTS
			seen = pageSeen
		}
	}
}