add a filtered search mode to history search

pkg/cmdrunner/cmdrunner.go

@@ -1858,7 +1858,11 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 	if err != nil {
 		return nil, err
 	}
-	opts := sstore.HistoryQueryOpts{MaxItems: HistoryViewPageSize + 1, Offset: offset}
+	rawOffset, err := resolveNonNegInt(pk.Kwargs["rawoffset"], 0)
+	if err != nil {
+		return nil, err
+	}
+	opts := sstore.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset}
 	if pk.Kwargs["text"] != "" {
 		opts.SearchText = pk.Kwargs["text"]
 	}
@@ -1893,17 +1897,15 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType
 	if err != nil {
 		return nil, fmt.Errorf("invalid meta arg (must be boolean): %v", err)
 	}
-	hitems, err := sstore.GetHistoryItems(ctx, opts)
+	hresult, err := sstore.GetHistoryItems(ctx, opts)
 	if err != nil {
 		return nil, err
 	}
-	hvdata := &sstore.HistoryViewData{Offset: offset}
+	hvdata := &sstore.HistoryViewData{
-	if len(hitems) > HistoryViewPageSize {
+		Items:         hresult.Items,
-		hvdata.HasMore = true
+		Offset:        hresult.Offset,
-		hvdata.Items = hitems[0:HistoryViewPageSize]
+		NextRawOffset: hresult.NextRawOffset,
-	} else {
+		HasMore:       hresult.HasMore,
-		hvdata.HasMore = false
-		hvdata.Items = hitems
 	}
 	lines, cmds, err := sstore.GetLineCmdsFromHistoryItems(ctx, hvdata.Items)
 	if err != nil {
@@ -1951,7 +1953,7 @@ func HistoryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (ssto
 		hWindowId = ""
 	}
 	hopts := sstore.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, WindowId: hWindowId}
-	hitems, err := sstore.GetHistoryItems(ctx, hopts)
+	hresult, err := sstore.GetHistoryItems(ctx, hopts)
 	if err != nil {
 		return nil, err
 	}
@@ -1967,7 +1969,7 @@ func HistoryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (ssto
 		HistoryType: htype,
 		SessionId:   ids.SessionId,
 		WindowId:    ids.WindowId,
-		Items:       hitems,
+		Items:       hresult.Items,
 		Show:        show,
 	}
 	return update, nil

pkg/sstore/dbops.go

@@ -207,7 +207,79 @@ func IsIncognitoScreen(ctx context.Context, sessionId string, screenId string) (
 	return rtn, txErr
 }
 
-func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts) ([]*HistoryItemType, error) {
+const HistoryQueryChunkSize = 1000
+
+func _getNextHistoryItem(items []*HistoryItemType, index int, filterFn func(*HistoryItemType) bool) (*HistoryItemType, int) {
+	for ; index < len(items); index++ {
+		item := items[index]
+		if filterFn(item) {
+			return item, index
+		}
+	}
+	return nil, index
+}
+
+func (result *HistoryQueryResult) processItem(item *HistoryItemType, rawOffset int) bool {
+	if len(result.Items) == result.MaxItems {
+		result.HasMore = true
+		result.NextRawOffset = rawOffset
+		return false
+	}
+	result.Items = append(result.Items, item)
+	return true
+}
+
+func runHistoryQueryWithFilter(tx *TxWrap, opts HistoryQueryOpts, filterFn func(*HistoryItemType) bool) (*HistoryQueryResult, error) {
+	if opts.MaxItems == 0 {
+		return nil, fmt.Errorf("invalid query, maxitems is 0")
+	}
+	if opts.RawOffset < opts.Offset {
+		return nil, fmt.Errorf("invalid query, rawoffset[%d] is less than offset[%d]", opts.RawOffset, opts.Offset)
+	}
+	rtn := &HistoryQueryResult{Offset: opts.RawOffset, MaxItems: opts.MaxItems}
+	if filterFn == nil {
+		results, err := runHistoryQuery(tx, opts, opts.RawOffset, opts.MaxItems+1)
+		if err != nil {
+			return nil, err
+		}
+		if len(results) > opts.MaxItems {
+			rtn.Items = results[0:opts.MaxItems]
+			rtn.HasMore = true
+			rtn.NextRawOffset = opts.RawOffset + opts.MaxItems
+		} else {
+			rtn.Items = results
+			rtn.HasMore = false
+			rtn.NextRawOffset = 0
+		}
+		return rtn, nil
+	}
+	rawOffset := opts.RawOffset
+	for {
+		resultItems, err := runHistoryQuery(tx, opts, rawOffset, HistoryQueryChunkSize)
+		if err != nil {
+			return nil, err
+		}
+		isDone := false
+		for resultIdx := 0; resultIdx < len(resultItems); resultIdx++ {
+			if !filterFn(resultItems[resultIdx]) {
+				continue
+			}
+			isDone = rtn.processItem(resultItems[resultIdx], rawOffset+resultIdx)
+			if isDone {
+				break
+			}
+		}
+		if isDone {
+			break
+		}
+		if len(resultItems) < HistoryQueryChunkSize {
+			break
+		}
+	}
+	return rtn, nil
+}
+
+func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts, realOffset int, itemLimit int) ([]*HistoryItemType, error) {
 	// check sessionid/windowid format because we are directly inserting them into the SQL
 	if opts.SessionId != "" {
 		_, err := uuid.Parse(opts.SessionId)
@@ -255,11 +327,7 @@ func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts) ([]*HistoryItemType, err
 	if opts.NoMeta {
 		whereClause += " AND NOT ismetacmd"
 	}
-	maxItems := opts.MaxItems
+	query := fmt.Sprintf("SELECT %s, '%s' || row_number() OVER win AS historynum FROM history %s WINDOW win AS (ORDER BY ts, historyid) ORDER BY ts DESC, historyid DESC LIMIT %d OFFSET %d", HistoryCols, hnumStr, whereClause, itemLimit, realOffset)
-	if maxItems == 0 {
-		maxItems = DefaultMaxHistoryItems
-	}
-	query := fmt.Sprintf("SELECT %s, '%s' || row_number() OVER win AS historynum FROM history %s WINDOW win AS (ORDER BY ts, historyid) ORDER BY ts DESC, historyid DESC LIMIT %d OFFSET %d", HistoryCols, hnumStr, whereClause, maxItems, opts.Offset)
 	marr := tx.SelectMaps(query, queryArgs...)
 	rtn := make([]*HistoryItemType, len(marr))
 	for idx, m := range marr {
@@ -269,11 +337,11 @@ func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts) ([]*HistoryItemType, err
 	return rtn, nil
 }
 
-func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) ([]*HistoryItemType, error) {
+func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) (*HistoryQueryResult, error) {
-	var rtn []*HistoryItemType
+	var rtn *HistoryQueryResult
 	txErr := WithTx(ctx, func(tx *TxWrap) error {
 		var err error
-		rtn, err = runHistoryQuery(tx, opts)
+		rtn, err = runHistoryQueryWithFilter(tx, opts, nil)
 		if err != nil {
 			return err
 		}

pkg/sstore/sstore.go

@@ -539,6 +539,15 @@ type HistoryQueryOpts struct {
 	RemoteId   string
 	WindowId   string
 	NoMeta     bool
+	RawOffset  int
+}
+
+type HistoryQueryResult struct {
+	MaxItems      int
+	Items         []*HistoryItemType
+	Offset        int // the offset shown to user
+	HasMore       bool
+	NextRawOffset int // internal offset used by pager for next query
 }
 
 type TermOpts struct {

pkg/sstore/updatebus.go

@@ -77,12 +77,12 @@ func InfoMsgUpdate(infoMsgFmt string, args ...interface{}) *ModelUpdate {
 }
 
 type HistoryViewData struct {
-	TotalCount int                `json:"totalcount"`
+	Items         []*HistoryItemType `json:"items"`
-	Offset     int                `json:"offset"`
+	Offset        int                `json:"offset"`
-	Items      []*HistoryItemType `json:"items"`
+	NextRawOffset int                `json:"rawoffset"`
-	Lines      []*LineType        `json:"lines"`
+	HasMore       bool               `json:"hasmore"`
-	Cmds       []*CmdType         `json:"cmds"`
+	Lines         []*LineType        `json:"lines"`
-	HasMore    bool               `json:"hasmore"`
+	Cmds          []*CmdType         `json:"cmds"`
 }
 
 type RemoteEditType struct {