From 501b067eadb501249fda5fbcdaf8e8513c7a8275 Mon Sep 17 00:00:00 2001 From: sawka Date: Mon, 6 Mar 2023 11:47:44 -0800 Subject: [PATCH] add a filtered search mode to history search --- pkg/cmdrunner/cmdrunner.go | 24 ++++++----- pkg/sstore/dbops.go | 86 ++++++++++++++++++++++++++++++++++---- pkg/sstore/sstore.go | 9 ++++ pkg/sstore/updatebus.go | 12 +++--- 4 files changed, 105 insertions(+), 26 deletions(-) diff --git a/pkg/cmdrunner/cmdrunner.go b/pkg/cmdrunner/cmdrunner.go index 6e7ae1f12..ae481b1c9 100644 --- a/pkg/cmdrunner/cmdrunner.go +++ b/pkg/cmdrunner/cmdrunner.go @@ -1858,7 +1858,11 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType if err != nil { return nil, err } - opts := sstore.HistoryQueryOpts{MaxItems: HistoryViewPageSize + 1, Offset: offset} + rawOffset, err := resolveNonNegInt(pk.Kwargs["rawoffset"], 0) + if err != nil { + return nil, err + } + opts := sstore.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset} if pk.Kwargs["text"] != "" { opts.SearchText = pk.Kwargs["text"] } @@ -1893,17 +1897,15 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType if err != nil { return nil, fmt.Errorf("invalid meta arg (must be boolean): %v", err) } - hitems, err := sstore.GetHistoryItems(ctx, opts) + hresult, err := sstore.GetHistoryItems(ctx, opts) if err != nil { return nil, err } - hvdata := &sstore.HistoryViewData{Offset: offset} - if len(hitems) > HistoryViewPageSize { - hvdata.HasMore = true - hvdata.Items = hitems[0:HistoryViewPageSize] - } else { - hvdata.HasMore = false - hvdata.Items = hitems + hvdata := &sstore.HistoryViewData{ + Items: hresult.Items, + Offset: hresult.Offset, + NextRawOffset: hresult.NextRawOffset, + HasMore: hresult.HasMore, } lines, cmds, err := sstore.GetLineCmdsFromHistoryItems(ctx, hvdata.Items) if err != nil { @@ -1951,7 +1953,7 @@ func HistoryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (ssto hWindowId = "" } hopts := sstore.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, WindowId: hWindowId} - hitems, err := sstore.GetHistoryItems(ctx, hopts) + hresult, err := sstore.GetHistoryItems(ctx, hopts) if err != nil { return nil, err } @@ -1967,7 +1969,7 @@ func HistoryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (ssto HistoryType: htype, SessionId: ids.SessionId, WindowId: ids.WindowId, - Items: hitems, + Items: hresult.Items, Show: show, } return update, nil diff --git a/pkg/sstore/dbops.go b/pkg/sstore/dbops.go index 0fa63c021..91bd0ecdd 100644 --- a/pkg/sstore/dbops.go +++ b/pkg/sstore/dbops.go @@ -207,7 +207,79 @@ func IsIncognitoScreen(ctx context.Context, sessionId string, screenId string) ( return rtn, txErr } -func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts) ([]*HistoryItemType, error) { +const HistoryQueryChunkSize = 1000 + +func _getNextHistoryItem(items []*HistoryItemType, index int, filterFn func(*HistoryItemType) bool) (*HistoryItemType, int) { + for ; index < len(items); index++ { + item := items[index] + if filterFn(item) { + return item, index + } + } + return nil, index +} + +func (result *HistoryQueryResult) processItem(item *HistoryItemType, rawOffset int) bool { + if len(result.Items) == result.MaxItems { + result.HasMore = true + result.NextRawOffset = rawOffset + return false + } + result.Items = append(result.Items, item) + return true +} + +func runHistoryQueryWithFilter(tx *TxWrap, opts HistoryQueryOpts, filterFn func(*HistoryItemType) bool) (*HistoryQueryResult, error) { + if opts.MaxItems == 0 { + return nil, fmt.Errorf("invalid query, maxitems is 0") + } + if opts.RawOffset < opts.Offset { + return nil, fmt.Errorf("invalid query, rawoffset[%d] is less than offset[%d]", opts.RawOffset, opts.Offset) + } + rtn := &HistoryQueryResult{Offset: opts.RawOffset, MaxItems: opts.MaxItems} + if filterFn == nil { + results, err := runHistoryQuery(tx, opts, opts.RawOffset, opts.MaxItems+1) + if err != nil { + return nil, err + } + if len(results) > opts.MaxItems { + rtn.Items = results[0:opts.MaxItems] + rtn.HasMore = true + rtn.NextRawOffset = opts.RawOffset + opts.MaxItems + } else { + rtn.Items = results + rtn.HasMore = false + rtn.NextRawOffset = 0 + } + return rtn, nil + } + rawOffset := opts.RawOffset + for { + resultItems, err := runHistoryQuery(tx, opts, rawOffset, HistoryQueryChunkSize) + if err != nil { + return nil, err + } + isDone := false + for resultIdx := 0; resultIdx < len(resultItems); resultIdx++ { + if !filterFn(resultItems[resultIdx]) { + continue + } + isDone = rtn.processItem(resultItems[resultIdx], rawOffset+resultIdx) + if isDone { + break + } + } + if isDone { + break + } + if len(resultItems) < HistoryQueryChunkSize { + break + } + } + return rtn, nil +} + +func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts, realOffset int, itemLimit int) ([]*HistoryItemType, error) { // check sessionid/windowid format because we are directly inserting them into the SQL if opts.SessionId != "" { _, err := uuid.Parse(opts.SessionId) @@ -255,11 +327,7 @@ func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts) ([]*HistoryItemType, err if opts.NoMeta { whereClause += " AND NOT ismetacmd" } - maxItems := opts.MaxItems - if maxItems == 0 { - maxItems = DefaultMaxHistoryItems - } - query := fmt.Sprintf("SELECT %s, '%s' || row_number() OVER win AS historynum FROM history %s WINDOW win AS (ORDER BY ts, historyid) ORDER BY ts DESC, historyid DESC LIMIT %d OFFSET %d", HistoryCols, hnumStr, whereClause, maxItems, opts.Offset) + query := fmt.Sprintf("SELECT %s, '%s' || row_number() OVER win AS historynum FROM history %s WINDOW win AS (ORDER BY ts, historyid) ORDER BY ts DESC, historyid DESC LIMIT %d OFFSET %d", HistoryCols, hnumStr, whereClause, itemLimit, realOffset) marr := tx.SelectMaps(query, queryArgs...) rtn := make([]*HistoryItemType, len(marr)) for idx, m := range marr { @@ -269,11 +337,11 @@ func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts) ([]*HistoryItemType, err return rtn, nil } -func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) ([]*HistoryItemType, error) { - var rtn []*HistoryItemType +func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) (*HistoryQueryResult, error) { + var rtn *HistoryQueryResult txErr := WithTx(ctx, func(tx *TxWrap) error { var err error - rtn, err = runHistoryQuery(tx, opts) + rtn, err = runHistoryQueryWithFilter(tx, opts, nil) if err != nil { return err } diff --git a/pkg/sstore/sstore.go b/pkg/sstore/sstore.go index a9816511d..93bb5d147 100644 --- a/pkg/sstore/sstore.go +++ b/pkg/sstore/sstore.go @@ -539,6 +539,15 @@ type HistoryQueryOpts struct { RemoteId string WindowId string NoMeta bool + RawOffset int +} + +type HistoryQueryResult struct { + MaxItems int + Items []*HistoryItemType + Offset int // the offset shown to user + HasMore bool + NextRawOffset int // internal offset used by pager for next query } type TermOpts struct { diff --git a/pkg/sstore/updatebus.go b/pkg/sstore/updatebus.go index cd7d7cf37..d01ba3cac 100644 --- a/pkg/sstore/updatebus.go +++ b/pkg/sstore/updatebus.go @@ -77,12 +77,12 @@ func InfoMsgUpdate(infoMsgFmt string, args ...interface{}) *ModelUpdate { } type HistoryViewData struct { - TotalCount int `json:"totalcount"` - Offset int `json:"offset"` - Items []*HistoryItemType `json:"items"` - Lines []*LineType `json:"lines"` - Cmds []*CmdType `json:"cmds"` - HasMore bool `json:"hasmore"` + Items []*HistoryItemType `json:"items"` + Offset int `json:"offset"` + NextRawOffset int `json:"rawoffset"` + HasMore bool `json:"hasmore"` + Lines []*LineType `json:"lines"` + Cmds []*CmdType `json:"cmds"` } type RemoteEditType struct {