diff --git a/wavesrv/cmd/main-server.go b/wavesrv/cmd/main-server.go index 96998170c..5abf25f2a 100644 --- a/wavesrv/cmd/main-server.go +++ b/wavesrv/cmd/main-server.go @@ -45,6 +45,7 @@ import ( "github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket" "github.com/wavetermdev/waveterm/wavesrv/pkg/scws" "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" + "github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry" "github.com/wavetermdev/waveterm/wavesrv/pkg/wsshell" ) @@ -211,7 +212,7 @@ func HandleLogActiveState(w http.ResponseWriter, r *http.Request) { WriteJsonError(w, fmt.Errorf(ErrorDecodingJson, err)) return } - activity := sstore.ActivityUpdate{} + activity := telemetry.ActivityUpdate{} if activeState.Fg { activity.FgMinutes = 1 } @@ -222,7 +223,7 @@ func HandleLogActiveState(w http.ResponseWriter, r *http.Request) { activity.OpenMinutes = 1 } activity.NumConns = remote.NumRemotes() - err = sstore.UpdateCurrentActivity(r.Context(), activity) + err = telemetry.UpdateCurrentActivity(r.Context(), activity) if err != nil { WriteJsonError(w, fmt.Errorf("error updating activity: %w", err)) return @@ -998,7 +999,7 @@ func main() { } log.Printf("PCLOUD_ENDPOINT=%s\n", pcloud.GetEndpoint()) - sstore.UpdateActivityWrap(context.Background(), sstore.ActivityUpdate{NumConns: remote.NumRemotes()}, "numconns") // set at least one record into activity + telemetry.UpdateActivityWrap(context.Background(), telemetry.ActivityUpdate{NumConns: remote.NumRemotes()}, "numconns") // set at least one record into activity installSignalHandlers() go telemetryLoop() go stdinReadWatch() diff --git a/wavesrv/pkg/bookmarks/bookmarks.go b/wavesrv/pkg/bookmarks/bookmarks.go new file mode 100644 index 000000000..e41c0e8f8 --- /dev/null +++ b/wavesrv/pkg/bookmarks/bookmarks.go @@ -0,0 +1,204 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package bookmarks + +import ( + "context" + "fmt" + + "github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil" + "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" +) + +type BookmarkType struct { + BookmarkId string `json:"bookmarkid"` + CreatedTs int64 `json:"createdts"` + CmdStr string `json:"cmdstr"` + Alias string `json:"alias,omitempty"` + Tags []string `json:"tags"` + Description string `json:"description"` + OrderIdx int64 `json:"orderidx"` + Remove bool `json:"remove,omitempty"` +} + +func (bm *BookmarkType) GetSimpleKey() string { + return bm.BookmarkId +} + +func (bm *BookmarkType) ToMap() map[string]interface{} { + rtn := make(map[string]interface{}) + rtn["bookmarkid"] = bm.BookmarkId + rtn["createdts"] = bm.CreatedTs + rtn["cmdstr"] = bm.CmdStr + rtn["alias"] = bm.Alias + rtn["description"] = bm.Description + rtn["tags"] = dbutil.QuickJsonArr(bm.Tags) + return rtn +} + +func (bm *BookmarkType) FromMap(m map[string]interface{}) bool { + dbutil.QuickSetStr(&bm.BookmarkId, m, "bookmarkid") + dbutil.QuickSetInt64(&bm.CreatedTs, m, "createdts") + dbutil.QuickSetStr(&bm.Alias, m, "alias") + dbutil.QuickSetStr(&bm.CmdStr, m, "cmdstr") + dbutil.QuickSetStr(&bm.Description, m, "description") + dbutil.QuickSetJsonArr(&bm.Tags, m, "tags") + return true +} + +type bookmarkOrderType struct { + BookmarkId string + OrderIdx int64 +} + +func GetBookmarks(ctx context.Context, tag string) ([]*BookmarkType, error) { + var bms []*BookmarkType + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + var query string + if tag == "" { + query = `SELECT * FROM bookmark` + bms = dbutil.SelectMapsGen[*BookmarkType](tx, query) + } else { + query = `SELECT * FROM bookmark WHERE EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)` + bms = dbutil.SelectMapsGen[*BookmarkType](tx, query, tag) + } + bmMap := dbutil.MakeGenMap(bms) + var orders []bookmarkOrderType + query = `SELECT bookmarkid, orderidx FROM bookmark_order WHERE tag = ?` + tx.Select(&orders, query, tag) + for _, bmOrder := range orders { + bm := bmMap[bmOrder.BookmarkId] + if bm != nil { + bm.OrderIdx = bmOrder.OrderIdx + } + } + return nil + }) + if txErr != nil { + return nil, txErr + } + return bms, nil +} + +func GetBookmarkById(ctx context.Context, bookmarkId string, tag string) (*BookmarkType, error) { + var rtn *BookmarkType + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `SELECT * FROM bookmark WHERE bookmarkid = ?` + rtn = dbutil.GetMapGen[*BookmarkType](tx, query, bookmarkId) + if rtn == nil { + return nil + } + query = `SELECT orderidx FROM bookmark_order WHERE bookmarkid = ? AND tag = ?` + orderIdx := tx.GetInt(query, bookmarkId, tag) + rtn.OrderIdx = int64(orderIdx) + return nil + }) + if txErr != nil { + return nil, txErr + } + return rtn, nil +} + +func GetBookmarkIdByArg(ctx context.Context, bookmarkArg string) (string, error) { + var rtnId string + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + if len(bookmarkArg) == 8 { + query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid LIKE (? || '%')` + rtnId = tx.GetString(query, bookmarkArg) + return nil + } + query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` + rtnId = tx.GetString(query, bookmarkArg) + return nil + }) + if txErr != nil { + return "", txErr + } + return rtnId, nil +} + +func GetBookmarkIdsByCmdStr(ctx context.Context, cmdStr string) ([]string, error) { + return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) ([]string, error) { + query := `SELECT bookmarkid FROM bookmark WHERE cmdstr = ?` + bmIds := tx.SelectStrings(query, cmdStr) + return bmIds, nil + }) +} + +// ignores OrderIdx field +func InsertBookmark(ctx context.Context, bm *BookmarkType) error { + if bm == nil || bm.BookmarkId == "" { + return fmt.Errorf("invalid empty bookmark id") + } + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` + if tx.Exists(query, bm.BookmarkId) { + return fmt.Errorf("bookmarkid already exists") + } + query = `INSERT INTO bookmark ( bookmarkid, createdts, cmdstr, alias, tags, description) + VALUES (:bookmarkid,:createdts,:cmdstr,:alias,:tags,:description)` + tx.NamedExec(query, bm.ToMap()) + for _, tag := range append(bm.Tags, "") { + query = `SELECT COALESCE(max(orderidx), 0) FROM bookmark_order WHERE tag = ?` + maxOrder := tx.GetInt(query, tag) + query = `INSERT INTO bookmark_order (tag, bookmarkid, orderidx) VALUES (?, ?, ?)` + tx.Exec(query, tag, bm.BookmarkId, maxOrder+1) + } + return nil + }) + return txErr +} + +const ( + BookmarkField_Desc = "desc" + BookmarkField_CmdStr = "cmdstr" +) + +func EditBookmark(ctx context.Context, bookmarkId string, editMap map[string]interface{}) error { + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` + if !tx.Exists(query, bookmarkId) { + return fmt.Errorf("bookmark not found") + } + if desc, found := editMap[BookmarkField_Desc]; found { + query = `UPDATE bookmark SET description = ? WHERE bookmarkid = ?` + tx.Exec(query, desc, bookmarkId) + } + if cmdStr, found := editMap[BookmarkField_CmdStr]; found { + query = `UPDATE bookmark SET cmdstr = ? WHERE bookmarkid = ?` + tx.Exec(query, cmdStr, bookmarkId) + } + return nil + }) + return txErr +} + +func fixupBookmarkOrder(tx *sstore.TxWrap) { + query := ` +WITH new_order AS ( + SELECT tag, bookmarkid, row_number() OVER (PARTITION BY tag ORDER BY orderidx) AS newidx FROM bookmark_order +) +UPDATE bookmark_order +SET orderidx = new_order.newidx +FROM new_order +WHERE bookmark_order.tag = new_order.tag AND bookmark_order.bookmarkid = new_order.bookmarkid +` + tx.Exec(query) +} + +func DeleteBookmark(ctx context.Context, bookmarkId string) error { + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` + if !tx.Exists(query, bookmarkId) { + return fmt.Errorf("bookmark not found") + } + query = `DELETE FROM bookmark WHERE bookmarkid = ?` + tx.Exec(query, bookmarkId) + query = `DELETE FROM bookmark_order WHERE bookmarkid = ?` + tx.Exec(query, bookmarkId) + fixupBookmarkOrder(tx) + return nil + }) + return txErr +} diff --git a/wavesrv/pkg/bookmarks/updatetypes.go b/wavesrv/pkg/bookmarks/updatetypes.go new file mode 100644 index 000000000..a0cf7b7c3 --- /dev/null +++ b/wavesrv/pkg/bookmarks/updatetypes.go @@ -0,0 +1,23 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package bookmarks + +import "github.com/wavetermdev/waveterm/wavesrv/pkg/scbus" + +type BookmarksUpdate struct { + Bookmarks []*BookmarkType `json:"bookmarks"` + SelectedBookmark string `json:"selectedbookmark,omitempty"` +} + +func (BookmarksUpdate) GetType() string { + return "bookmarks" +} + +func AddBookmarksUpdate(update *scbus.ModelUpdatePacketType, bookmarks []*BookmarkType, selectedBookmark *string) { + if selectedBookmark == nil { + update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks}) + } else { + update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks, SelectedBookmark: *selectedBookmark}) + } +} diff --git a/wavesrv/pkg/cmdrunner/cmdrunner.go b/wavesrv/pkg/cmdrunner/cmdrunner.go index 5709c9061..5d8f410e6 100644 --- a/wavesrv/pkg/cmdrunner/cmdrunner.go +++ b/wavesrv/pkg/cmdrunner/cmdrunner.go @@ -35,8 +35,10 @@ import ( "github.com/wavetermdev/waveterm/waveshell/pkg/shellutil" "github.com/wavetermdev/waveterm/waveshell/pkg/shexec" "github.com/wavetermdev/waveterm/waveshell/pkg/utilfn" + "github.com/wavetermdev/waveterm/wavesrv/pkg/bookmarks" "github.com/wavetermdev/waveterm/wavesrv/pkg/comp" "github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil" + "github.com/wavetermdev/waveterm/wavesrv/pkg/history" "github.com/wavetermdev/waveterm/wavesrv/pkg/pcloud" "github.com/wavetermdev/waveterm/wavesrv/pkg/promptenc" "github.com/wavetermdev/waveterm/wavesrv/pkg/releasechecker" @@ -47,6 +49,7 @@ import ( "github.com/wavetermdev/waveterm/wavesrv/pkg/scbus" "github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket" "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" + "github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry" "golang.org/x/mod/semver" ) @@ -473,7 +476,7 @@ func doHistoryExpansion(ctx context.Context, ids resolvedIds, hnum int) (string, foundHistoryNum := hnum if hnum == -1 { var err error - foundHistoryNum, err = sstore.GetLastHistoryLineNum(ctx, ids.ScreenId) + foundHistoryNum, err = history.GetLastHistoryLineNum(ctx, ids.ScreenId) if err != nil { return "", fmt.Errorf("cannot expand history, error finding last history item: %v", err) } @@ -481,7 +484,7 @@ func doHistoryExpansion(ctx context.Context, ids resolvedIds, hnum int) (string, return "", fmt.Errorf("cannot expand history, no last history item") } } - hitem, err := sstore.GetHistoryItemByLineNum(ctx, ids.ScreenId, foundHistoryNum) + hitem, err := history.GetHistoryItemByLineNum(ctx, ids.ScreenId, foundHistoryNum) if err != nil { return "", fmt.Errorf("cannot get history item '%d': %v", foundHistoryNum, err) } @@ -666,7 +669,7 @@ func addToHistory(ctx context.Context, pk *scpacket.FeCommandPacketType, history if err != nil { return err } - hitem := &sstore.HistoryItemType{ + hitem := &history.HistoryItemType{ HistoryId: scbase.GenWaveUUID(), Ts: time.Now().UnixMilli(), UserId: DefaultUserId, @@ -690,7 +693,7 @@ func addToHistory(ctx context.Context, pk *scpacket.FeCommandPacketType, history if !isMetaCmd && historyContext.RemotePtr != nil { hitem.Remote = *historyContext.RemotePtr } - err = sstore.InsertHistoryItem(ctx, hitem) + err = history.InsertHistoryItem(ctx, hitem) if err != nil { return err } @@ -706,7 +709,7 @@ func EvalCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbus.U } evalDepth := getEvalDepth(ctx) if pk.Interactive && evalDepth == 0 { - sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{NumCommands: 1}, "numcommands") + telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{NumCommands: 1}, "numcommands") } if evalDepth > MaxEvalDepth { return nil, fmt.Errorf("alias/history expansion max-depth exceeded") @@ -3352,8 +3355,8 @@ func validateRemoteColor(color string, typeStr string) error { } func SessionOpenSharedCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbus.UpdatePacket, error) { - activity := sstore.ActivityUpdate{ClickShared: 1} - sstore.UpdateActivityWrap(ctx, activity, "click-shared") + activity := telemetry.ActivityUpdate{ClickShared: 1} + telemetry.UpdateActivityWrap(ctx, activity, "click-shared") return nil, fmt.Errorf("shared sessions are not available in this version of prompt (stay tuned)") } @@ -3615,11 +3618,11 @@ func MainViewCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scb update := scbus.MakeUpdatePacket() mainViewArg := pk.Args[0] if mainViewArg == sstore.MainViewSession { - update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewSession}) + update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewSession}) } else if mainViewArg == sstore.MainViewConnections { - update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewConnections}) + update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewConnections}) } else if mainViewArg == sstore.MainViewSettings { - update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewSettings}) + update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewSettings}) } else if mainViewArg == sstore.MainViewHistory { return nil, fmt.Errorf("use /history instead") } else if mainViewArg == sstore.MainViewBookmarks { @@ -3825,7 +3828,7 @@ func HistoryPurgeCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) } historyIds = append(historyIds, historyArg) } - err := sstore.PurgeHistoryByIds(ctx, historyIds) + err := history.PurgeHistoryByIds(ctx, historyIds) if err != nil { return nil, fmt.Errorf("/history:purge error purging items: %v", err) } @@ -3837,7 +3840,7 @@ const HistoryViewPageSize = 50 var cmdFilterLs = regexp.MustCompile(`^ls(\s|$)`) var cmdFilterCd = regexp.MustCompile(`^cd(\s|$)`) -func historyCmdFilter(hitem *sstore.HistoryItemType) bool { +func historyCmdFilter(hitem *history.HistoryItemType) bool { cmdStr := hitem.CmdStr if cmdStr == "" || strings.Index(cmdStr, ";") != -1 || strings.Index(cmdStr, "\n") != -1 { return true @@ -3864,7 +3867,7 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType if err != nil { return nil, err } - opts := sstore.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset} + opts := history.HistoryQueryOpts{MaxItems: HistoryViewPageSize, Offset: offset, RawOffset: rawOffset} if pk.Kwargs["text"] != "" { opts.SearchText = pk.Kwargs["text"] } @@ -3903,25 +3906,25 @@ func HistoryViewAllCommand(ctx context.Context, pk *scpacket.FeCommandPacketType if err != nil { return nil, fmt.Errorf("invalid meta arg (must be boolean): %v", err) } - hresult, err := sstore.GetHistoryItems(ctx, opts) + hresult, err := history.GetHistoryItems(ctx, opts) if err != nil { return nil, err } - hvdata := &sstore.HistoryViewData{ + hvdata := &history.HistoryViewData{ Items: hresult.Items, Offset: hresult.Offset, RawOffset: hresult.RawOffset, NextRawOffset: hresult.NextRawOffset, HasMore: hresult.HasMore, } - lines, cmds, err := sstore.GetLineCmdsFromHistoryItems(ctx, hvdata.Items) + lines, cmds, err := history.GetLineCmdsFromHistoryItems(ctx, hvdata.Items) if err != nil { return nil, err } hvdata.Lines = lines hvdata.Cmds = cmds update := scbus.MakeUpdatePacket() - update.AddUpdate(&sstore.MainViewUpdate{MainView: sstore.MainViewHistory, HistoryView: hvdata}) + update.AddUpdate(&MainViewUpdate{MainView: sstore.MainViewHistory, HistoryView: hvdata}) return update, nil } @@ -3957,17 +3960,17 @@ func HistoryCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) (scbu } else if htype == HistoryTypeSession { hScreenId = "" } - hopts := sstore.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, ScreenId: hScreenId} - hresult, err := sstore.GetHistoryItems(ctx, hopts) + hopts := history.HistoryQueryOpts{MaxItems: maxItems, SessionId: hSessionId, ScreenId: hScreenId} + hresult, err := history.GetHistoryItems(ctx, hopts) if err != nil { return nil, err } show := !resolveBool(pk.Kwargs["noshow"], false) if show { - sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{HistoryView: 1}, "history") + telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{HistoryView: 1}, "history") } update := scbus.MakeUpdatePacket() - update.AddUpdate(sstore.HistoryInfoType{ + update.AddUpdate(history.HistoryInfoType{ HistoryType: htype, SessionId: ids.SessionId, ScreenId: ids.ScreenId, @@ -4310,16 +4313,16 @@ func BookmarksShowCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) if len(pk.Args) > 0 { tagName = pk.Args[0] } - bms, err := sstore.GetBookmarks(ctx, tagName) + bms, err := bookmarks.GetBookmarks(ctx, tagName) if err != nil { return nil, fmt.Errorf("cannot retrieve bookmarks: %v", err) } - sstore.UpdateActivityWrap(ctx, sstore.ActivityUpdate{BookmarksView: 1}, "bookmarks") + telemetry.UpdateActivityWrap(ctx, telemetry.ActivityUpdate{BookmarksView: 1}, "bookmarks") update := scbus.MakeUpdatePacket() - update.AddUpdate(&sstore.MainViewUpdate{ + update.AddUpdate(&MainViewUpdate{ MainView: sstore.MainViewBookmarks, - BookmarksView: &sstore.BookmarksUpdate{Bookmarks: bms}, + BookmarksView: &bookmarks.BookmarksUpdate{Bookmarks: bms}, }) return update, nil } @@ -4329,7 +4332,7 @@ func BookmarkSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) ( return nil, fmt.Errorf("/bookmark:set requires one argument (bookmark id)") } bookmarkArg := pk.Args[0] - bookmarkId, err := sstore.GetBookmarkIdByArg(ctx, bookmarkArg) + bookmarkId, err := bookmarks.GetBookmarkIdByArg(ctx, bookmarkArg) if err != nil { return nil, fmt.Errorf("error trying to resolve bookmark: %v", err) } @@ -4338,25 +4341,25 @@ func BookmarkSetCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) ( } editMap := make(map[string]interface{}) if descStr, found := pk.Kwargs["desc"]; found { - editMap[sstore.BookmarkField_Desc] = descStr + editMap[bookmarks.BookmarkField_Desc] = descStr } if cmdStr, found := pk.Kwargs["cmdstr"]; found { - editMap[sstore.BookmarkField_CmdStr] = cmdStr + editMap[bookmarks.BookmarkField_CmdStr] = cmdStr } if len(editMap) == 0 { return nil, fmt.Errorf("no fields set, can set %s", formatStrs([]string{"desc", "cmdstr"}, "or", false)) } - err = sstore.EditBookmark(ctx, bookmarkId, editMap) + err = bookmarks.EditBookmark(ctx, bookmarkId, editMap) if err != nil { return nil, fmt.Errorf("error trying to edit bookmark: %v", err) } - bm, err := sstore.GetBookmarkById(ctx, bookmarkId, "") + bm, err := bookmarks.GetBookmarkById(ctx, bookmarkId, "") if err != nil { return nil, fmt.Errorf("error retrieving edited bookmark: %v", err) } - bms := []*sstore.BookmarkType{bm} + bms := []*bookmarks.BookmarkType{bm} update := scbus.MakeUpdatePacket() - sstore.AddBookmarksUpdate(update, bms, nil) + bookmarks.AddBookmarksUpdate(update, bms, nil) update.AddUpdate(sstore.InfoMsgUpdate("bookmark edited")) return update, nil } @@ -4366,20 +4369,20 @@ func BookmarkDeleteCommand(ctx context.Context, pk *scpacket.FeCommandPacketType return nil, fmt.Errorf("/bookmark:delete requires one argument (bookmark id)") } bookmarkArg := pk.Args[0] - bookmarkId, err := sstore.GetBookmarkIdByArg(ctx, bookmarkArg) + bookmarkId, err := bookmarks.GetBookmarkIdByArg(ctx, bookmarkArg) if err != nil { return nil, fmt.Errorf("error trying to resolve bookmark: %v", err) } if bookmarkId == "" { return nil, fmt.Errorf("bookmark not found") } - err = sstore.DeleteBookmark(ctx, bookmarkId) + err = bookmarks.DeleteBookmark(ctx, bookmarkId) if err != nil { return nil, fmt.Errorf("error deleting bookmark: %v", err) } update := scbus.MakeUpdatePacket() - bms := []*sstore.BookmarkType{{BookmarkId: bookmarkId, Remove: true}} - sstore.AddBookmarksUpdate(update, bms, nil) + bms := []*bookmarks.BookmarkType{{BookmarkId: bookmarkId, Remove: true}} + bookmarks.AddBookmarksUpdate(update, bms, nil) update.AddUpdate(sstore.InfoMsgUpdate("bookmark deleted")) return update, nil } @@ -4407,7 +4410,7 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) if cmdObj == nil { return nil, fmt.Errorf("cannot bookmark non-cmd line") } - existingBmIds, err := sstore.GetBookmarkIdsByCmdStr(ctx, cmdObj.CmdStr) + existingBmIds, err := bookmarks.GetBookmarkIdsByCmdStr(ctx, cmdObj.CmdStr) if err != nil { return nil, fmt.Errorf("error trying to retrieve current boookmarks: %v", err) } @@ -4415,7 +4418,7 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) if len(existingBmIds) > 0 { newBmId = existingBmIds[0] } else { - newBm := &sstore.BookmarkType{ + newBm := &bookmarks.BookmarkType{ BookmarkId: uuid.New().String(), CreatedTs: time.Now().UnixMilli(), CmdStr: cmdObj.CmdStr, @@ -4423,17 +4426,17 @@ func LineBookmarkCommand(ctx context.Context, pk *scpacket.FeCommandPacketType) Tags: nil, Description: "", } - err = sstore.InsertBookmark(ctx, newBm) + err = bookmarks.InsertBookmark(ctx, newBm) if err != nil { return nil, fmt.Errorf("cannot insert bookmark: %v", err) } newBmId = newBm.BookmarkId } - bms, err := sstore.GetBookmarks(ctx, "") + bms, err := bookmarks.GetBookmarks(ctx, "") update := scbus.MakeUpdatePacket() - update.AddUpdate(&sstore.MainViewUpdate{ + update.AddUpdate(&MainViewUpdate{ MainView: sstore.MainViewBookmarks, - BookmarksView: &sstore.BookmarksUpdate{Bookmarks: bms, SelectedBookmark: newBmId}, + BookmarksView: &bookmarks.BookmarksUpdate{Bookmarks: bms, SelectedBookmark: newBmId}, }) return update, nil } diff --git a/wavesrv/pkg/cmdrunner/updatetypes.go b/wavesrv/pkg/cmdrunner/updatetypes.go new file mode 100644 index 000000000..16e15c32c --- /dev/null +++ b/wavesrv/pkg/cmdrunner/updatetypes.go @@ -0,0 +1,19 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package cmdrunner + +import ( + "github.com/wavetermdev/waveterm/wavesrv/pkg/bookmarks" + "github.com/wavetermdev/waveterm/wavesrv/pkg/history" +) + +type MainViewUpdate struct { + MainView string `json:"mainview"` + HistoryView *history.HistoryViewData `json:"historyview,omitempty"` + BookmarksView *bookmarks.BookmarksUpdate `json:"bookmarksview,omitempty"` +} + +func (MainViewUpdate) GetType() string { + return "mainview" +} diff --git a/wavesrv/pkg/history/history.go b/wavesrv/pkg/history/history.go new file mode 100644 index 000000000..b38994f23 --- /dev/null +++ b/wavesrv/pkg/history/history.go @@ -0,0 +1,326 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package history + +import ( + "context" + "fmt" + "strings" + + "github.com/google/uuid" + "github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil" + "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" +) + +type HistoryItemType struct { + HistoryId string `json:"historyid"` + Ts int64 `json:"ts"` + UserId string `json:"userid"` + SessionId string `json:"sessionid"` + ScreenId string `json:"screenid"` + LineId string `json:"lineid"` + HadError bool `json:"haderror"` + CmdStr string `json:"cmdstr"` + Remote sstore.RemotePtrType `json:"remote"` + IsMetaCmd bool `json:"ismetacmd"` + ExitCode *int64 `json:"exitcode,omitempty"` + DurationMs *int64 `json:"durationms,omitempty"` + FeState sstore.FeStateType `json:"festate,omitempty"` + Tags map[string]bool `json:"tags,omitempty"` + LineNum int64 `json:"linenum" dbmap:"-"` + Status string `json:"status"` + + // only for updates + Remove bool `json:"remove" dbmap:"-"` + + // transient (string because of different history orderings) + HistoryNum string `json:"historynum" dbmap:"-"` +} + +func (h *HistoryItemType) ToMap() map[string]interface{} { + rtn := make(map[string]interface{}) + rtn["historyid"] = h.HistoryId + rtn["ts"] = h.Ts + rtn["userid"] = h.UserId + rtn["sessionid"] = h.SessionId + rtn["screenid"] = h.ScreenId + rtn["lineid"] = h.LineId + rtn["linenum"] = h.LineNum + rtn["haderror"] = h.HadError + rtn["cmdstr"] = h.CmdStr + rtn["remoteownerid"] = h.Remote.OwnerId + rtn["remoteid"] = h.Remote.RemoteId + rtn["remotename"] = h.Remote.Name + rtn["ismetacmd"] = h.IsMetaCmd + rtn["exitcode"] = h.ExitCode + rtn["durationms"] = h.DurationMs + rtn["festate"] = dbutil.QuickJson(h.FeState) + rtn["tags"] = dbutil.QuickJson(h.Tags) + rtn["status"] = h.Status + return rtn +} + +func (h *HistoryItemType) FromMap(m map[string]interface{}) bool { + dbutil.QuickSetStr(&h.HistoryId, m, "historyid") + dbutil.QuickSetInt64(&h.Ts, m, "ts") + dbutil.QuickSetStr(&h.UserId, m, "userid") + dbutil.QuickSetStr(&h.SessionId, m, "sessionid") + dbutil.QuickSetStr(&h.ScreenId, m, "screenid") + dbutil.QuickSetStr(&h.LineId, m, "lineid") + dbutil.QuickSetBool(&h.HadError, m, "haderror") + dbutil.QuickSetStr(&h.CmdStr, m, "cmdstr") + dbutil.QuickSetStr(&h.Remote.OwnerId, m, "remoteownerid") + dbutil.QuickSetStr(&h.Remote.RemoteId, m, "remoteid") + dbutil.QuickSetStr(&h.Remote.Name, m, "remotename") + dbutil.QuickSetBool(&h.IsMetaCmd, m, "ismetacmd") + dbutil.QuickSetStr(&h.HistoryNum, m, "historynum") + dbutil.QuickSetInt64(&h.LineNum, m, "linenum") + dbutil.QuickSetNullableInt64(&h.ExitCode, m, "exitcode") + dbutil.QuickSetNullableInt64(&h.DurationMs, m, "durationms") + dbutil.QuickSetJson(&h.FeState, m, "festate") + dbutil.QuickSetJson(&h.Tags, m, "tags") + dbutil.QuickSetStr(&h.Status, m, "status") + return true +} + +type HistoryQueryOpts struct { + Offset int + MaxItems int + FromTs int64 + SearchText string + SessionId string + RemoteId string + ScreenId string + NoMeta bool + RawOffset int + FilterFn func(*HistoryItemType) bool +} + +type HistoryQueryResult struct { + MaxItems int + Items []*HistoryItemType + Offset int // the offset shown to user + RawOffset int // internal offset + HasMore bool + NextRawOffset int // internal offset used by pager for next query + + prevItems int // holds number of items skipped by RawOffset +} + +type HistoryViewData struct { + Items []*HistoryItemType `json:"items"` + Offset int `json:"offset"` + RawOffset int `json:"rawoffset"` + NextRawOffset int `json:"nextrawoffset"` + HasMore bool `json:"hasmore"` + Lines []*sstore.LineType `json:"lines"` + Cmds []*sstore.CmdType `json:"cmds"` +} + +const HistoryCols = "h.historyid, h.ts, h.userid, h.sessionid, h.screenid, h.lineid, h.haderror, h.cmdstr, h.remoteownerid, h.remoteid, h.remotename, h.ismetacmd, h.linenum, h.exitcode, h.durationms, h.festate, h.tags, h.status" +const DefaultMaxHistoryItems = 1000 + +func InsertHistoryItem(ctx context.Context, hitem *HistoryItemType) error { + if hitem == nil { + return fmt.Errorf("cannot insert nil history item") + } + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `INSERT INTO history + ( historyid, ts, userid, sessionid, screenid, lineid, haderror, cmdstr, remoteownerid, remoteid, remotename, ismetacmd, linenum, exitcode, durationms, festate, tags, status) VALUES + (:historyid,:ts,:userid,:sessionid,:screenid,:lineid,:haderror,:cmdstr,:remoteownerid,:remoteid,:remotename,:ismetacmd,:linenum,:exitcode,:durationms,:festate,:tags,:status)` + tx.NamedExec(query, hitem.ToMap()) + return nil + }) + return txErr +} + +const HistoryQueryChunkSize = 1000 + +func _getNextHistoryItem(items []*HistoryItemType, index int, filterFn func(*HistoryItemType) bool) (*HistoryItemType, int) { + for ; index < len(items); index++ { + item := items[index] + if filterFn(item) { + return item, index + } + } + return nil, index +} + +// returns true if done, false if we still need to process more items +func (result *HistoryQueryResult) processItem(item *HistoryItemType, rawOffset int) bool { + if result.prevItems < result.Offset { + result.prevItems++ + return false + } + if len(result.Items) == result.MaxItems { + result.HasMore = true + result.NextRawOffset = rawOffset + return true + } + if len(result.Items) == 0 { + result.RawOffset = rawOffset + } + result.Items = append(result.Items, item) + return false +} + +func runHistoryQueryWithFilter(tx *sstore.TxWrap, opts HistoryQueryOpts) (*HistoryQueryResult, error) { + if opts.MaxItems == 0 { + return nil, fmt.Errorf("invalid query, maxitems is 0") + } + rtn := &HistoryQueryResult{Offset: opts.Offset, MaxItems: opts.MaxItems} + var rawOffset int + if opts.RawOffset >= opts.Offset { + rtn.prevItems = opts.Offset + rawOffset = opts.RawOffset + } else { + rawOffset = 0 + } + for { + resultItems, err := runHistoryQuery(tx, opts, rawOffset, HistoryQueryChunkSize) + if err != nil { + return nil, err + } + isDone := false + for resultIdx := 0; resultIdx < len(resultItems); resultIdx++ { + if opts.FilterFn != nil && !opts.FilterFn(resultItems[resultIdx]) { + continue + } + isDone = rtn.processItem(resultItems[resultIdx], rawOffset+resultIdx) + if isDone { + break + } + } + if isDone { + break + } + if len(resultItems) < HistoryQueryChunkSize { + break + } + rawOffset += HistoryQueryChunkSize + } + return rtn, nil +} + +func runHistoryQuery(tx *sstore.TxWrap, opts HistoryQueryOpts, realOffset int, itemLimit int) ([]*HistoryItemType, error) { + // check sessionid/screenid format because we are directly inserting them into the SQL + if opts.SessionId != "" { + _, err := uuid.Parse(opts.SessionId) + if err != nil { + return nil, fmt.Errorf("malformed sessionid") + } + } + if opts.ScreenId != "" { + _, err := uuid.Parse(opts.ScreenId) + if err != nil { + return nil, fmt.Errorf("malformed screenid") + } + } + if opts.RemoteId != "" { + _, err := uuid.Parse(opts.RemoteId) + if err != nil { + return nil, fmt.Errorf("malformed remoteid") + } + } + whereClause := "WHERE 1" + var queryArgs []interface{} + hNumStr := "" + if opts.SessionId != "" && opts.ScreenId != "" { + whereClause += fmt.Sprintf(" AND h.sessionid = '%s' AND h.screenid = '%s'", opts.SessionId, opts.ScreenId) + hNumStr = "" + } else if opts.SessionId != "" { + whereClause += fmt.Sprintf(" AND h.sessionid = '%s'", opts.SessionId) + hNumStr = "s" + } else { + hNumStr = "g" + } + if opts.SearchText != "" { + whereClause += " AND h.cmdstr LIKE ? ESCAPE '\\'" + likeArg := opts.SearchText + likeArg = strings.ReplaceAll(likeArg, "%", "\\%") + likeArg = strings.ReplaceAll(likeArg, "_", "\\_") + queryArgs = append(queryArgs, "%"+likeArg+"%") + } + if opts.FromTs > 0 { + whereClause += fmt.Sprintf(" AND h.ts <= %d", opts.FromTs) + } + if opts.RemoteId != "" { + whereClause += fmt.Sprintf(" AND h.remoteid = '%s'", opts.RemoteId) + } + if opts.NoMeta { + whereClause += " AND NOT h.ismetacmd" + } + query := fmt.Sprintf("SELECT %s, ('%s' || CAST((row_number() OVER win) as text)) historynum FROM history h %s WINDOW win AS (ORDER BY h.ts, h.historyid) ORDER BY h.ts DESC, h.historyid DESC LIMIT %d OFFSET %d", HistoryCols, hNumStr, whereClause, itemLimit, realOffset) + marr := tx.SelectMaps(query, queryArgs...) + rtn := make([]*HistoryItemType, len(marr)) + for idx, m := range marr { + hitem := dbutil.FromMap[*HistoryItemType](m) + rtn[idx] = hitem + } + return rtn, nil +} + +func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) (*HistoryQueryResult, error) { + var rtn *HistoryQueryResult + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + var err error + rtn, err = runHistoryQueryWithFilter(tx, opts) + if err != nil { + return err + } + return nil + }) + if txErr != nil { + return nil, txErr + } + return rtn, nil +} + +func GetHistoryItemByLineNum(ctx context.Context, screenId string, lineNum int) (*HistoryItemType, error) { + return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*HistoryItemType, error) { + query := `SELECT * FROM history WHERE screenid = ? AND linenum = ?` + hitem := dbutil.GetMapGen[*HistoryItemType](tx, query, screenId, lineNum) + return hitem, nil + }) +} + +func GetLastHistoryLineNum(ctx context.Context, screenId string) (int, error) { + return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (int, error) { + query := `SELECT COALESCE(max(linenum), 0) FROM history WHERE screenid = ?` + maxLineNum := tx.GetInt(query, screenId) + return maxLineNum, nil + }) +} + +func getLineIdsFromHistoryItems(historyItems []*HistoryItemType) []string { + var rtn []string + for _, hitem := range historyItems { + if hitem.LineId != "" { + rtn = append(rtn, hitem.LineId) + } + } + return rtn +} + +func GetLineCmdsFromHistoryItems(ctx context.Context, historyItems []*HistoryItemType) ([]*sstore.LineType, []*sstore.CmdType, error) { + if len(historyItems) == 0 { + return nil, nil, nil + } + return sstore.WithTxRtn3(ctx, func(tx *sstore.TxWrap) ([]*sstore.LineType, []*sstore.CmdType, error) { + lineIdsJsonArr := dbutil.QuickJsonArr(getLineIdsFromHistoryItems(historyItems)) + query := `SELECT * FROM line WHERE lineid IN (SELECT value FROM json_each(?))` + lineArr := dbutil.SelectMappable[*sstore.LineType](tx, query, lineIdsJsonArr) + query = `SELECT * FROM cmd WHERE lineid IN (SELECT value FROM json_each(?))` + cmdArr := dbutil.SelectMapsGen[*sstore.CmdType](tx, query, lineIdsJsonArr) + return lineArr, cmdArr, nil + }) +} + +func PurgeHistoryByIds(ctx context.Context, historyIds []string) error { + return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `DELETE FROM history WHERE historyid IN (SELECT value FROM json_each(?))` + tx.Exec(query, dbutil.QuickJsonArr(historyIds)) + return nil + }) +} diff --git a/wavesrv/pkg/history/updatetypes.go b/wavesrv/pkg/history/updatetypes.go new file mode 100644 index 000000000..264afa841 --- /dev/null +++ b/wavesrv/pkg/history/updatetypes.go @@ -0,0 +1,16 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package history + +type HistoryInfoType struct { + HistoryType string `json:"historytype"` + SessionId string `json:"sessionid,omitempty"` + ScreenId string `json:"screenid,omitempty"` + Items []*HistoryItemType `json:"items"` + Show bool `json:"show"` +} + +func (HistoryInfoType) GetType() string { + return "history" +} diff --git a/wavesrv/pkg/pcloud/pcloud.go b/wavesrv/pkg/pcloud/pcloud.go index b7fc9edd0..f599ed108 100644 --- a/wavesrv/pkg/pcloud/pcloud.go +++ b/wavesrv/pkg/pcloud/pcloud.go @@ -23,6 +23,7 @@ import ( "github.com/wavetermdev/waveterm/wavesrv/pkg/rtnstate" "github.com/wavetermdev/waveterm/wavesrv/pkg/scbase" "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" + "github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry" ) const PCloudEndpoint = "https://api.waveterm.dev/central" @@ -156,7 +157,7 @@ func SendTelemetry(ctx context.Context, force bool) error { if !force && clientData.ClientOpts.NoTelemetry { return nil } - activity, err := sstore.GetNonUploadedActivity(ctx) + activity, err := telemetry.GetNonUploadedActivity(ctx) if err != nil { return fmt.Errorf("cannot get activity: %v", err) } @@ -164,7 +165,7 @@ func SendTelemetry(ctx context.Context, force bool) error { return nil } log.Printf("[pcloud] sending telemetry data\n") - dayStr := sstore.GetCurDayStr() + dayStr := telemetry.GetCurDayStr() defaultShellType := shellapi.DetectLocalShellType() input := TelemetryInputType{UserId: clientData.UserId, ClientId: clientData.ClientId, CurDay: dayStr, DefaultShell: defaultShellType, Activity: activity} req, err := makeAnonPostReq(ctx, TelemetryUrl, input) @@ -175,7 +176,7 @@ func SendTelemetry(ctx context.Context, force bool) error { if err != nil { return err } - err = sstore.MarkActivityAsUploaded(ctx, activity) + err = telemetry.MarkActivityAsUploaded(ctx, activity) if err != nil { return fmt.Errorf("error marking activity as uploaded: %v", err) } diff --git a/wavesrv/pkg/pcloud/pclouddata.go b/wavesrv/pkg/pcloud/pclouddata.go index 0b4401c0b..548b589cf 100644 --- a/wavesrv/pkg/pcloud/pclouddata.go +++ b/wavesrv/pkg/pcloud/pclouddata.go @@ -11,6 +11,7 @@ import ( "github.com/wavetermdev/waveterm/wavesrv/pkg/remote" "github.com/wavetermdev/waveterm/wavesrv/pkg/rtnstate" "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" + "github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry" ) type NoTelemetryInputType struct { @@ -19,11 +20,11 @@ type NoTelemetryInputType struct { } type TelemetryInputType struct { - UserId string `json:"userid"` - ClientId string `json:"clientid"` - CurDay string `json:"curday"` - DefaultShell string `json:"defaultshell"` - Activity []*sstore.ActivityType `json:"activity"` + UserId string `json:"userid"` + ClientId string `json:"clientid"` + CurDay string `json:"curday"` + DefaultShell string `json:"defaultshell"` + Activity []*telemetry.ActivityType `json:"activity"` } type WebShareUpdateType struct { diff --git a/wavesrv/pkg/playbook/playbook.go b/wavesrv/pkg/playbook/playbook.go new file mode 100644 index 000000000..69760becb --- /dev/null +++ b/wavesrv/pkg/playbook/playbook.go @@ -0,0 +1,162 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package playbook + +import ( + "context" + "fmt" + + "github.com/google/uuid" + "github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil" + "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" +) + +type PlaybookType struct { + PlaybookId string `json:"playbookid"` + PlaybookName string `json:"playbookname"` + Description string `json:"description"` + EntryIds []string `json:"entryids"` + + // this is not persisted to DB, just for transport to FE + Entries []*PlaybookEntry `json:"entries"` +} + +func (p *PlaybookType) ToMap() map[string]interface{} { + rtn := make(map[string]interface{}) + rtn["playbookid"] = p.PlaybookId + rtn["playbookname"] = p.PlaybookName + rtn["description"] = p.Description + rtn["entryids"] = dbutil.QuickJsonArr(p.EntryIds) + return rtn +} + +func (p *PlaybookType) FromMap(m map[string]interface{}) bool { + dbutil.QuickSetStr(&p.PlaybookId, m, "playbookid") + dbutil.QuickSetStr(&p.PlaybookName, m, "playbookname") + dbutil.QuickSetStr(&p.Description, m, "description") + dbutil.QuickSetJsonArr(&p.Entries, m, "entries") + return true +} + +// reorders p.Entries to match p.EntryIds +func (p *PlaybookType) OrderEntries() { + if len(p.Entries) == 0 { + return + } + m := make(map[string]*PlaybookEntry) + for _, entry := range p.Entries { + m[entry.EntryId] = entry + } + newList := make([]*PlaybookEntry, 0, len(p.EntryIds)) + for _, entryId := range p.EntryIds { + entry := m[entryId] + if entry != nil { + newList = append(newList, entry) + } + } + p.Entries = newList +} + +// removes from p.EntryIds (not from p.Entries) +func (p *PlaybookType) RemoveEntry(entryIdToRemove string) { + if len(p.EntryIds) == 0 { + return + } + newList := make([]string, 0, len(p.EntryIds)-1) + for _, entryId := range p.EntryIds { + if entryId == entryIdToRemove { + continue + } + newList = append(newList, entryId) + } + p.EntryIds = newList +} + +type PlaybookEntry struct { + PlaybookId string `json:"playbookid"` + EntryId string `json:"entryid"` + Alias string `json:"alias"` + CmdStr string `json:"cmdstr"` + UpdatedTs int64 `json:"updatedts"` + CreatedTs int64 `json:"createdts"` + Description string `json:"description"` + Remove bool `json:"remove,omitempty"` +} + +func CreatePlaybook(ctx context.Context, name string) (*PlaybookType, error) { + return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*PlaybookType, error) { + query := `SELECT playbookid FROM playbook WHERE name = ?` + if tx.Exists(query, name) { + return nil, fmt.Errorf("playbook %q already exists", name) + } + rtn := &PlaybookType{} + rtn.PlaybookId = uuid.New().String() + rtn.PlaybookName = name + query = `INSERT INTO playbook ( playbookid, playbookname, description, entryids) + VALUES (:playbookid,:playbookname,:description,:entryids)` + tx.Exec(query, rtn.ToMap()) + return rtn, nil + }) +} + +func selectPlaybook(tx *sstore.TxWrap, playbookId string) *PlaybookType { + query := `SELECT * FROM playbook where playbookid = ?` + playbook := dbutil.GetMapGen[*PlaybookType](tx, query, playbookId) + return playbook +} + +func AddPlaybookEntry(ctx context.Context, entry *PlaybookEntry) error { + if entry.EntryId == "" { + return fmt.Errorf("invalid entryid") + } + return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + playbook := selectPlaybook(tx, entry.PlaybookId) + if playbook == nil { + return fmt.Errorf("cannot add entry, playbook does not exist") + } + query := `SELECT entryid FROM playbook_entry WHERE entryid = ?` + if tx.Exists(query, entry.EntryId) { + return fmt.Errorf("cannot add entry, entryid already exists") + } + query = `INSERT INTO playbook_entry ( entryid, playbookid, description, alias, cmdstr, createdts, updatedts) + VALUES (:entryid,:playbookid,:description,:alias,:cmdstr,:createdts,:updatedts)` + tx.Exec(query, entry) + playbook.EntryIds = append(playbook.EntryIds, entry.EntryId) + query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?` + tx.Exec(query, dbutil.QuickJsonArr(playbook.EntryIds), entry.PlaybookId) + return nil + }) +} + +func RemovePlaybookEntry(ctx context.Context, playbookId string, entryId string) error { + return sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + playbook := selectPlaybook(tx, playbookId) + if playbook == nil { + return fmt.Errorf("cannot remove playbook entry, playbook does not exist") + } + query := `SELECT entryid FROM playbook_entry WHERE entryid = ?` + if !tx.Exists(query, entryId) { + return fmt.Errorf("cannot remove playbook entry, entry does not exist") + } + query = `DELETE FROM playbook_entry WHERE entryid = ?` + tx.Exec(query, entryId) + playbook.RemoveEntry(entryId) + query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?` + tx.Exec(query, dbutil.QuickJsonArr(playbook.EntryIds), playbookId) + return nil + }) +} + +func GetPlaybookById(ctx context.Context, playbookId string) (*PlaybookType, error) { + return sstore.WithTxRtn(ctx, func(tx *sstore.TxWrap) (*PlaybookType, error) { + rtn := selectPlaybook(tx, playbookId) + if rtn == nil { + return nil, nil + } + query := `SELECT * FROM playbook_entry WHERE playbookid = ?` + tx.Select(&rtn.Entries, query, playbookId) + rtn.OrderEntries() + return rtn, nil + }) +} diff --git a/wavesrv/pkg/remote/remote.go b/wavesrv/pkg/remote/remote.go index 8956a198c..8dac46722 100644 --- a/wavesrv/pkg/remote/remote.go +++ b/wavesrv/pkg/remote/remote.go @@ -39,6 +39,7 @@ import ( "github.com/wavetermdev/waveterm/wavesrv/pkg/scbus" "github.com/wavetermdev/waveterm/wavesrv/pkg/scpacket" "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" + "github.com/wavetermdev/waveterm/wavesrv/pkg/telemetry" "github.com/wavetermdev/waveterm/wavesrv/pkg/userinput" "golang.org/x/crypto/ssh" @@ -1419,8 +1420,8 @@ func getStateVarsFromInitPk(initPk *packet.InitPacketType) map[string]string { return rtn } -func makeReinitErrorUpdate(shellType string) sstore.ActivityUpdate { - rtn := sstore.ActivityUpdate{} +func makeReinitErrorUpdate(shellType string) telemetry.ActivityUpdate { + rtn := telemetry.ActivityUpdate{} if shellType == packet.ShellType_bash { rtn.ReinitBashErrors = 1 } else if shellType == packet.ShellType_zsh { @@ -1441,7 +1442,7 @@ func (msh *MShellProc) ReInit(ctx context.Context, ck base.CommandKey, shellType } defer func() { if rtnErr != nil { - sstore.UpdateActivityWrap(ctx, makeReinitErrorUpdate(shellType), "reiniterror") + telemetry.UpdateActivityWrap(ctx, makeReinitErrorUpdate(shellType), "reiniterror") } }() startTs := time.Now() diff --git a/wavesrv/pkg/sstore/dbops.go b/wavesrv/pkg/sstore/dbops.go index 8accfbf82..cc38cad66 100644 --- a/wavesrv/pkg/sstore/dbops.go +++ b/wavesrv/pkg/sstore/dbops.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/google/uuid" "github.com/jmoiron/sqlx" "github.com/sawka/txwrap" "github.com/wavetermdev/waveterm/waveshell/pkg/base" @@ -25,9 +24,6 @@ import ( "github.com/wavetermdev/waveterm/wavesrv/pkg/scbus" ) -const HistoryCols = "h.historyid, h.ts, h.userid, h.sessionid, h.screenid, h.lineid, h.haderror, h.cmdstr, h.remoteownerid, h.remoteid, h.remotename, h.ismetacmd, h.linenum, h.exitcode, h.durationms, h.festate, h.tags, h.status" -const DefaultMaxHistoryItems = 1000 - var updateWriterCVar = sync.NewCond(&sync.Mutex{}) var WebScreenPtyPosLock = &sync.Mutex{} var WebScreenPtyPosDelIntent = make(map[string]bool) // map[screenid + ":" + lineid] -> bool @@ -235,178 +231,6 @@ func UpdateRemoteStateVars(ctx context.Context, remoteId string, stateVars map[s }) } -func InsertHistoryItem(ctx context.Context, hitem *HistoryItemType) error { - if hitem == nil { - return fmt.Errorf("cannot insert nil history item") - } - txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `INSERT INTO history - ( historyid, ts, userid, sessionid, screenid, lineid, haderror, cmdstr, remoteownerid, remoteid, remotename, ismetacmd, linenum, exitcode, durationms, festate, tags, status) VALUES - (:historyid,:ts,:userid,:sessionid,:screenid,:lineid,:haderror,:cmdstr,:remoteownerid,:remoteid,:remotename,:ismetacmd,:linenum,:exitcode,:durationms,:festate,:tags,:status)` - tx.NamedExec(query, hitem.ToMap()) - return nil - }) - return txErr -} - -const HistoryQueryChunkSize = 1000 - -func _getNextHistoryItem(items []*HistoryItemType, index int, filterFn func(*HistoryItemType) bool) (*HistoryItemType, int) { - for ; index < len(items); index++ { - item := items[index] - if filterFn(item) { - return item, index - } - } - return nil, index -} - -// returns true if done, false if we still need to process more items -func (result *HistoryQueryResult) processItem(item *HistoryItemType, rawOffset int) bool { - if result.prevItems < result.Offset { - result.prevItems++ - return false - } - if len(result.Items) == result.MaxItems { - result.HasMore = true - result.NextRawOffset = rawOffset - return true - } - if len(result.Items) == 0 { - result.RawOffset = rawOffset - } - result.Items = append(result.Items, item) - return false -} - -func runHistoryQueryWithFilter(tx *TxWrap, opts HistoryQueryOpts) (*HistoryQueryResult, error) { - if opts.MaxItems == 0 { - return nil, fmt.Errorf("invalid query, maxitems is 0") - } - rtn := &HistoryQueryResult{Offset: opts.Offset, MaxItems: opts.MaxItems} - var rawOffset int - if opts.RawOffset >= opts.Offset { - rtn.prevItems = opts.Offset - rawOffset = opts.RawOffset - } else { - rawOffset = 0 - } - for { - resultItems, err := runHistoryQuery(tx, opts, rawOffset, HistoryQueryChunkSize) - if err != nil { - return nil, err - } - isDone := false - for resultIdx := 0; resultIdx < len(resultItems); resultIdx++ { - if opts.FilterFn != nil && !opts.FilterFn(resultItems[resultIdx]) { - continue - } - isDone = rtn.processItem(resultItems[resultIdx], rawOffset+resultIdx) - if isDone { - break - } - } - if isDone { - break - } - if len(resultItems) < HistoryQueryChunkSize { - break - } - rawOffset += HistoryQueryChunkSize - } - return rtn, nil -} - -func runHistoryQuery(tx *TxWrap, opts HistoryQueryOpts, realOffset int, itemLimit int) ([]*HistoryItemType, error) { - // check sessionid/screenid format because we are directly inserting them into the SQL - if opts.SessionId != "" { - _, err := uuid.Parse(opts.SessionId) - if err != nil { - return nil, fmt.Errorf("malformed sessionid") - } - } - if opts.ScreenId != "" { - _, err := uuid.Parse(opts.ScreenId) - if err != nil { - return nil, fmt.Errorf("malformed screenid") - } - } - if opts.RemoteId != "" { - _, err := uuid.Parse(opts.RemoteId) - if err != nil { - return nil, fmt.Errorf("malformed remoteid") - } - } - whereClause := "WHERE 1" - var queryArgs []interface{} - hNumStr := "" - if opts.SessionId != "" && opts.ScreenId != "" { - whereClause += fmt.Sprintf(" AND h.sessionid = '%s' AND h.screenid = '%s'", opts.SessionId, opts.ScreenId) - hNumStr = "" - } else if opts.SessionId != "" { - whereClause += fmt.Sprintf(" AND h.sessionid = '%s'", opts.SessionId) - hNumStr = "s" - } else { - hNumStr = "g" - } - if opts.SearchText != "" { - whereClause += " AND h.cmdstr LIKE ? ESCAPE '\\'" - likeArg := opts.SearchText - likeArg = strings.ReplaceAll(likeArg, "%", "\\%") - likeArg = strings.ReplaceAll(likeArg, "_", "\\_") - queryArgs = append(queryArgs, "%"+likeArg+"%") - } - if opts.FromTs > 0 { - whereClause += fmt.Sprintf(" AND h.ts <= %d", opts.FromTs) - } - if opts.RemoteId != "" { - whereClause += fmt.Sprintf(" AND h.remoteid = '%s'", opts.RemoteId) - } - if opts.NoMeta { - whereClause += " AND NOT h.ismetacmd" - } - query := fmt.Sprintf("SELECT %s, ('%s' || CAST((row_number() OVER win) as text)) historynum FROM history h %s WINDOW win AS (ORDER BY h.ts, h.historyid) ORDER BY h.ts DESC, h.historyid DESC LIMIT %d OFFSET %d", HistoryCols, hNumStr, whereClause, itemLimit, realOffset) - marr := tx.SelectMaps(query, queryArgs...) - rtn := make([]*HistoryItemType, len(marr)) - for idx, m := range marr { - hitem := dbutil.FromMap[*HistoryItemType](m) - rtn[idx] = hitem - } - return rtn, nil -} - -func GetHistoryItems(ctx context.Context, opts HistoryQueryOpts) (*HistoryQueryResult, error) { - var rtn *HistoryQueryResult - txErr := WithTx(ctx, func(tx *TxWrap) error { - var err error - rtn, err = runHistoryQueryWithFilter(tx, opts) - if err != nil { - return err - } - return nil - }) - if txErr != nil { - return nil, txErr - } - return rtn, nil -} - -func GetHistoryItemByLineNum(ctx context.Context, screenId string, lineNum int) (*HistoryItemType, error) { - return WithTxRtn(ctx, func(tx *TxWrap) (*HistoryItemType, error) { - query := `SELECT * FROM history WHERE screenid = ? AND linenum = ?` - hitem := dbutil.GetMapGen[*HistoryItemType](tx, query, screenId, lineNum) - return hitem, nil - }) -} - -func GetLastHistoryLineNum(ctx context.Context, screenId string) (int, error) { - return WithTxRtn(ctx, func(tx *TxWrap) (int, error) { - query := `SELECT COALESCE(max(linenum), 0) FROM history WHERE screenid = ?` - maxLineNum := tx.GetInt(query, screenId) - return maxLineNum, nil - }) -} - // includes archived sessions func GetBareSessions(ctx context.Context) ([]*SessionType, error) { var rtn []*SessionType @@ -2273,92 +2097,6 @@ func GetRIsForScreen(ctx context.Context, sessionId string, screenId string) ([] return rtn, nil } -func GetCurDayStr() string { - now := time.Now() - dayStr := now.Format("2006-01-02") - return dayStr -} - -// Wraps UpdateCurrentActivity, but ignores errors -func UpdateActivityWrap(ctx context.Context, update ActivityUpdate, debugStr string) { - err := UpdateCurrentActivity(ctx, update) - if err != nil { - // ignore error, just log, since this is not critical - log.Printf("error updating current activity (%s): %v\n", debugStr, err) - } -} - -func UpdateCurrentActivity(ctx context.Context, update ActivityUpdate) error { - now := time.Now() - dayStr := GetCurDayStr() - txErr := WithTx(ctx, func(tx *TxWrap) error { - var tdata TelemetryData - query := `SELECT tdata FROM activity WHERE day = ?` - found := tx.Get(&tdata, query, dayStr) - if !found { - query = `INSERT INTO activity (day, uploaded, tdata, tzname, tzoffset, clientversion, clientarch, buildtime, osrelease) - VALUES (?, 0, ?, ?, ?, ?, ? , ? , ?)` - tzName, tzOffset := now.Zone() - if len(tzName) > MaxTzNameLen { - tzName = tzName[0:MaxTzNameLen] - } - tx.Exec(query, dayStr, tdata, tzName, tzOffset, scbase.WaveVersion, scbase.ClientArch(), scbase.BuildTime, scbase.UnameKernelRelease()) - } - tdata.NumCommands += update.NumCommands - tdata.FgMinutes += update.FgMinutes - tdata.ActiveMinutes += update.ActiveMinutes - tdata.OpenMinutes += update.OpenMinutes - tdata.ClickShared += update.ClickShared - tdata.HistoryView += update.HistoryView - tdata.BookmarksView += update.BookmarksView - tdata.ReinitBashErrors += update.ReinitBashErrors - tdata.ReinitZshErrors += update.ReinitZshErrors - if update.NumConns > 0 { - tdata.NumConns = update.NumConns - } - query = `UPDATE activity - SET tdata = ?, - clientversion = ?, - buildtime = ? - WHERE day = ?` - tx.Exec(query, tdata, scbase.WaveVersion, scbase.BuildTime, dayStr) - return nil - }) - if txErr != nil { - return txErr - } - return nil -} - -func GetNonUploadedActivity(ctx context.Context) ([]*ActivityType, error) { - var rtn []*ActivityType - txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `SELECT * FROM activity WHERE uploaded = 0 ORDER BY day DESC LIMIT 30` - tx.Select(&rtn, query) - return nil - }) - if txErr != nil { - return nil, txErr - } - return rtn, nil -} - -// note, will not mark the current day as uploaded -func MarkActivityAsUploaded(ctx context.Context, activityArr []*ActivityType) error { - dayStr := GetCurDayStr() - txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `UPDATE activity SET uploaded = 1 WHERE day = ?` - for _, activity := range activityArr { - if activity.Day == dayStr { - continue - } - tx.Exec(query, activity.Day) - } - return nil - }) - return txErr -} - func foundInStrArr(strs []string, s string) bool { for _, sval := range strs { if s == sval { @@ -2423,271 +2161,6 @@ func GetDBVersion(ctx context.Context) (int, error) { return version, txErr } -type bookmarkOrderType struct { - BookmarkId string - OrderIdx int64 -} - -func GetBookmarks(ctx context.Context, tag string) ([]*BookmarkType, error) { - var bms []*BookmarkType - txErr := WithTx(ctx, func(tx *TxWrap) error { - var query string - if tag == "" { - query = `SELECT * FROM bookmark` - bms = dbutil.SelectMapsGen[*BookmarkType](tx, query) - } else { - query = `SELECT * FROM bookmark WHERE EXISTS (SELECT 1 FROM json_each(tags) WHERE value = ?)` - bms = dbutil.SelectMapsGen[*BookmarkType](tx, query, tag) - } - bmMap := dbutil.MakeGenMap(bms) - var orders []bookmarkOrderType - query = `SELECT bookmarkid, orderidx FROM bookmark_order WHERE tag = ?` - tx.Select(&orders, query, tag) - for _, bmOrder := range orders { - bm := bmMap[bmOrder.BookmarkId] - if bm != nil { - bm.OrderIdx = bmOrder.OrderIdx - } - } - return nil - }) - if txErr != nil { - return nil, txErr - } - return bms, nil -} - -func GetBookmarkById(ctx context.Context, bookmarkId string, tag string) (*BookmarkType, error) { - var rtn *BookmarkType - txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `SELECT * FROM bookmark WHERE bookmarkid = ?` - rtn = dbutil.GetMapGen[*BookmarkType](tx, query, bookmarkId) - if rtn == nil { - return nil - } - query = `SELECT orderidx FROM bookmark_order WHERE bookmarkid = ? AND tag = ?` - orderIdx := tx.GetInt(query, bookmarkId, tag) - rtn.OrderIdx = int64(orderIdx) - return nil - }) - if txErr != nil { - return nil, txErr - } - return rtn, nil -} - -func GetBookmarkIdByArg(ctx context.Context, bookmarkArg string) (string, error) { - var rtnId string - txErr := WithTx(ctx, func(tx *TxWrap) error { - if len(bookmarkArg) == 8 { - query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid LIKE (? || '%')` - rtnId = tx.GetString(query, bookmarkArg) - return nil - } - query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` - rtnId = tx.GetString(query, bookmarkArg) - return nil - }) - if txErr != nil { - return "", txErr - } - return rtnId, nil -} - -func GetBookmarkIdsByCmdStr(ctx context.Context, cmdStr string) ([]string, error) { - return WithTxRtn(ctx, func(tx *TxWrap) ([]string, error) { - query := `SELECT bookmarkid FROM bookmark WHERE cmdstr = ?` - bmIds := tx.SelectStrings(query, cmdStr) - return bmIds, nil - }) -} - -// ignores OrderIdx field -func InsertBookmark(ctx context.Context, bm *BookmarkType) error { - if bm == nil || bm.BookmarkId == "" { - return fmt.Errorf("invalid empty bookmark id") - } - txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` - if tx.Exists(query, bm.BookmarkId) { - return fmt.Errorf("bookmarkid already exists") - } - query = `INSERT INTO bookmark ( bookmarkid, createdts, cmdstr, alias, tags, description) - VALUES (:bookmarkid,:createdts,:cmdstr,:alias,:tags,:description)` - tx.NamedExec(query, bm.ToMap()) - for _, tag := range append(bm.Tags, "") { - query = `SELECT COALESCE(max(orderidx), 0) FROM bookmark_order WHERE tag = ?` - maxOrder := tx.GetInt(query, tag) - query = `INSERT INTO bookmark_order (tag, bookmarkid, orderidx) VALUES (?, ?, ?)` - tx.Exec(query, tag, bm.BookmarkId, maxOrder+1) - } - return nil - }) - return txErr -} - -const ( - BookmarkField_Desc = "desc" - BookmarkField_CmdStr = "cmdstr" -) - -func EditBookmark(ctx context.Context, bookmarkId string, editMap map[string]interface{}) error { - txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` - if !tx.Exists(query, bookmarkId) { - return fmt.Errorf("bookmark not found") - } - if desc, found := editMap[BookmarkField_Desc]; found { - query = `UPDATE bookmark SET description = ? WHERE bookmarkid = ?` - tx.Exec(query, desc, bookmarkId) - } - if cmdStr, found := editMap[BookmarkField_CmdStr]; found { - query = `UPDATE bookmark SET cmdstr = ? WHERE bookmarkid = ?` - tx.Exec(query, cmdStr, bookmarkId) - } - return nil - }) - return txErr -} - -func fixupBookmarkOrder(tx *TxWrap) { - query := ` -WITH new_order AS ( - SELECT tag, bookmarkid, row_number() OVER (PARTITION BY tag ORDER BY orderidx) AS newidx FROM bookmark_order -) -UPDATE bookmark_order -SET orderidx = new_order.newidx -FROM new_order -WHERE bookmark_order.tag = new_order.tag AND bookmark_order.bookmarkid = new_order.bookmarkid -` - tx.Exec(query) -} - -func DeleteBookmark(ctx context.Context, bookmarkId string) error { - txErr := WithTx(ctx, func(tx *TxWrap) error { - query := `SELECT bookmarkid FROM bookmark WHERE bookmarkid = ?` - if !tx.Exists(query, bookmarkId) { - return fmt.Errorf("bookmark not found") - } - query = `DELETE FROM bookmark WHERE bookmarkid = ?` - tx.Exec(query, bookmarkId) - query = `DELETE FROM bookmark_order WHERE bookmarkid = ?` - tx.Exec(query, bookmarkId) - fixupBookmarkOrder(tx) - return nil - }) - return txErr -} - -func CreatePlaybook(ctx context.Context, name string) (*PlaybookType, error) { - return WithTxRtn(ctx, func(tx *TxWrap) (*PlaybookType, error) { - query := `SELECT playbookid FROM playbook WHERE name = ?` - if tx.Exists(query, name) { - return nil, fmt.Errorf("playbook %q already exists", name) - } - rtn := &PlaybookType{} - rtn.PlaybookId = uuid.New().String() - rtn.PlaybookName = name - query = `INSERT INTO playbook ( playbookid, playbookname, description, entryids) - VALUES (:playbookid,:playbookname,:description,:entryids)` - tx.Exec(query, rtn.ToMap()) - return rtn, nil - }) -} - -func selectPlaybook(tx *TxWrap, playbookId string) *PlaybookType { - query := `SELECT * FROM playbook where playbookid = ?` - playbook := dbutil.GetMapGen[*PlaybookType](tx, query, playbookId) - return playbook -} - -func AddPlaybookEntry(ctx context.Context, entry *PlaybookEntry) error { - if entry.EntryId == "" { - return fmt.Errorf("invalid entryid") - } - return WithTx(ctx, func(tx *TxWrap) error { - playbook := selectPlaybook(tx, entry.PlaybookId) - if playbook == nil { - return fmt.Errorf("cannot add entry, playbook does not exist") - } - query := `SELECT entryid FROM playbook_entry WHERE entryid = ?` - if tx.Exists(query, entry.EntryId) { - return fmt.Errorf("cannot add entry, entryid already exists") - } - query = `INSERT INTO playbook_entry ( entryid, playbookid, description, alias, cmdstr, createdts, updatedts) - VALUES (:entryid,:playbookid,:description,:alias,:cmdstr,:createdts,:updatedts)` - tx.Exec(query, entry) - playbook.EntryIds = append(playbook.EntryIds, entry.EntryId) - query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?` - tx.Exec(query, quickJsonArr(playbook.EntryIds), entry.PlaybookId) - return nil - }) -} - -func RemovePlaybookEntry(ctx context.Context, playbookId string, entryId string) error { - return WithTx(ctx, func(tx *TxWrap) error { - playbook := selectPlaybook(tx, playbookId) - if playbook == nil { - return fmt.Errorf("cannot remove playbook entry, playbook does not exist") - } - query := `SELECT entryid FROM playbook_entry WHERE entryid = ?` - if !tx.Exists(query, entryId) { - return fmt.Errorf("cannot remove playbook entry, entry does not exist") - } - query = `DELETE FROM playbook_entry WHERE entryid = ?` - tx.Exec(query, entryId) - playbook.RemoveEntry(entryId) - query = `UPDATE playbook SET entryids = ? WHERE playbookid = ?` - tx.Exec(query, quickJsonArr(playbook.EntryIds), playbookId) - return nil - }) -} - -func GetPlaybookById(ctx context.Context, playbookId string) (*PlaybookType, error) { - return WithTxRtn(ctx, func(tx *TxWrap) (*PlaybookType, error) { - rtn := selectPlaybook(tx, playbookId) - if rtn == nil { - return nil, nil - } - query := `SELECT * FROM playbook_entry WHERE playbookid = ?` - tx.Select(&rtn.Entries, query, playbookId) - rtn.OrderEntries() - return rtn, nil - }) -} - -func getLineIdsFromHistoryItems(historyItems []*HistoryItemType) []string { - var rtn []string - for _, hitem := range historyItems { - if hitem.LineId != "" { - rtn = append(rtn, hitem.LineId) - } - } - return rtn -} - -func GetLineCmdsFromHistoryItems(ctx context.Context, historyItems []*HistoryItemType) ([]*LineType, []*CmdType, error) { - if len(historyItems) == 0 { - return nil, nil, nil - } - return WithTxRtn3(ctx, func(tx *TxWrap) ([]*LineType, []*CmdType, error) { - lineIdsJsonArr := quickJsonArr(getLineIdsFromHistoryItems(historyItems)) - query := `SELECT * FROM line WHERE lineid IN (SELECT value FROM json_each(?))` - lineArr := dbutil.SelectMappable[*LineType](tx, query, lineIdsJsonArr) - query = `SELECT * FROM cmd WHERE lineid IN (SELECT value FROM json_each(?))` - cmdArr := dbutil.SelectMapsGen[*CmdType](tx, query, lineIdsJsonArr) - return lineArr, cmdArr, nil - }) -} - -func PurgeHistoryByIds(ctx context.Context, historyIds []string) error { - return WithTx(ctx, func(tx *TxWrap) error { - query := `DELETE FROM history WHERE historyid IN (SELECT value FROM json_each(?))` - tx.Exec(query, quickJsonArr(historyIds)) - return nil - }) -} - func CountScreenWebShares(ctx context.Context) (int, error) { return WithTxRtn(ctx, func(tx *TxWrap) (int, error) { query := `SELECT count(*) FROM screen WHERE sharemode = ?` @@ -2704,37 +2177,38 @@ func CountScreenLines(ctx context.Context, screenId string) (int, error) { }) } -func CanScreenWebShare(ctx context.Context, screen *ScreenType) error { - if screen == nil { - return fmt.Errorf("cannot share screen, not found") - } - if screen.ShareMode == ShareModeWeb { - return fmt.Errorf("screen is already shared to web") - } - if screen.ShareMode != ShareModeLocal { - return fmt.Errorf("screen cannot be shared, invalid current share mode %q (must be local)", screen.ShareMode) - } - if screen.Archived { - return fmt.Errorf("screen cannot be shared, must un-archive before sharing") - } - webShareCount, err := CountScreenWebShares(ctx) - if err != nil { - return fmt.Errorf("screen cannot be share: error getting webshare count: %v", err) - } - if webShareCount >= MaxWebShareScreenCount { - go UpdateCurrentActivity(context.Background(), ActivityUpdate{WebShareLimit: 1}) - return fmt.Errorf("screen cannot be shared, limited to a maximum of %d shared screen(s)", MaxWebShareScreenCount) - } - lineCount, err := CountScreenLines(ctx, screen.ScreenId) - if err != nil { - return fmt.Errorf("screen cannot be share: error getting screen line count: %v", err) - } - if lineCount > MaxWebShareLineCount { - go UpdateCurrentActivity(context.Background(), ActivityUpdate{WebShareLimit: 1}) - return fmt.Errorf("screen cannot be shared, limited to a maximum of %d lines", MaxWebShareLineCount) - } - return nil -} +// Below is currently not used and is causing circular dependency due to moving telemetry code to a new package. It will likely be rewritten whenever we add back webshare and should be moved to a different package then. +// func CanScreenWebShare(ctx context.Context, screen *ScreenType) error { +// if screen == nil { +// return fmt.Errorf("cannot share screen, not found") +// } +// if screen.ShareMode == ShareModeWeb { +// return fmt.Errorf("screen is already shared to web") +// } +// if screen.ShareMode != ShareModeLocal { +// return fmt.Errorf("screen cannot be shared, invalid current share mode %q (must be local)", screen.ShareMode) +// } +// if screen.Archived { +// return fmt.Errorf("screen cannot be shared, must un-archive before sharing") +// } +// webShareCount, err := CountScreenWebShares(ctx) +// if err != nil { +// return fmt.Errorf("screen cannot be share: error getting webshare count: %v", err) +// } +// if webShareCount >= MaxWebShareScreenCount { +// go UpdateCurrentActivity(context.Background(), ActivityUpdate{WebShareLimit: 1}) +// return fmt.Errorf("screen cannot be shared, limited to a maximum of %d shared screen(s)", MaxWebShareScreenCount) +// } +// lineCount, err := CountScreenLines(ctx, screen.ScreenId) +// if err != nil { +// return fmt.Errorf("screen cannot be share: error getting screen line count: %v", err) +// } +// if lineCount > MaxWebShareLineCount { +// go UpdateCurrentActivity(context.Background(), ActivityUpdate{WebShareLimit: 1}) +// return fmt.Errorf("screen cannot be shared, limited to a maximum of %d lines", MaxWebShareLineCount) +// } +// return nil +// } func ScreenWebShareStart(ctx context.Context, screenId string, shareOpts ScreenWebShareOpts) error { return WithTx(ctx, func(tx *TxWrap) error { diff --git a/wavesrv/pkg/sstore/sstore.go b/wavesrv/pkg/sstore/sstore.go index d82a08b4f..e637782b5 100644 --- a/wavesrv/pkg/sstore/sstore.go +++ b/wavesrv/pkg/sstore/sstore.go @@ -110,6 +110,7 @@ const ( SSHConfigSrcTypeImport = "sshconfig-import" ) +// TODO: move to webshare package once sstore code is more modular const ( ShareModeLocal = "local" ShareModeWeb = "web" @@ -154,8 +155,6 @@ const ( UpdateType_PtyPos = "pty:pos" ) -const MaxTzNameLen = 50 - var globalDBLock = &sync.Mutex{} var globalDB *sqlx.DB var globalDBErr error @@ -233,56 +232,6 @@ type ClientWinSizeType struct { FullScreen bool `json:"fullscreen,omitempty"` } -type ActivityUpdate struct { - FgMinutes int - ActiveMinutes int - OpenMinutes int - NumCommands int - ClickShared int - HistoryView int - BookmarksView int - NumConns int - WebShareLimit int - ReinitBashErrors int - ReinitZshErrors int - BuildTime string -} - -type ActivityType struct { - Day string `json:"day"` - Uploaded bool `json:"-"` - TData TelemetryData `json:"tdata"` - TzName string `json:"tzname"` - TzOffset int `json:"tzoffset"` - ClientVersion string `json:"clientversion"` - ClientArch string `json:"clientarch"` - BuildTime string `json:"buildtime"` - DefaultShell string `json:"defaultshell"` - OSRelease string `json:"osrelease"` -} - -type TelemetryData struct { - NumCommands int `json:"numcommands"` - ActiveMinutes int `json:"activeminutes"` - FgMinutes int `json:"fgminutes"` - OpenMinutes int `json:"openminutes"` - ClickShared int `json:"clickshared,omitempty"` - HistoryView int `json:"historyview,omitempty"` - BookmarksView int `json:"bookmarksview,omitempty"` - NumConns int `json:"numconns"` - WebShareLimit int `json:"websharelimit,omitempty"` - ReinitBashErrors int `json:"reinitbasherrors,omitempty"` - ReinitZshErrors int `json:"reinitzsherrors,omitempty"` -} - -func (tdata TelemetryData) Value() (driver.Value, error) { - return quickValueJson(tdata) -} - -func (tdata *TelemetryData) Scan(val interface{}) error { - return quickScanJson(tdata, val) -} - type SidebarValueType struct { Collapsed bool `json:"collapsed"` Width int `json:"width"` @@ -398,52 +347,6 @@ type SessionStatsType struct { DiskStats SessionDiskSizeType `json:"diskstats"` } -func (h *HistoryItemType) ToMap() map[string]interface{} { - rtn := make(map[string]interface{}) - rtn["historyid"] = h.HistoryId - rtn["ts"] = h.Ts - rtn["userid"] = h.UserId - rtn["sessionid"] = h.SessionId - rtn["screenid"] = h.ScreenId - rtn["lineid"] = h.LineId - rtn["linenum"] = h.LineNum - rtn["haderror"] = h.HadError - rtn["cmdstr"] = h.CmdStr - rtn["remoteownerid"] = h.Remote.OwnerId - rtn["remoteid"] = h.Remote.RemoteId - rtn["remotename"] = h.Remote.Name - rtn["ismetacmd"] = h.IsMetaCmd - rtn["exitcode"] = h.ExitCode - rtn["durationms"] = h.DurationMs - rtn["festate"] = quickJson(h.FeState) - rtn["tags"] = quickJson(h.Tags) - rtn["status"] = h.Status - return rtn -} - -func (h *HistoryItemType) FromMap(m map[string]interface{}) bool { - quickSetStr(&h.HistoryId, m, "historyid") - quickSetInt64(&h.Ts, m, "ts") - quickSetStr(&h.UserId, m, "userid") - quickSetStr(&h.SessionId, m, "sessionid") - quickSetStr(&h.ScreenId, m, "screenid") - quickSetStr(&h.LineId, m, "lineid") - quickSetBool(&h.HadError, m, "haderror") - quickSetStr(&h.CmdStr, m, "cmdstr") - quickSetStr(&h.Remote.OwnerId, m, "remoteownerid") - quickSetStr(&h.Remote.RemoteId, m, "remoteid") - quickSetStr(&h.Remote.Name, m, "remotename") - quickSetBool(&h.IsMetaCmd, m, "ismetacmd") - quickSetStr(&h.HistoryNum, m, "historynum") - quickSetInt64(&h.LineNum, m, "linenum") - dbutil.QuickSetNullableInt64(&h.ExitCode, m, "exitcode") - dbutil.QuickSetNullableInt64(&h.DurationMs, m, "durationms") - quickSetJson(&h.FeState, m, "festate") - quickSetJson(&h.Tags, m, "tags") - quickSetStr(&h.Status, m, "status") - return true -} - type ScreenOptsType struct { TabColor string `json:"tabcolor,omitempty"` TabIcon string `json:"tabicon,omitempty"` @@ -619,55 +522,6 @@ type ScreenAnchorType struct { AnchorOffset int `json:"anchoroffset,omitempty"` } -type HistoryItemType struct { - HistoryId string `json:"historyid"` - Ts int64 `json:"ts"` - UserId string `json:"userid"` - SessionId string `json:"sessionid"` - ScreenId string `json:"screenid"` - LineId string `json:"lineid"` - HadError bool `json:"haderror"` - CmdStr string `json:"cmdstr"` - Remote RemotePtrType `json:"remote"` - IsMetaCmd bool `json:"ismetacmd"` - ExitCode *int64 `json:"exitcode,omitempty"` - DurationMs *int64 `json:"durationms,omitempty"` - FeState FeStateType `json:"festate,omitempty"` - Tags map[string]bool `json:"tags,omitempty"` - LineNum int64 `json:"linenum" dbmap:"-"` - Status string `json:"status"` - - // only for updates - Remove bool `json:"remove" dbmap:"-"` - - // transient (string because of different history orderings) - HistoryNum string `json:"historynum" dbmap:"-"` -} - -type HistoryQueryOpts struct { - Offset int - MaxItems int - FromTs int64 - SearchText string - SessionId string - RemoteId string - ScreenId string - NoMeta bool - RawOffset int - FilterFn func(*HistoryItemType) bool -} - -type HistoryQueryResult struct { - MaxItems int - Items []*HistoryItemType - Offset int // the offset shown to user - RawOffset int // internal offset - HasMore bool - NextRawOffset int // internal offset used by pager for next query - - prevItems int // holds number of items skipped by RawOffset -} - type TermOpts struct { Rows int64 `json:"rows"` Cols int64 `json:"cols"` @@ -850,114 +704,6 @@ type OpenAIResponse struct { Choices []OpenAIChoiceType `json:"choices,omitempty"` } -type PlaybookType struct { - PlaybookId string `json:"playbookid"` - PlaybookName string `json:"playbookname"` - Description string `json:"description"` - EntryIds []string `json:"entryids"` - - // this is not persisted to DB, just for transport to FE - Entries []*PlaybookEntry `json:"entries"` -} - -func (p *PlaybookType) ToMap() map[string]interface{} { - rtn := make(map[string]interface{}) - rtn["playbookid"] = p.PlaybookId - rtn["playbookname"] = p.PlaybookName - rtn["description"] = p.Description - rtn["entryids"] = quickJsonArr(p.EntryIds) - return rtn -} - -func (p *PlaybookType) FromMap(m map[string]interface{}) bool { - quickSetStr(&p.PlaybookId, m, "playbookid") - quickSetStr(&p.PlaybookName, m, "playbookname") - quickSetStr(&p.Description, m, "description") - quickSetJsonArr(&p.Entries, m, "entries") - return true -} - -// reorders p.Entries to match p.EntryIds -func (p *PlaybookType) OrderEntries() { - if len(p.Entries) == 0 { - return - } - m := make(map[string]*PlaybookEntry) - for _, entry := range p.Entries { - m[entry.EntryId] = entry - } - newList := make([]*PlaybookEntry, 0, len(p.EntryIds)) - for _, entryId := range p.EntryIds { - entry := m[entryId] - if entry != nil { - newList = append(newList, entry) - } - } - p.Entries = newList -} - -// removes from p.EntryIds (not from p.Entries) -func (p *PlaybookType) RemoveEntry(entryIdToRemove string) { - if len(p.EntryIds) == 0 { - return - } - newList := make([]string, 0, len(p.EntryIds)-1) - for _, entryId := range p.EntryIds { - if entryId == entryIdToRemove { - continue - } - newList = append(newList, entryId) - } - p.EntryIds = newList -} - -type PlaybookEntry struct { - PlaybookId string `json:"playbookid"` - EntryId string `json:"entryid"` - Alias string `json:"alias"` - CmdStr string `json:"cmdstr"` - UpdatedTs int64 `json:"updatedts"` - CreatedTs int64 `json:"createdts"` - Description string `json:"description"` - Remove bool `json:"remove,omitempty"` -} - -type BookmarkType struct { - BookmarkId string `json:"bookmarkid"` - CreatedTs int64 `json:"createdts"` - CmdStr string `json:"cmdstr"` - Alias string `json:"alias,omitempty"` - Tags []string `json:"tags"` - Description string `json:"description"` - OrderIdx int64 `json:"orderidx"` - Remove bool `json:"remove,omitempty"` -} - -func (bm *BookmarkType) GetSimpleKey() string { - return bm.BookmarkId -} - -func (bm *BookmarkType) ToMap() map[string]interface{} { - rtn := make(map[string]interface{}) - rtn["bookmarkid"] = bm.BookmarkId - rtn["createdts"] = bm.CreatedTs - rtn["cmdstr"] = bm.CmdStr - rtn["alias"] = bm.Alias - rtn["description"] = bm.Description - rtn["tags"] = quickJsonArr(bm.Tags) - return rtn -} - -func (bm *BookmarkType) FromMap(m map[string]interface{}) bool { - quickSetStr(&bm.BookmarkId, m, "bookmarkid") - quickSetInt64(&bm.CreatedTs, m, "createdts") - quickSetStr(&bm.Alias, m, "alias") - quickSetStr(&bm.CmdStr, m, "cmdstr") - quickSetStr(&bm.Description, m, "description") - quickSetJsonArr(&bm.Tags, m, "tags") - return true -} - type ResolveItem struct { Name string Num int diff --git a/wavesrv/pkg/sstore/updatetypes.go b/wavesrv/pkg/sstore/updatetypes.go index a4cea78a5..2b777f421 100644 --- a/wavesrv/pkg/sstore/updatetypes.go +++ b/wavesrv/pkg/sstore/updatetypes.go @@ -91,18 +91,6 @@ func (ClearInfoUpdate) GetType() string { return "clearinfo" } -type HistoryInfoType struct { - HistoryType string `json:"historytype"` - SessionId string `json:"sessionid,omitempty"` - ScreenId string `json:"screenid,omitempty"` - Items []*HistoryItemType `json:"items"` - Show bool `json:"show"` -} - -func (HistoryInfoType) GetType() string { - return "history" -} - type InteractiveUpdate bool func (InteractiveUpdate) GetType() string { @@ -122,43 +110,6 @@ func (ConnectUpdate) GetType() string { return "connect" } -type MainViewUpdate struct { - MainView string `json:"mainview"` - HistoryView *HistoryViewData `json:"historyview,omitempty"` - BookmarksView *BookmarksUpdate `json:"bookmarksview,omitempty"` -} - -func (MainViewUpdate) GetType() string { - return "mainview" -} - -type BookmarksUpdate struct { - Bookmarks []*BookmarkType `json:"bookmarks"` - SelectedBookmark string `json:"selectedbookmark,omitempty"` -} - -func (BookmarksUpdate) GetType() string { - return "bookmarks" -} - -func AddBookmarksUpdate(update *scbus.ModelUpdatePacketType, bookmarks []*BookmarkType, selectedBookmark *string) { - if selectedBookmark == nil { - update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks}) - } else { - update.AddUpdate(BookmarksUpdate{Bookmarks: bookmarks, SelectedBookmark: *selectedBookmark}) - } -} - -type HistoryViewData struct { - Items []*HistoryItemType `json:"items"` - Offset int `json:"offset"` - RawOffset int `json:"rawoffset"` - NextRawOffset int `json:"nextrawoffset"` - HasMore bool `json:"hasmore"` - Lines []*LineType `json:"lines"` - Cmds []*CmdType `json:"cmds"` -} - type RemoteEditType struct { RemoteEdit bool `json:"remoteedit"` RemoteId string `json:"remoteid,omitempty"` diff --git a/wavesrv/pkg/telemetry/telemetry.go b/wavesrv/pkg/telemetry/telemetry.go new file mode 100644 index 000000000..865d949aa --- /dev/null +++ b/wavesrv/pkg/telemetry/telemetry.go @@ -0,0 +1,153 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package telemetry + +import ( + "context" + "database/sql/driver" + "log" + "time" + + "github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil" + "github.com/wavetermdev/waveterm/wavesrv/pkg/scbase" + "github.com/wavetermdev/waveterm/wavesrv/pkg/sstore" +) + +const MaxTzNameLen = 50 + +type ActivityUpdate struct { + FgMinutes int + ActiveMinutes int + OpenMinutes int + NumCommands int + ClickShared int + HistoryView int + BookmarksView int + NumConns int + WebShareLimit int + ReinitBashErrors int + ReinitZshErrors int + BuildTime string +} + +type ActivityType struct { + Day string `json:"day"` + Uploaded bool `json:"-"` + TData TelemetryData `json:"tdata"` + TzName string `json:"tzname"` + TzOffset int `json:"tzoffset"` + ClientVersion string `json:"clientversion"` + ClientArch string `json:"clientarch"` + BuildTime string `json:"buildtime"` + DefaultShell string `json:"defaultshell"` + OSRelease string `json:"osrelease"` +} + +type TelemetryData struct { + NumCommands int `json:"numcommands"` + ActiveMinutes int `json:"activeminutes"` + FgMinutes int `json:"fgminutes"` + OpenMinutes int `json:"openminutes"` + ClickShared int `json:"clickshared,omitempty"` + HistoryView int `json:"historyview,omitempty"` + BookmarksView int `json:"bookmarksview,omitempty"` + NumConns int `json:"numconns"` + WebShareLimit int `json:"websharelimit,omitempty"` + ReinitBashErrors int `json:"reinitbasherrors,omitempty"` + ReinitZshErrors int `json:"reinitzsherrors,omitempty"` +} + +func (tdata TelemetryData) Value() (driver.Value, error) { + return dbutil.QuickValueJson(tdata) +} + +func (tdata *TelemetryData) Scan(val interface{}) error { + return dbutil.QuickScanJson(tdata, val) +} + +// Wraps UpdateCurrentActivity, but ignores errors +func UpdateActivityWrap(ctx context.Context, update ActivityUpdate, debugStr string) { + err := UpdateCurrentActivity(ctx, update) + if err != nil { + // ignore error, just log, since this is not critical + log.Printf("error updating current activity (%s): %v\n", debugStr, err) + } +} + +func GetCurDayStr() string { + now := time.Now() + dayStr := now.Format("2006-01-02") + return dayStr +} + +func UpdateCurrentActivity(ctx context.Context, update ActivityUpdate) error { + now := time.Now() + dayStr := GetCurDayStr() + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + var tdata TelemetryData + query := `SELECT tdata FROM activity WHERE day = ?` + found := tx.Get(&tdata, query, dayStr) + if !found { + query = `INSERT INTO activity (day, uploaded, tdata, tzname, tzoffset, clientversion, clientarch, buildtime, osrelease) + VALUES (?, 0, ?, ?, ?, ?, ? , ? , ?)` + tzName, tzOffset := now.Zone() + if len(tzName) > MaxTzNameLen { + tzName = tzName[0:MaxTzNameLen] + } + tx.Exec(query, dayStr, tdata, tzName, tzOffset, scbase.WaveVersion, scbase.ClientArch(), scbase.BuildTime, scbase.UnameKernelRelease()) + } + tdata.NumCommands += update.NumCommands + tdata.FgMinutes += update.FgMinutes + tdata.ActiveMinutes += update.ActiveMinutes + tdata.OpenMinutes += update.OpenMinutes + tdata.ClickShared += update.ClickShared + tdata.HistoryView += update.HistoryView + tdata.BookmarksView += update.BookmarksView + tdata.ReinitBashErrors += update.ReinitBashErrors + tdata.ReinitZshErrors += update.ReinitZshErrors + if update.NumConns > 0 { + tdata.NumConns = update.NumConns + } + query = `UPDATE activity + SET tdata = ?, + clientversion = ?, + buildtime = ? + WHERE day = ?` + tx.Exec(query, tdata, scbase.WaveVersion, scbase.BuildTime, dayStr) + return nil + }) + if txErr != nil { + return txErr + } + return nil +} + +func GetNonUploadedActivity(ctx context.Context) ([]*ActivityType, error) { + var rtn []*ActivityType + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `SELECT * FROM activity WHERE uploaded = 0 ORDER BY day DESC LIMIT 30` + tx.Select(&rtn, query) + return nil + }) + if txErr != nil { + return nil, txErr + } + return rtn, nil +} + +// note, will not mark the current day as uploaded +func MarkActivityAsUploaded(ctx context.Context, activityArr []*ActivityType) error { + dayStr := GetCurDayStr() + txErr := sstore.WithTx(ctx, func(tx *sstore.TxWrap) error { + query := `UPDATE activity SET uploaded = 1 WHERE day = ?` + for _, activity := range activityArr { + if activity.Day == dayStr { + continue + } + tx.Exec(query, activity.Day) + } + return nil + }) + return txErr +}