diff --git a/cmd/server/main-server.go b/cmd/server/main-server.go index 0a8f6d297..c916ea6a8 100644 --- a/cmd/server/main-server.go +++ b/cmd/server/main-server.go @@ -9,7 +9,6 @@ import ( "log" "os" "os/signal" - "runtime/debug" "runtime" "sync" @@ -19,6 +18,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/authkey" "github.com/wavetermdev/waveterm/pkg/blockcontroller" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote/conncontroller" "github.com/wavetermdev/waveterm/pkg/service" "github.com/wavetermdev/waveterm/pkg/telemetry" @@ -111,15 +111,16 @@ func telemetryLoop() { } } +func panicTelemetryHandler() { + activity := telemetry.ActivityUpdate{NumPanics: 1} + err := telemetry.UpdateActivity(context.Background(), activity) + if err != nil { + log.Printf("error updating activity (panicTelemetryHandler): %v\n", err) + } +} + func sendTelemetryWrapper() { - defer func() { - r := recover() - if r == nil { - return - } - log.Printf("[error] in sendTelemetryWrapper: %v\n", r) - debug.PrintStack() - }() + defer panichandler.PanicHandler("sendTelemetryWrapper") ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() beforeSendActivityUpdate(ctx) @@ -254,7 +255,9 @@ func main() { log.Printf("error initializing wstore: %v\n", err) return } + panichandler.PanicTelemetryHandler = panicTelemetryHandler go func() { + defer panichandler.PanicHandler("InitCustomShellStartupFiles") err := shellutil.InitCustomShellStartupFiles() if err != nil { log.Printf("error initializing wsh and shell-integration files: %v\n", err) diff --git a/cmd/wsh/cmd/wshcmd-connserver.go b/cmd/wsh/cmd/wshcmd-connserver.go index 4f82c1067..e7db38c0b 100644 --- a/cmd/wsh/cmd/wshcmd-connserver.go +++ b/cmd/wsh/cmd/wshcmd-connserver.go @@ -14,6 +14,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/packetparser" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -53,6 +54,7 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) { var routeIdContainer atomic.Pointer[string] proxy := wshutil.MakeRpcProxy() go func() { + defer panichandler.PanicHandler("handleNewListenerConn:AdaptOutputChToStream") writeErr := wshutil.AdaptOutputChToStream(proxy.ToRemoteCh, conn) if writeErr != nil { log.Printf("error writing to domain socket: %v\n", writeErr) @@ -60,6 +62,7 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) { }() go func() { // when input is closed, close the connection + defer panichandler.PanicHandler("handleNewListenerConn:AdaptStreamToMsgCh") defer func() { conn.Close() routeIdPtr := routeIdContainer.Load() @@ -136,6 +139,7 @@ func serverRunRouter() error { rawCh := make(chan []byte, wshutil.DefaultOutputChSize) go packetparser.Parse(os.Stdin, termProxy.FromRemoteCh, rawCh) go func() { + defer panichandler.PanicHandler("serverRunRouter:WritePackets") for msg := range termProxy.ToRemoteCh { packetparser.WritePacket(os.Stdout, msg) } diff --git a/pkg/blockcontroller/blockcontroller.go b/pkg/blockcontroller/blockcontroller.go index f2d6a2751..befaf8517 100644 --- a/pkg/blockcontroller/blockcontroller.go +++ b/pkg/blockcontroller/blockcontroller.go @@ -16,6 +16,7 @@ import ( "time" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/conncontroller" "github.com/wavetermdev/waveterm/pkg/shellexec" @@ -354,6 +355,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj ptyBuffer := wshutil.MakePtyBuffer(wshutil.WaveOSCPrefix, bc.ShellProc.Cmd, wshProxy.FromRemoteCh) go func() { // handles regular output from the pty (goes to the blockfile and xterm) + defer panichandler.PanicHandler("blockcontroller:shellproc-pty-read-loop") defer func() { log.Printf("[shellproc] pty-read loop done\n") bc.ShellProc.Close() @@ -386,6 +388,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj go func() { // handles input from the shellInputCh, sent to pty // use shellInputCh instead of bc.ShellInputCh (because we want to be attached to *this* ch. bc.ShellInputCh can be updated) + defer panichandler.PanicHandler("blockcontroller:shellproc-input-loop") for ic := range shellInputCh { if len(ic.InputData) > 0 { bc.ShellProc.Cmd.Write(ic.InputData) @@ -403,6 +406,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj } }() go func() { + defer panichandler.PanicHandler("blockcontroller:shellproc-output-loop") // handles outputCh -> shellInputCh for msg := range wshProxy.ToRemoteCh { encodedMsg, err := wshutil.EncodeWaveOSCBytes(wshutil.WaveServerOSC, msg) @@ -413,6 +417,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj } }() go func() { + defer panichandler.PanicHandler("blockcontroller:shellproc-wait-loop") // wait for the shell to finish defer func() { wshutil.DefaultRouter.UnregisterRoute(wshutil.MakeControllerRouteId(bc.BlockId)) @@ -484,6 +489,7 @@ func (bc *BlockController) run(bdata *waveobj.Block, blockMeta map[string]any, r runOnStart := getBoolFromMeta(blockMeta, waveobj.MetaKey_CmdRunOnStart, true) if runOnStart { go func() { + defer panichandler.PanicHandler("blockcontroller:run-shell-command") var termSize waveobj.TermSize if rtOpts != nil { termSize = rtOpts.TermSize diff --git a/pkg/filestore/blockstore.go b/pkg/filestore/blockstore.go index 60980fb9e..ca2dd22a5 100644 --- a/pkg/filestore/blockstore.go +++ b/pkg/filestore/blockstore.go @@ -12,12 +12,12 @@ import ( "fmt" "io/fs" "log" - "runtime/debug" "sync" "sync/atomic" "time" "github.com/wavetermdev/waveterm/pkg/ijson" + "github.com/wavetermdev/waveterm/pkg/panichandler" ) const ( @@ -514,12 +514,7 @@ func (s *FileStore) runFlushWithNewContext() (FlushStats, error) { } func (s *FileStore) runFlusher() { - defer func() { - if r := recover(); r != nil { - log.Printf("panic in filestore flusher: %v\n", r) - debug.PrintStack() - } - }() + defer panichandler.PanicHandler("filestore flusher") for { stats, err := s.runFlushWithNewContext() if err != nil || stats.NumDirtyEntries > 0 { diff --git a/pkg/panichandler/panichandler.go b/pkg/panichandler/panichandler.go new file mode 100644 index 000000000..55b2ac4bd --- /dev/null +++ b/pkg/panichandler/panichandler.go @@ -0,0 +1,43 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package panichandler + +import ( + "fmt" + "log" + "runtime/debug" +) + +// to log NumPanics into the local telemetry system +// gets around import cycles +var PanicTelemetryHandler func() + +func PanicHandlerNoTelemetry(debugStr string) { + r := recover() + if r == nil { + return + } + log.Printf("[panic] in %s: %v\n", debugStr, r) + debug.PrintStack() +} + +// returns an error (wrapping the panic) if a panic occurred +func PanicHandler(debugStr string) error { + r := recover() + if r == nil { + return nil + } + log.Printf("[panic] in %s: %v\n", debugStr, r) + debug.PrintStack() + if PanicTelemetryHandler != nil { + go func() { + defer PanicHandlerNoTelemetry("PanicTelemetryHandler") + PanicTelemetryHandler() + }() + } + if err, ok := r.(error); ok { + return fmt.Errorf("panic in %s: %w", debugStr, err) + } + return fmt.Errorf("panic in %s: %v", debugStr, r) +} diff --git a/pkg/remote/conncontroller/conncontroller.go b/pkg/remote/conncontroller/conncontroller.go index 93b4b6316..b8de2ced0 100644 --- a/pkg/remote/conncontroller/conncontroller.go +++ b/pkg/remote/conncontroller/conncontroller.go @@ -20,6 +20,7 @@ import ( "github.com/kevinburke/ssh_config" "github.com/skeema/knownhosts" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/userinput" @@ -193,6 +194,7 @@ func (conn *SSHConn) OpenDomainSocketListener() error { conn.DomainSockListener = listener }) go func() { + defer panichandler.PanicHandler("conncontroller:OpenDomainSocketListener") defer conn.WithLock(func() { conn.DomainSockListener = nil conn.SockName = "" @@ -252,6 +254,7 @@ func (conn *SSHConn) StartConnServer() error { }) // service the I/O go func() { + defer panichandler.PanicHandler("conncontroller:sshSession.Wait") // wait for termination, clear the controller defer conn.WithLock(func() { conn.ConnController = nil @@ -260,6 +263,7 @@ func (conn *SSHConn) StartConnServer() error { log.Printf("conn controller (%q) terminated: %v", conn.GetName(), waitErr) }() go func() { + defer panichandler.PanicHandler("conncontroller:sshSession-output") readErr := wshutil.StreamToLines(pipeRead, func(line []byte) { lineStr := string(line) if !strings.HasSuffix(lineStr, "\n") { diff --git a/pkg/remote/connutil.go b/pkg/remote/connutil.go index 07dbab920..593cb8e0c 100644 --- a/pkg/remote/connutil.go +++ b/pkg/remote/connutil.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" + "github.com/wavetermdev/waveterm/pkg/panichandler" "golang.org/x/crypto/ssh" ) @@ -288,6 +289,7 @@ func CpHostToRemote(client *ssh.Client, sourcePath string, destPath string) erro } go func() { + defer panichandler.PanicHandler("connutil:CpHostToRemote") io.Copy(installStdin, input) session.Close() // this allows the command to complete for reasons i don't fully understand }() diff --git a/pkg/service/objectservice/objectservice.go b/pkg/service/objectservice/objectservice.go index 08338da1f..4e69fe9a4 100644 --- a/pkg/service/objectservice/objectservice.go +++ b/pkg/service/objectservice/objectservice.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wcore" @@ -94,6 +95,7 @@ func (svc *ObjectService) AddTabToWorkspace(windowId string, tabName string, act } updates := waveobj.ContextGetUpdatesRtn(ctx) go func() { + defer panichandler.PanicHandler("ObjectService:AddTabToWorkspace:SendUpdateEvents") wps.Broker.SendUpdateEvents(updates) }() return tabId, updates, nil @@ -142,6 +144,7 @@ func (svc *ObjectService) SetActiveTab(windowId string, tabId string) (waveobj.U } updates := waveobj.ContextGetUpdatesRtn(ctx) go func() { + defer panichandler.PanicHandler("ObjectService:SetActiveTab:SendUpdateEvents") wps.Broker.SendUpdateEvents(updates) }() var extraUpdates waveobj.UpdatesRtnType diff --git a/pkg/service/windowservice/windowservice.go b/pkg/service/windowservice/windowservice.go index 2735e7dd8..5366d12b5 100644 --- a/pkg/service/windowservice/windowservice.go +++ b/pkg/service/windowservice/windowservice.go @@ -11,6 +11,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/blockcontroller" "github.com/wavetermdev/waveterm/pkg/eventbus" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/waveobj" @@ -75,6 +76,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s } } go func() { + defer panichandler.PanicHandler("WindowService:CloseTab:StopBlockControllers") for _, blockId := range tab.BlockIds { blockcontroller.StopBlockController(blockId) } @@ -107,6 +109,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s } updates := waveobj.ContextGetUpdatesRtn(ctx) go func() { + defer panichandler.PanicHandler("WindowService:CloseTab:SendUpdateEvents") wps.Broker.SendUpdateEvents(updates) }() return rtn, updates, nil diff --git a/pkg/shellexec/conninterface.go b/pkg/shellexec/conninterface.go index b16dffc9c..4da15181c 100644 --- a/pkg/shellexec/conninterface.go +++ b/pkg/shellexec/conninterface.go @@ -9,6 +9,7 @@ import ( "time" "github.com/creack/pty" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/wsl" "golang.org/x/crypto/ssh" ) @@ -51,6 +52,7 @@ func (cw CmdWrap) KillGraceful(timeout time.Duration) { cw.Cmd.Process.Signal(syscall.SIGTERM) } go func() { + defer panichandler.PanicHandler("KillGraceful:Kill") time.Sleep(timeout) if cw.Cmd.ProcessState == nil || !cw.Cmd.ProcessState.Exited() { cw.Cmd.Process.Kill() // force kill if it is already not exited @@ -159,6 +161,7 @@ func (wcw WslCmdWrap) KillGraceful(timeout time.Duration) { } process.Signal(os.Interrupt) go func() { + defer panichandler.PanicHandler("KillGraceful-wsl:Kill") time.Sleep(timeout) process := wcw.WslCmd.GetProcess() processState := wcw.WslCmd.GetProcessState() diff --git a/pkg/shellexec/shellexec.go b/pkg/shellexec/shellexec.go index ff985352d..dd76af838 100644 --- a/pkg/shellexec/shellexec.go +++ b/pkg/shellexec/shellexec.go @@ -19,6 +19,7 @@ import ( "time" "github.com/creack/pty" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/conncontroller" "github.com/wavetermdev/waveterm/pkg/util/shellutil" @@ -51,6 +52,7 @@ type ShellProc struct { func (sp *ShellProc) Close() { sp.Cmd.KillGraceful(DefaultGracefulKillWait) go func() { + defer panichandler.PanicHandler("ShellProc.Close") waitErr := sp.Cmd.Wait() sp.SetWaitErrorAndSignalDone(waitErr) @@ -450,6 +452,7 @@ func RunSimpleCmdInPty(ecmd *exec.Cmd, termSize waveobj.TermSize) ([]byte, error ioDone := make(chan bool) var outputBuf bytes.Buffer go func() { + panichandler.PanicHandler("RunSimpleCmdInPty:ioCopy") // ignore error (/dev/ptmx has read error when process is done) defer close(ioDone) io.Copy(&outputBuf, cmdPty) diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go index 0ba5ded7a..e5f968323 100644 --- a/pkg/telemetry/telemetry.go +++ b/pkg/telemetry/telemetry.go @@ -9,6 +9,7 @@ import ( "log" "time" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/daystr" "github.com/wavetermdev/waveterm/pkg/util/dbutil" "github.com/wavetermdev/waveterm/pkg/wavebase" @@ -36,6 +37,7 @@ type ActivityUpdate struct { NumSSHConn int `json:"numsshconn,omitempty"` NumWSLConn int `json:"numwslconn,omitempty"` NumMagnify int `json:"nummagnify,omitempty"` + NumPanics int `json:"numpanics,omitempty"` Startup int `json:"startup,omitempty"` Shutdown int `json:"shutdown,omitempty"` SetTabTheme int `json:"settabtheme,omitempty"` @@ -71,6 +73,7 @@ type TelemetryData struct { NewTab int `json:"newtab"` NumStartup int `json:"numstartup,omitempty"` NumShutdown int `json:"numshutdown,omitempty"` + NumPanics int `json:"numpanics,omitempty"` SetTabTheme int `json:"settabtheme,omitempty"` Displays []ActivityDisplayType `json:"displays,omitempty"` Renderers map[string]int `json:"renderers,omitempty"` @@ -104,6 +107,7 @@ func AutoUpdateChannel() string { // Wraps UpdateCurrentActivity, spawns goroutine, and logs errors func GoUpdateActivityWrap(update ActivityUpdate, debugStr string) { go func() { + defer panichandler.PanicHandlerNoTelemetry("GoUpdateActivityWrap") ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() err := UpdateActivity(ctx, update) @@ -138,6 +142,7 @@ func UpdateActivity(ctx context.Context, update ActivityUpdate) error { tdata.NumShutdown += update.Shutdown tdata.SetTabTheme += update.SetTabTheme tdata.NumMagnify += update.NumMagnify + tdata.NumPanics += update.NumPanics if update.NumTabs > 0 { tdata.NumTabs = update.NumTabs } diff --git a/pkg/waveai/anthropicbackend.go b/pkg/waveai/anthropicbackend.go index 008b272cc..3ec6f264a 100644 --- a/pkg/waveai/anthropicbackend.go +++ b/pkg/waveai/anthropicbackend.go @@ -10,11 +10,10 @@ import ( "errors" "fmt" "io" - "log" "net/http" - "runtime/debug" "strings" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -115,17 +114,10 @@ func (AnthropicBackend) StreamCompletion(ctx context.Context, request wshrpc.Ope go func() { defer func() { - if r := recover(); r != nil { - // Convert panic to error and send it - log.Printf("panic: %v\n", r) - debug.PrintStack() - err, ok := r.(error) - if !ok { - err = fmt.Errorf("anthropic backend panic: %v", r) - } - rtn <- makeAIError(err) + panicErr := panichandler.PanicHandler("AnthropicBackend.StreamCompletion") + if panicErr != nil { + rtn <- makeAIError(panicErr) } - // Always close the channel close(rtn) }() diff --git a/pkg/waveai/cloudbackend.go b/pkg/waveai/cloudbackend.go index 03b5abba6..ded005702 100644 --- a/pkg/waveai/cloudbackend.go +++ b/pkg/waveai/cloudbackend.go @@ -11,6 +11,7 @@ import ( "log" "github.com/gorilla/websocket" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/wcloud" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -37,7 +38,13 @@ func (WaveAICloudBackend) StreamCompletion(ctx context.Context, request wshrpc.O rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.OpenAIPacketType]) wsEndpoint := wcloud.GetWSEndpoint() go func() { - defer close(rtn) + defer func() { + panicErr := panichandler.PanicHandler("WaveAICloudBackend.StreamCompletion") + if panicErr != nil { + rtn <- makeAIError(panicErr) + } + close(rtn) + }() if wsEndpoint == "" { rtn <- makeAIError(fmt.Errorf("no cloud ws endpoint found")) return diff --git a/pkg/waveai/openaibackend.go b/pkg/waveai/openaibackend.go index a7c7b4388..c5e30b7ad 100644 --- a/pkg/waveai/openaibackend.go +++ b/pkg/waveai/openaibackend.go @@ -8,12 +8,11 @@ import ( "errors" "fmt" "io" - "log" "regexp" - "runtime/debug" "strings" openaiapi "github.com/sashabaranov/go-openai" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -75,17 +74,10 @@ func (OpenAIBackend) StreamCompletion(ctx context.Context, request wshrpc.OpenAi rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.OpenAIPacketType]) go func() { defer func() { - if r := recover(); r != nil { - // Convert panic to error and send it - log.Printf("panic: %v\n", r) - debug.PrintStack() - err, ok := r.(error) - if !ok { - err = fmt.Errorf("openai backend panic: %v", r) - } - rtn <- makeAIError(err) + panicErr := panichandler.PanicHandler("OpenAIBackend.StreamCompletion") + if panicErr != nil { + rtn <- makeAIError(panicErr) } - // Always close the channel close(rtn) }() if request.Opts == nil { diff --git a/pkg/waveapp/waveappserverimpl.go b/pkg/waveapp/waveappserverimpl.go index 33cff5519..2148592b2 100644 --- a/pkg/waveapp/waveappserverimpl.go +++ b/pkg/waveapp/waveappserverimpl.go @@ -10,6 +10,7 @@ import ( "log" "net/http" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/vdom" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -23,12 +24,11 @@ func (*WaveAppServerImpl) WshServerImpl() {} func (impl *WaveAppServerImpl) VDomRenderCommand(ctx context.Context, feUpdate vdom.VDomFrontendUpdate) chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate] { respChan := make(chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate], 5) - defer func() { - if r := recover(); r != nil { - log.Printf("panic in VDomRenderCommand: %v\n", r) + panicErr := panichandler.PanicHandler("VDomRenderCommand") + if panicErr != nil { respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{ - Error: fmt.Errorf("internal error: %v", r), + Error: panicErr, } close(respChan) } @@ -88,6 +88,7 @@ func (impl *WaveAppServerImpl) VDomRenderCommand(ctx context.Context, feUpdate v // Split the update into chunks and send them sequentially updates := vdom.SplitBackendUpdate(update) go func() { + defer panichandler.PanicHandler("VDomRenderCommand:splitUpdates") defer close(respChan) for _, splitUpdate := range updates { respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{ @@ -108,9 +109,10 @@ func (impl *WaveAppServerImpl) VDomUrlRequestCommand(ctx context.Context, data w defer writer.Close() // Ensures writer is closed before the channel is closed defer func() { - if r := recover(); r != nil { + panicErr := panichandler.PanicHandler("VDomUrlRequestCommand") + if panicErr != nil { writer.WriteHeader(http.StatusInternalServerError) - writer.Write([]byte(fmt.Sprintf("internal server error: %v", r))) + writer.Write([]byte(fmt.Sprintf("internal server error: %v", panicErr))) } }() diff --git a/pkg/wconfig/filewatcher.go b/pkg/wconfig/filewatcher.go index 3a4f4d8ef..7b8533db9 100644 --- a/pkg/wconfig/filewatcher.go +++ b/pkg/wconfig/filewatcher.go @@ -10,6 +10,7 @@ import ( "sync" "github.com/fsnotify/fsnotify" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wps" ) @@ -64,6 +65,7 @@ func (w *Watcher) Start() { w.sendInitialValues() go func() { + defer panichandler.PanicHandler("filewatcher:Start") for { select { case event, ok := <-w.watcher.Events: diff --git a/pkg/wcore/wcore.go b/pkg/wcore/wcore.go index 719a99246..0e699f5a5 100644 --- a/pkg/wcore/wcore.go +++ b/pkg/wcore/wcore.go @@ -12,6 +12,7 @@ import ( "github.com/google/uuid" "github.com/wavetermdev/waveterm/pkg/blockcontroller" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" @@ -267,6 +268,7 @@ func CreateBlock(ctx context.Context, tabId string, blockDef *waveobj.BlockDef, return nil, fmt.Errorf("error creating block: %w", err) } go func() { + defer panichandler.PanicHandler("CreateBlock:telemetry") blockView := blockDef.Meta.GetString(waveobj.MetaKey_View, "") if blockView == "" { return diff --git a/pkg/web/web.go b/pkg/web/web.go index ebe967482..1d51a5ffc 100644 --- a/pkg/web/web.go +++ b/pkg/web/web.go @@ -15,7 +15,6 @@ import ( "net/http" "os" "path/filepath" - "runtime/debug" "strconv" "time" @@ -25,6 +24,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/authkey" "github.com/wavetermdev/waveterm/pkg/docsite" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/service" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -358,21 +358,18 @@ type ClientActiveState struct { func WebFnWrap(opts WebFnOpts, fn WebFnType) WebFnType { return func(w http.ResponseWriter, r *http.Request) { defer func() { - recErr := recover() + recErr := panichandler.PanicHandler("WebFnWrap") if recErr == nil { return } - panicStr := fmt.Sprintf("panic: %v", recErr) - log.Printf("panic: %v\n", recErr) - debug.PrintStack() if opts.JsonErrors { - jsonRtn := marshalReturnValue(nil, fmt.Errorf(panicStr)) + jsonRtn := marshalReturnValue(nil, recErr) w.Header().Set(ContentTypeHeaderKey, ContentTypeJson) w.Header().Set(ContentLengthHeaderKey, fmt.Sprintf("%d", len(jsonRtn))) w.WriteHeader(http.StatusOK) w.Write(jsonRtn) } else { - http.Error(w, panicStr, http.StatusInternalServerError) + http.Error(w, recErr.Error(), http.StatusInternalServerError) } }() if !opts.AllowCaching { diff --git a/pkg/web/ws.go b/pkg/web/ws.go index c0374efff..233aeb166 100644 --- a/pkg/web/ws.go +++ b/pkg/web/ws.go @@ -9,7 +9,6 @@ import ( "log" "net" "net/http" - "runtime/debug" "sync" "time" @@ -18,6 +17,7 @@ import ( "github.com/gorilla/websocket" "github.com/wavetermdev/waveterm/pkg/authkey" "github.com/wavetermdev/waveterm/pkg/eventbus" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/web/webcmd" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" @@ -81,11 +81,9 @@ func getStringFromMap(jmsg map[string]any, key string) string { func processWSCommand(jmsg map[string]any, outputCh chan any, rpcInputCh chan []byte) { var rtnErr error defer func() { - r := recover() - if r != nil { - rtnErr = fmt.Errorf("panic: %v", r) - log.Printf("[websocket] panic in processMessage: %v\n", r) - debug.PrintStack() + panicErr := panichandler.PanicHandler("processWSCommand") + if panicErr != nil { + rtnErr = panicErr } if rtnErr == nil { return @@ -304,6 +302,7 @@ func HandleWsInternal(w http.ResponseWriter, r *http.Request) error { wg := &sync.WaitGroup{} wg.Add(2) go func() { + defer panichandler.PanicHandler("HandleWsInternal:outputCh") // no waitgroup add here // move values from rpcOutputCh to outputCh for msgBytes := range wproxy.ToRemoteCh { @@ -315,11 +314,13 @@ func HandleWsInternal(w http.ResponseWriter, r *http.Request) error { } }() go func() { + defer panichandler.PanicHandler("HandleWsInternal:ReadLoop") // read loop defer wg.Done() ReadLoop(conn, outputCh, closeCh, wproxy.FromRemoteCh, routeId) }() go func() { + defer panichandler.PanicHandler("HandleWsInternal:WriteLoop") // write loop defer wg.Done() WriteLoop(conn, outputCh, closeCh, routeId) diff --git a/pkg/wshrpc/wshclient/wshclientutil.go b/pkg/wshrpc/wshclient/wshclientutil.go index fcddbd7f8..1a79dede9 100644 --- a/pkg/wshrpc/wshclient/wshclientutil.go +++ b/pkg/wshrpc/wshclient/wshclientutil.go @@ -6,6 +6,7 @@ package wshclient import ( "errors" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" @@ -39,6 +40,7 @@ func sendRpcRequestCallHelper[T any](w *wshutil.WshRpc, command string, data int func rtnErr[T any](ch chan wshrpc.RespOrErrorUnion[T], err error) { go func() { + defer panichandler.PanicHandler("wshclientutil:rtnErr") ch <- wshrpc.RespOrErrorUnion[T]{Error: err} close(ch) }() @@ -63,6 +65,7 @@ func sendRpcRequestResponseStreamHelper[T any](w *wshutil.WshRpc, command string reqHandler.SendCancel() } go func() { + defer panichandler.PanicHandler("sendRpcRequestResponseStreamHelper") defer close(respChan) for { if reqHandler.ResponseDone() { diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 93905f815..c8f69afa6 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -18,6 +18,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/blockcontroller" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/conncontroller" "github.com/wavetermdev/waveterm/pkg/telemetry" @@ -45,11 +46,7 @@ func (*WshServer) WshServerImpl() {} var WshServerImpl = WshServer{} func (ws *WshServer) TestCommand(ctx context.Context, data string) error { - defer func() { - if r := recover(); r != nil { - log.Printf("panic in TestCommand: %v", r) - } - }() + defer panichandler.PanicHandler("TestCommand") rpcSource := wshutil.GetRpcSourceFromContext(ctx) log.Printf("TEST src:%s | %s\n", rpcSource, data) return nil @@ -65,6 +62,7 @@ func (ws *WshServer) MessageCommand(ctx context.Context, data wshrpc.CommandMess func (ws *WshServer) StreamTestCommand(ctx context.Context) chan wshrpc.RespOrErrorUnion[int] { rtn := make(chan wshrpc.RespOrErrorUnion[int]) go func() { + defer panichandler.PanicHandler("StreamTestCommand") for i := 1; i <= 5; i++ { rtn <- wshrpc.RespOrErrorUnion[int]{Response: i} time.Sleep(1 * time.Second) diff --git a/pkg/wshutil/wshadapter.go b/pkg/wshutil/wshadapter.go index 9f21fd300..48ebfb2db 100644 --- a/pkg/wshutil/wshadapter.go +++ b/pkg/wshutil/wshadapter.go @@ -8,6 +8,7 @@ import ( "reflect" "strings" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -131,6 +132,7 @@ func serverImplAdapter(impl any) func(*RpcResponseHandler) bool { return true } go func() { + defer panichandler.PanicHandler("serverImplAdapter:responseStream") defer handler.Finalize() // must use reflection here because we don't know the generic type of RespOrErrorUnion for { diff --git a/pkg/wshutil/wshmultiproxy.go b/pkg/wshutil/wshmultiproxy.go index be2888bf1..2287c514d 100644 --- a/pkg/wshutil/wshmultiproxy.go +++ b/pkg/wshutil/wshmultiproxy.go @@ -9,6 +9,7 @@ import ( "sync" "github.com/google/uuid" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -112,6 +113,7 @@ func (p *WshRpcMultiProxy) handleUnauthMessage(msgBytes []byte) { p.setRouteInfo(routeInfo.AuthToken, routeInfo) p.sendAuthResponse(msg, routeId, routeInfo.AuthToken) go func() { + defer panichandler.PanicHandler("WshRpcMultiProxy:handleUnauthMessage") for msgBytes := range routeInfo.Proxy.ToRemoteCh { p.ToRemoteCh <- msgBytes } diff --git a/pkg/wshutil/wshrouter.go b/pkg/wshutil/wshrouter.go index 10b5df517..8166cf3e7 100644 --- a/pkg/wshutil/wshrouter.go +++ b/pkg/wshutil/wshrouter.go @@ -13,6 +13,7 @@ import ( "time" "github.com/google/uuid" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wps" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -91,6 +92,7 @@ func noRouteErr(routeId string) error { } func (router *WshRouter) SendEvent(routeId string, event wps.WaveEvent) { + defer panichandler.PanicHandler("WshRouter.SendEvent") rpc := router.GetRpc(routeId) if rpc == nil { return @@ -296,6 +298,7 @@ func (router *WshRouter) RegisterRoute(routeId string, rpc AbstractRpcClient, sh } router.RouteMap[routeId] = rpc go func() { + defer panichandler.PanicHandler("WshRouter:registerRoute:recvloop") // announce if shouldAnnounce && !alreadyExists && router.GetUpstreamClient() != nil { announceMsg := RpcMessage{Command: wshrpc.Command_RouteAnnounce, Source: routeId} @@ -341,6 +344,7 @@ func (router *WshRouter) UnregisterRoute(routeId string) { } } go func() { + defer panichandler.PanicHandler("WshRouter:unregisterRoute:routegone") wps.Broker.UnsubscribeAll(routeId) wps.Broker.Publish(wps.WaveEvent{Event: wps.Event_RouteGone, Scopes: []string{routeId}}) }() diff --git a/pkg/wshutil/wshrpc.go b/pkg/wshutil/wshrpc.go index 7d31246ac..4bbbbd01e 100644 --- a/pkg/wshutil/wshrpc.go +++ b/pkg/wshutil/wshrpc.go @@ -10,12 +10,12 @@ import ( "fmt" "log" "reflect" - "runtime/debug" "sync" "sync/atomic" "time" "github.com/google/uuid" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wps" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -281,15 +281,6 @@ func (w *WshRpc) handleRequest(req *RpcMessage) { } var respHandler *RpcResponseHandler - defer func() { - if r := recover(); r != nil { - log.Printf("panic in handleRequest: %v\n", r) - debug.PrintStack() - if respHandler != nil { - respHandler.SendResponseError(fmt.Errorf("panic: %v", r)) - } - } - }() timeoutMs := req.Timeout if timeoutMs <= 0 { timeoutMs = DefaultTimeoutMs @@ -313,13 +304,13 @@ func (w *WshRpc) handleRequest(req *RpcMessage) { w.registerResponseHandler(req.ReqId, respHandler) isAsync := false defer func() { - if r := recover(); r != nil { - log.Printf("panic in handleRequest: %v\n", r) - debug.PrintStack() - respHandler.SendResponseError(fmt.Errorf("panic: %v", r)) + panicErr := panichandler.PanicHandler("handleRequest") + if panicErr != nil { + respHandler.SendResponseError(panicErr) } if isAsync { go func() { + defer panichandler.PanicHandler("handleRequest:finalize") <-ctx.Done() respHandler.Finalize() }() @@ -394,6 +385,7 @@ func (w *WshRpc) registerRpc(ctx context.Context, reqId string) chan *RpcMessage Ctx: ctx, } go func() { + defer panichandler.PanicHandler("registerRpc:timeout") <-ctx.Done() w.unregisterRpc(reqId, fmt.Errorf("EC-TIME: timeout waiting for response")) }() @@ -463,12 +455,7 @@ func (handler *RpcRequestHandler) Context() context.Context { } func (handler *RpcRequestHandler) SendCancel() { - defer func() { - if r := recover(); r != nil { - // this is likely a write to closed channel - log.Printf("panic in SendCancel: %v\n", r) - } - }() + defer panichandler.PanicHandler("SendCancel") msg := &RpcMessage{ Cancel: true, ReqId: handler.reqId, @@ -573,13 +560,7 @@ func (handler *RpcResponseHandler) SendMessage(msg string) { } func (handler *RpcResponseHandler) SendResponse(data any, done bool) error { - defer func() { - if r := recover(); r != nil { - // this is likely a write to closed channel - log.Printf("panic in SendResponse: %v\n", r) - handler.close() - } - }() + defer panichandler.PanicHandler("SendResponse") if handler.reqId == "" { return nil // no response expected } @@ -604,13 +585,7 @@ func (handler *RpcResponseHandler) SendResponse(data any, done bool) error { } func (handler *RpcResponseHandler) SendResponseError(err error) { - defer func() { - if r := recover(); r != nil { - // this is likely a write to closed channel - log.Printf("panic in SendResponseError: %v\n", r) - handler.close() - } - }() + defer panichandler.PanicHandler("SendResponseError") if handler.reqId == "" || handler.done.Load() { return } @@ -659,12 +634,7 @@ func (w *WshRpc) SendComplexRequest(command string, data any, opts *wshrpc.RpcOp if timeoutMs <= 0 { timeoutMs = DefaultTimeoutMs } - defer func() { - if r := recover(); r != nil { - log.Printf("panic in SendComplexRequest: %v\n", r) - rtnErr = fmt.Errorf("panic: %v", r) - } - }() + defer panichandler.PanicHandler("SendComplexRequest") if command == "" { return nil, fmt.Errorf("command cannot be empty") } diff --git a/pkg/wshutil/wshutil.go b/pkg/wshutil/wshutil.go index 8be9c908a..733bf1c54 100644 --- a/pkg/wshutil/wshutil.go +++ b/pkg/wshutil/wshutil.go @@ -19,6 +19,7 @@ import ( "github.com/golang-jwt/jwt/v5" "github.com/google/uuid" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/packetparser" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -150,6 +151,7 @@ func installShutdownSignalHandlers(quiet bool) { sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) go func() { + defer panichandler.PanicHandlerNoTelemetry("installShutdownSignalHandlers") for sig := range sigCh { DoShutdown(fmt.Sprintf("got signal %v", sig), 1, quiet) break @@ -198,6 +200,7 @@ func SetupTerminalRpcClient(serverImpl ServerImpl) (*WshRpc, io.Reader) { ptyBuf := MakePtyBuffer(WaveServerOSCPrefix, os.Stdin, messageCh) rpcClient := MakeWshRpc(messageCh, outputCh, wshrpc.RpcContext{}, serverImpl) go func() { + defer panichandler.PanicHandler("SetupTerminalRpcClient") for msg := range outputCh { barr, err := EncodeWaveOSCBytes(WaveOSC, msg) if err != nil { @@ -218,6 +221,7 @@ func SetupPacketRpcClient(input io.Reader, output io.Writer, serverImpl ServerIm rpcClient := MakeWshRpc(messageCh, outputCh, wshrpc.RpcContext{}, serverImpl) go packetparser.Parse(input, messageCh, rawCh) go func() { + defer panichandler.PanicHandler("SetupPacketRpcClient:outputloop") for msg := range outputCh { packetparser.WritePacket(output, msg) } @@ -230,6 +234,7 @@ func SetupConnRpcClient(conn net.Conn, serverImpl ServerImpl) (*WshRpc, chan err outputCh := make(chan []byte, DefaultOutputChSize) writeErrCh := make(chan error, 1) go func() { + defer panichandler.PanicHandler("SetupConnRpcClient:AdaptOutputChToStream") writeErr := AdaptOutputChToStream(outputCh, conn) if writeErr != nil { writeErrCh <- writeErr @@ -237,6 +242,7 @@ func SetupConnRpcClient(conn net.Conn, serverImpl ServerImpl) (*WshRpc, chan err } }() go func() { + defer panichandler.PanicHandler("SetupConnRpcClient:AdaptStreamToMsgCh") // when input is closed, close the connection defer conn.Close() AdaptStreamToMsgCh(conn, inputCh) @@ -264,6 +270,7 @@ func SetupDomainSocketRpcClient(sockName string, serverImpl ServerImpl) (*WshRpc } rtn, errCh, err := SetupConnRpcClient(conn, serverImpl) go func() { + defer panichandler.PanicHandler("SetupDomainSocketRpcClient:closeConn") defer conn.Close() err := <-errCh if err != nil && err != io.EOF { @@ -410,9 +417,11 @@ func HandleStdIOClient(logName string, input io.Reader, output io.Writer) { proxy.DisposeRoutes() } go func() { + defer panichandler.PanicHandler("HandleStdIOClient:RunUnauthLoop") proxy.RunUnauthLoop() }() go func() { + defer panichandler.PanicHandler("HandleStdIOClient:ToRemoteChLoop") defer closeDoneCh() for msg := range proxy.ToRemoteCh { err := packetparser.WritePacket(output, msg) @@ -423,6 +432,7 @@ func HandleStdIOClient(logName string, input io.Reader, output io.Writer) { } }() go func() { + defer panichandler.PanicHandler("HandleStdIOClient:RawChLoop") defer closeDoneCh() for msg := range rawCh { log.Printf("[%s:stdout] %s", logName, msg) @@ -435,6 +445,7 @@ func handleDomainSocketClient(conn net.Conn) { var routeIdContainer atomic.Pointer[string] proxy := MakeRpcProxy() go func() { + defer panichandler.PanicHandler("handleDomainSocketClient:AdaptOutputChToStream") writeErr := AdaptOutputChToStream(proxy.ToRemoteCh, conn) if writeErr != nil { log.Printf("error writing to domain socket: %v\n", writeErr) @@ -442,6 +453,7 @@ func handleDomainSocketClient(conn net.Conn) { }() go func() { // when input is closed, close the connection + defer panichandler.PanicHandler("handleDomainSocketClient:AdaptStreamToMsgCh") defer func() { conn.Close() routeIdPtr := routeIdContainer.Load() diff --git a/pkg/wsl/wsl-util.go b/pkg/wsl/wsl-util.go index 5d1f70d35..231ac3f1b 100644 --- a/pkg/wsl/wsl-util.go +++ b/pkg/wsl/wsl-util.go @@ -15,6 +15,8 @@ import ( "path/filepath" "strings" "time" + + "github.com/wavetermdev/waveterm/pkg/panichandler" ) func DetectShell(ctx context.Context, client *Distro) (string, error) { @@ -233,6 +235,7 @@ func CpHostToRemote(ctx context.Context, client *Distro, sourcePath string, dest return fmt.Errorf("cannot open local file %s to send to host: %v", sourcePath, err) } go func() { + defer panichandler.PanicHandler("wslutil:cpHostToRemote:catStdin") io.Copy(catStdin, input) installStepCmds["cat"].Cancel() diff --git a/pkg/wsl/wsl.go b/pkg/wsl/wsl.go index 916843db6..eb364144e 100644 --- a/pkg/wsl/wsl.go +++ b/pkg/wsl/wsl.go @@ -13,6 +13,7 @@ import ( "sync/atomic" "time" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/userinput" "github.com/wavetermdev/waveterm/pkg/util/shellutil" @@ -234,6 +235,7 @@ func (conn *WslConn) StartConnServer() error { }) // service the I/O go func() { + defer panichandler.PanicHandler("wsl:StartConnServer:wait") // wait for termination, clear the controller defer conn.WithLock(func() { conn.ConnController = nil @@ -242,6 +244,7 @@ func (conn *WslConn) StartConnServer() error { log.Printf("conn controller (%q) terminated: %v", conn.GetName(), waitErr) }() go func() { + defer panichandler.PanicHandler("wsl:StartConnServer:handleStdIOClient") logName := fmt.Sprintf("conncontroller:%s", conn.GetName()) wshutil.HandleStdIOClient(logName, pipeRead, inputPipeWrite) }() diff --git a/pkg/wstore/wstore_dbops.go b/pkg/wstore/wstore_dbops.go index de7aa5c59..a2086ac2a 100644 --- a/pkg/wstore/wstore_dbops.go +++ b/pkg/wstore/wstore_dbops.go @@ -11,6 +11,7 @@ import ( "time" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/util/dbutil" "github.com/wavetermdev/waveterm/pkg/waveobj" ) @@ -211,6 +212,7 @@ func DBDelete(ctx context.Context, otype string, id string) error { return err } go func() { + defer panichandler.PanicHandler("DBDelete:filestore.DeleteZone") // we spawn a go routine here because we don't want to reuse the DB connection // since DBDelete is called in a transaction from DeleteTab deleteCtx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)