panic handlers everywhere (#1327)

cmd/server/main-server.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"os"
 	"os/signal"
-	"runtime/debug"
 
 	"runtime"
 	"sync"
@@ -19,6 +18,7 @@ import (
 	"github.com/wavetermdev/waveterm/pkg/authkey"
 	"github.com/wavetermdev/waveterm/pkg/blockcontroller"
 	"github.com/wavetermdev/waveterm/pkg/filestore"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
 	"github.com/wavetermdev/waveterm/pkg/service"
 	"github.com/wavetermdev/waveterm/pkg/telemetry"
@@ -111,15 +111,16 @@ func telemetryLoop() {
 	}
 }
 
-func sendTelemetryWrapper() {
+func panicTelemetryHandler() {
-	defer func() {
+	activity := telemetry.ActivityUpdate{NumPanics: 1}
-		r := recover()
+	err := telemetry.UpdateActivity(context.Background(), activity)
-		if r == nil {
+	if err != nil {
-			return
+		log.Printf("error updating activity (panicTelemetryHandler): %v\n", err)
 	}
-		log.Printf("[error] in sendTelemetryWrapper: %v\n", r)
+}
-		debug.PrintStack()
+
-	}()
+func sendTelemetryWrapper() {
+	defer panichandler.PanicHandler("sendTelemetryWrapper")
 	ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
 	defer cancelFn()
 	beforeSendActivityUpdate(ctx)
@@ -254,7 +255,9 @@ func main() {
 		log.Printf("error initializing wstore: %v\n", err)
 		return
 	}
+	panichandler.PanicTelemetryHandler = panicTelemetryHandler
 	go func() {
+		defer panichandler.PanicHandler("InitCustomShellStartupFiles")
 		err := shellutil.InitCustomShellStartupFiles()
 		if err != nil {
 			log.Printf("error initializing wsh and shell-integration files: %v\n", err)

cmd/wsh/cmd/wshcmd-connserver.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/packetparser"
 	"github.com/wavetermdev/waveterm/pkg/wavebase"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
@@ -53,6 +54,7 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) {
 	var routeIdContainer atomic.Pointer[string]
 	proxy := wshutil.MakeRpcProxy()
 	go func() {
+		defer panichandler.PanicHandler("handleNewListenerConn:AdaptOutputChToStream")
 		writeErr := wshutil.AdaptOutputChToStream(proxy.ToRemoteCh, conn)
 		if writeErr != nil {
 			log.Printf("error writing to domain socket: %v\n", writeErr)
@@ -60,6 +62,7 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) {
 	}()
 	go func() {
 		// when input is closed, close the connection
+		defer panichandler.PanicHandler("handleNewListenerConn:AdaptStreamToMsgCh")
 		defer func() {
 			conn.Close()
 			routeIdPtr := routeIdContainer.Load()
@@ -136,6 +139,7 @@ func serverRunRouter() error {
 	rawCh := make(chan []byte, wshutil.DefaultOutputChSize)
 	go packetparser.Parse(os.Stdin, termProxy.FromRemoteCh, rawCh)
 	go func() {
+		defer panichandler.PanicHandler("serverRunRouter:WritePackets")
 		for msg := range termProxy.ToRemoteCh {
 			packetparser.WritePacket(os.Stdout, msg)
 		}

pkg/blockcontroller/blockcontroller.go

@@ -16,6 +16,7 @@ import (
 	"time"
 
 	"github.com/wavetermdev/waveterm/pkg/filestore"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote"
 	"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
 	"github.com/wavetermdev/waveterm/pkg/shellexec"
@@ -354,6 +355,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 	ptyBuffer := wshutil.MakePtyBuffer(wshutil.WaveOSCPrefix, bc.ShellProc.Cmd, wshProxy.FromRemoteCh)
 	go func() {
 		// handles regular output from the pty (goes to the blockfile and xterm)
+		defer panichandler.PanicHandler("blockcontroller:shellproc-pty-read-loop")
 		defer func() {
 			log.Printf("[shellproc] pty-read loop done\n")
 			bc.ShellProc.Close()
@@ -386,6 +388,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 	go func() {
 		// handles input from the shellInputCh, sent to pty
 		// use shellInputCh instead of bc.ShellInputCh (because we want to be attached to *this* ch.  bc.ShellInputCh can be updated)
+		defer panichandler.PanicHandler("blockcontroller:shellproc-input-loop")
 		for ic := range shellInputCh {
 			if len(ic.InputData) > 0 {
 				bc.ShellProc.Cmd.Write(ic.InputData)
@@ -403,6 +406,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 		}
 	}()
 	go func() {
+		defer panichandler.PanicHandler("blockcontroller:shellproc-output-loop")
 		// handles outputCh -> shellInputCh
 		for msg := range wshProxy.ToRemoteCh {
 			encodedMsg, err := wshutil.EncodeWaveOSCBytes(wshutil.WaveServerOSC, msg)
@@ -413,6 +417,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
 		}
 	}()
 	go func() {
+		defer panichandler.PanicHandler("blockcontroller:shellproc-wait-loop")
 		// wait for the shell to finish
 		defer func() {
 			wshutil.DefaultRouter.UnregisterRoute(wshutil.MakeControllerRouteId(bc.BlockId))
@@ -484,6 +489,7 @@ func (bc *BlockController) run(bdata *waveobj.Block, blockMeta map[string]any, r
 	runOnStart := getBoolFromMeta(blockMeta, waveobj.MetaKey_CmdRunOnStart, true)
 	if runOnStart {
 		go func() {
+			defer panichandler.PanicHandler("blockcontroller:run-shell-command")
 			var termSize waveobj.TermSize
 			if rtOpts != nil {
 				termSize = rtOpts.TermSize

pkg/filestore/blockstore.go

@@ -12,12 +12,12 @@ import (
 	"fmt"
 	"io/fs"
 	"log"
-	"runtime/debug"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/wavetermdev/waveterm/pkg/ijson"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 )
 
 const (
@@ -514,12 +514,7 @@ func (s *FileStore) runFlushWithNewContext() (FlushStats, error) {
 }
 
 func (s *FileStore) runFlusher() {
-	defer func() {
+	defer panichandler.PanicHandler("filestore flusher")
-		if r := recover(); r != nil {
-			log.Printf("panic in filestore flusher: %v\n", r)
-			debug.PrintStack()
-		}
-	}()
 	for {
 		stats, err := s.runFlushWithNewContext()
 		if err != nil || stats.NumDirtyEntries > 0 {

pkg/panichandler/panichandler.go

@@ -0,0 +1,43 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package panichandler
+
+import (
+	"fmt"
+	"log"
+	"runtime/debug"
+)
+
+// to log NumPanics into the local telemetry system
+// gets around import cycles
+var PanicTelemetryHandler func()
+
+func PanicHandlerNoTelemetry(debugStr string) {
+	r := recover()
+	if r == nil {
+		return
+	}
+	log.Printf("[panic] in %s: %v\n", debugStr, r)
+	debug.PrintStack()
+}
+
+// returns an error (wrapping the panic) if a panic occurred
+func PanicHandler(debugStr string) error {
+	r := recover()
+	if r == nil {
+		return nil
+	}
+	log.Printf("[panic] in %s: %v\n", debugStr, r)
+	debug.PrintStack()
+	if PanicTelemetryHandler != nil {
+		go func() {
+			defer PanicHandlerNoTelemetry("PanicTelemetryHandler")
+			PanicTelemetryHandler()
+		}()
+	}
+	if err, ok := r.(error); ok {
+		return fmt.Errorf("panic in %s: %w", debugStr, err)
+	}
+	return fmt.Errorf("panic in %s: %v", debugStr, r)
+}

pkg/remote/conncontroller/conncontroller.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/kevinburke/ssh_config"
 	"github.com/skeema/knownhosts"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote"
 	"github.com/wavetermdev/waveterm/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/pkg/userinput"
@@ -193,6 +194,7 @@ func (conn *SSHConn) OpenDomainSocketListener() error {
 		conn.DomainSockListener = listener
 	})
 	go func() {
+		defer panichandler.PanicHandler("conncontroller:OpenDomainSocketListener")
 		defer conn.WithLock(func() {
 			conn.DomainSockListener = nil
 			conn.SockName = ""
@@ -252,6 +254,7 @@ func (conn *SSHConn) StartConnServer() error {
 	})
 	// service the I/O
 	go func() {
+		defer panichandler.PanicHandler("conncontroller:sshSession.Wait")
 		// wait for termination, clear the controller
 		defer conn.WithLock(func() {
 			conn.ConnController = nil
@@ -260,6 +263,7 @@ func (conn *SSHConn) StartConnServer() error {
 		log.Printf("conn controller (%q) terminated: %v", conn.GetName(), waitErr)
 	}()
 	go func() {
+		defer panichandler.PanicHandler("conncontroller:sshSession-output")
 		readErr := wshutil.StreamToLines(pipeRead, func(line []byte) {
 			lineStr := string(line)
 			if !strings.HasSuffix(lineStr, "\n") {

pkg/remote/connutil.go

@@ -16,6 +16,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"golang.org/x/crypto/ssh"
 )
 
@@ -288,6 +289,7 @@ func CpHostToRemote(client *ssh.Client, sourcePath string, destPath string) erro
 	}
 
 	go func() {
+		defer panichandler.PanicHandler("connutil:CpHostToRemote")
 		io.Copy(installStdin, input)
 		session.Close() // this allows the command to complete for reasons i don't fully understand
 	}()

pkg/service/objectservice/objectservice.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta"
 	"github.com/wavetermdev/waveterm/pkg/waveobj"
 	"github.com/wavetermdev/waveterm/pkg/wcore"
@@ -94,6 +95,7 @@ func (svc *ObjectService) AddTabToWorkspace(windowId string, tabName string, act
 	}
 	updates := waveobj.ContextGetUpdatesRtn(ctx)
 	go func() {
+		defer panichandler.PanicHandler("ObjectService:AddTabToWorkspace:SendUpdateEvents")
 		wps.Broker.SendUpdateEvents(updates)
 	}()
 	return tabId, updates, nil
@@ -142,6 +144,7 @@ func (svc *ObjectService) SetActiveTab(windowId string, tabId string) (waveobj.U
 	}
 	updates := waveobj.ContextGetUpdatesRtn(ctx)
 	go func() {
+		defer panichandler.PanicHandler("ObjectService:SetActiveTab:SendUpdateEvents")
 		wps.Broker.SendUpdateEvents(updates)
 	}()
 	var extraUpdates waveobj.UpdatesRtnType

pkg/service/windowservice/windowservice.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/wavetermdev/waveterm/pkg/blockcontroller"
 	"github.com/wavetermdev/waveterm/pkg/eventbus"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta"
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
 	"github.com/wavetermdev/waveterm/pkg/waveobj"
@@ -75,6 +76,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s
 		}
 	}
 	go func() {
+		defer panichandler.PanicHandler("WindowService:CloseTab:StopBlockControllers")
 		for _, blockId := range tab.BlockIds {
 			blockcontroller.StopBlockController(blockId)
 		}
@@ -107,6 +109,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s
 	}
 	updates := waveobj.ContextGetUpdatesRtn(ctx)
 	go func() {
+		defer panichandler.PanicHandler("WindowService:CloseTab:SendUpdateEvents")
 		wps.Broker.SendUpdateEvents(updates)
 	}()
 	return rtn, updates, nil

pkg/shellexec/conninterface.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/creack/pty"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/wsl"
 	"golang.org/x/crypto/ssh"
 )
@@ -51,6 +52,7 @@ func (cw CmdWrap) KillGraceful(timeout time.Duration) {
 		cw.Cmd.Process.Signal(syscall.SIGTERM)
 	}
 	go func() {
+		defer panichandler.PanicHandler("KillGraceful:Kill")
 		time.Sleep(timeout)
 		if cw.Cmd.ProcessState == nil || !cw.Cmd.ProcessState.Exited() {
 			cw.Cmd.Process.Kill() // force kill if it is already not exited
@@ -159,6 +161,7 @@ func (wcw WslCmdWrap) KillGraceful(timeout time.Duration) {
 	}
 	process.Signal(os.Interrupt)
 	go func() {
+		defer panichandler.PanicHandler("KillGraceful-wsl:Kill")
 		time.Sleep(timeout)
 		process := wcw.WslCmd.GetProcess()
 		processState := wcw.WslCmd.GetProcessState()

pkg/shellexec/shellexec.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/creack/pty"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote"
 	"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
 	"github.com/wavetermdev/waveterm/pkg/util/shellutil"
@@ -51,6 +52,7 @@ type ShellProc struct {
 func (sp *ShellProc) Close() {
 	sp.Cmd.KillGraceful(DefaultGracefulKillWait)
 	go func() {
+		defer panichandler.PanicHandler("ShellProc.Close")
 		waitErr := sp.Cmd.Wait()
 		sp.SetWaitErrorAndSignalDone(waitErr)
 
@@ -450,6 +452,7 @@ func RunSimpleCmdInPty(ecmd *exec.Cmd, termSize waveobj.TermSize) ([]byte, error
 	ioDone := make(chan bool)
 	var outputBuf bytes.Buffer
 	go func() {
+		panichandler.PanicHandler("RunSimpleCmdInPty:ioCopy")
 		// ignore error (/dev/ptmx has read error when process is done)
 		defer close(ioDone)
 		io.Copy(&outputBuf, cmdPty)

pkg/telemetry/telemetry.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"time"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/daystr"
 	"github.com/wavetermdev/waveterm/pkg/util/dbutil"
 	"github.com/wavetermdev/waveterm/pkg/wavebase"
@@ -36,6 +37,7 @@ type ActivityUpdate struct {
 	NumSSHConn    int                   `json:"numsshconn,omitempty"`
 	NumWSLConn    int                   `json:"numwslconn,omitempty"`
 	NumMagnify    int                   `json:"nummagnify,omitempty"`
+	NumPanics     int                   `json:"numpanics,omitempty"`
 	Startup       int                   `json:"startup,omitempty"`
 	Shutdown      int                   `json:"shutdown,omitempty"`
 	SetTabTheme   int                   `json:"settabtheme,omitempty"`
@@ -71,6 +73,7 @@ type TelemetryData struct {
 	NewTab        int                   `json:"newtab"`
 	NumStartup    int                   `json:"numstartup,omitempty"`
 	NumShutdown   int                   `json:"numshutdown,omitempty"`
+	NumPanics     int                   `json:"numpanics,omitempty"`
 	SetTabTheme   int                   `json:"settabtheme,omitempty"`
 	Displays      []ActivityDisplayType `json:"displays,omitempty"`
 	Renderers     map[string]int        `json:"renderers,omitempty"`
@@ -104,6 +107,7 @@ func AutoUpdateChannel() string {
 // Wraps UpdateCurrentActivity, spawns goroutine, and logs errors
 func GoUpdateActivityWrap(update ActivityUpdate, debugStr string) {
 	go func() {
+		defer panichandler.PanicHandlerNoTelemetry("GoUpdateActivityWrap")
 		ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
 		defer cancelFn()
 		err := UpdateActivity(ctx, update)
@@ -138,6 +142,7 @@ func UpdateActivity(ctx context.Context, update ActivityUpdate) error {
 		tdata.NumShutdown += update.Shutdown
 		tdata.SetTabTheme += update.SetTabTheme
 		tdata.NumMagnify += update.NumMagnify
+		tdata.NumPanics += update.NumPanics
 		if update.NumTabs > 0 {
 			tdata.NumTabs = update.NumTabs
 		}

pkg/waveai/anthropicbackend.go

@@ -10,11 +10,10 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
-	"runtime/debug"
 	"strings"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 )
 
@@ -115,17 +114,10 @@ func (AnthropicBackend) StreamCompletion(ctx context.Context, request wshrpc.Ope
 
 	go func() {
 		defer func() {
-			if r := recover(); r != nil {
+			panicErr := panichandler.PanicHandler("AnthropicBackend.StreamCompletion")
-				// Convert panic to error and send it
+			if panicErr != nil {
-				log.Printf("panic: %v\n", r)
+				rtn <- makeAIError(panicErr)
-				debug.PrintStack()
-				err, ok := r.(error)
-				if !ok {
-					err = fmt.Errorf("anthropic backend panic: %v", r)
 			}
-				rtn <- makeAIError(err)
-			}
-			// Always close the channel
 			close(rtn)
 		}()
 

pkg/waveai/cloudbackend.go

@@ -11,6 +11,7 @@ import (
 	"log"
 
 	"github.com/gorilla/websocket"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/wcloud"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 )
@@ -37,7 +38,13 @@ func (WaveAICloudBackend) StreamCompletion(ctx context.Context, request wshrpc.O
 	rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.OpenAIPacketType])
 	wsEndpoint := wcloud.GetWSEndpoint()
 	go func() {
-		defer close(rtn)
+		defer func() {
+			panicErr := panichandler.PanicHandler("WaveAICloudBackend.StreamCompletion")
+			if panicErr != nil {
+				rtn <- makeAIError(panicErr)
+			}
+			close(rtn)
+		}()
 		if wsEndpoint == "" {
 			rtn <- makeAIError(fmt.Errorf("no cloud ws endpoint found"))
 			return

pkg/waveai/openaibackend.go

@@ -8,12 +8,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"regexp"
-	"runtime/debug"
 	"strings"
 
 	openaiapi "github.com/sashabaranov/go-openai"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 )
 
@@ -75,17 +74,10 @@ func (OpenAIBackend) StreamCompletion(ctx context.Context, request wshrpc.OpenAi
 	rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.OpenAIPacketType])
 	go func() {
 		defer func() {
-			if r := recover(); r != nil {
+			panicErr := panichandler.PanicHandler("OpenAIBackend.StreamCompletion")
-				// Convert panic to error and send it
+			if panicErr != nil {
-				log.Printf("panic: %v\n", r)
+				rtn <- makeAIError(panicErr)
-				debug.PrintStack()
-				err, ok := r.(error)
-				if !ok {
-					err = fmt.Errorf("openai backend panic: %v", r)
 			}
-				rtn <- makeAIError(err)
-			}
-			// Always close the channel
 			close(rtn)
 		}()
 		if request.Opts == nil {

pkg/waveapp/waveappserverimpl.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"net/http"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/vdom"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 )
@@ -23,12 +24,11 @@ func (*WaveAppServerImpl) WshServerImpl() {}
 
 func (impl *WaveAppServerImpl) VDomRenderCommand(ctx context.Context, feUpdate vdom.VDomFrontendUpdate) chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate] {
 	respChan := make(chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate], 5)
-
 	defer func() {
-		if r := recover(); r != nil {
+		panicErr := panichandler.PanicHandler("VDomRenderCommand")
-			log.Printf("panic in VDomRenderCommand: %v\n", r)
+		if panicErr != nil {
 			respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{
-				Error: fmt.Errorf("internal error: %v", r),
+				Error: panicErr,
 			}
 			close(respChan)
 		}
@@ -88,6 +88,7 @@ func (impl *WaveAppServerImpl) VDomRenderCommand(ctx context.Context, feUpdate v
 	// Split the update into chunks and send them sequentially
 	updates := vdom.SplitBackendUpdate(update)
 	go func() {
+		defer panichandler.PanicHandler("VDomRenderCommand:splitUpdates")
 		defer close(respChan)
 		for _, splitUpdate := range updates {
 			respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{
@@ -108,9 +109,10 @@ func (impl *WaveAppServerImpl) VDomUrlRequestCommand(ctx context.Context, data w
 		defer writer.Close()  // Ensures writer is closed before the channel is closed
 
 		defer func() {
-			if r := recover(); r != nil {
+			panicErr := panichandler.PanicHandler("VDomUrlRequestCommand")
+			if panicErr != nil {
 				writer.WriteHeader(http.StatusInternalServerError)
-				writer.Write([]byte(fmt.Sprintf("internal server error: %v", r)))
+				writer.Write([]byte(fmt.Sprintf("internal server error: %v", panicErr)))
 			}
 		}()
 

pkg/wconfig/filewatcher.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 
 	"github.com/fsnotify/fsnotify"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/wavebase"
 	"github.com/wavetermdev/waveterm/pkg/wps"
 )
@@ -64,6 +65,7 @@ func (w *Watcher) Start() {
 	w.sendInitialValues()
 
 	go func() {
+		defer panichandler.PanicHandler("filewatcher:Start")
 		for {
 			select {
 			case event, ok := <-w.watcher.Events:

pkg/wcore/wcore.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/wavetermdev/waveterm/pkg/blockcontroller"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/pkg/waveobj"
 	"github.com/wavetermdev/waveterm/pkg/wps"
@@ -267,6 +268,7 @@ func CreateBlock(ctx context.Context, tabId string, blockDef *waveobj.BlockDef,
 		return nil, fmt.Errorf("error creating block: %w", err)
 	}
 	go func() {
+		defer panichandler.PanicHandler("CreateBlock:telemetry")
 		blockView := blockDef.Meta.GetString(waveobj.MetaKey_View, "")
 		if blockView == "" {
 			return

pkg/web/web.go

@@ -15,7 +15,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"runtime/debug"
 	"strconv"
 	"time"
 
@@ -25,6 +24,7 @@ import (
 	"github.com/wavetermdev/waveterm/pkg/authkey"
 	"github.com/wavetermdev/waveterm/pkg/docsite"
 	"github.com/wavetermdev/waveterm/pkg/filestore"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/service"
 	"github.com/wavetermdev/waveterm/pkg/wavebase"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
@@ -358,21 +358,18 @@ type ClientActiveState struct {
 func WebFnWrap(opts WebFnOpts, fn WebFnType) WebFnType {
 	return func(w http.ResponseWriter, r *http.Request) {
 		defer func() {
-			recErr := recover()
+			recErr := panichandler.PanicHandler("WebFnWrap")
 			if recErr == nil {
 				return
 			}
-			panicStr := fmt.Sprintf("panic: %v", recErr)
-			log.Printf("panic: %v\n", recErr)
-			debug.PrintStack()
 			if opts.JsonErrors {
-				jsonRtn := marshalReturnValue(nil, fmt.Errorf(panicStr))
+				jsonRtn := marshalReturnValue(nil, recErr)
 				w.Header().Set(ContentTypeHeaderKey, ContentTypeJson)
 				w.Header().Set(ContentLengthHeaderKey, fmt.Sprintf("%d", len(jsonRtn)))
 				w.WriteHeader(http.StatusOK)
 				w.Write(jsonRtn)
 			} else {
-				http.Error(w, panicStr, http.StatusInternalServerError)
+				http.Error(w, recErr.Error(), http.StatusInternalServerError)
 			}
 		}()
 		if !opts.AllowCaching {

pkg/web/ws.go

@@ -9,7 +9,6 @@ import (
 	"log"
 	"net"
 	"net/http"
-	"runtime/debug"
 	"sync"
 	"time"
 
@@ -18,6 +17,7 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/wavetermdev/waveterm/pkg/authkey"
 	"github.com/wavetermdev/waveterm/pkg/eventbus"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/web/webcmd"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 	"github.com/wavetermdev/waveterm/pkg/wshutil"
@@ -81,11 +81,9 @@ func getStringFromMap(jmsg map[string]any, key string) string {
 func processWSCommand(jmsg map[string]any, outputCh chan any, rpcInputCh chan []byte) {
 	var rtnErr error
 	defer func() {
-		r := recover()
+		panicErr := panichandler.PanicHandler("processWSCommand")
-		if r != nil {
+		if panicErr != nil {
-			rtnErr = fmt.Errorf("panic: %v", r)
+			rtnErr = panicErr
-			log.Printf("[websocket] panic in processMessage: %v\n", r)
-			debug.PrintStack()
 		}
 		if rtnErr == nil {
 			return
@@ -304,6 +302,7 @@ func HandleWsInternal(w http.ResponseWriter, r *http.Request) error {
 	wg := &sync.WaitGroup{}
 	wg.Add(2)
 	go func() {
+		defer panichandler.PanicHandler("HandleWsInternal:outputCh")
 		// no waitgroup add here
 		// move values from rpcOutputCh to outputCh
 		for msgBytes := range wproxy.ToRemoteCh {
@@ -315,11 +314,13 @@ func HandleWsInternal(w http.ResponseWriter, r *http.Request) error {
 		}
 	}()
 	go func() {
+		defer panichandler.PanicHandler("HandleWsInternal:ReadLoop")
 		// read loop
 		defer wg.Done()
 		ReadLoop(conn, outputCh, closeCh, wproxy.FromRemoteCh, routeId)
 	}()
 	go func() {
+		defer panichandler.PanicHandler("HandleWsInternal:WriteLoop")
 		// write loop
 		defer wg.Done()
 		WriteLoop(conn, outputCh, closeCh, routeId)

pkg/wshrpc/wshclient/wshclientutil.go

@@ -6,6 +6,7 @@ package wshclient
 import (
 	"errors"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 	"github.com/wavetermdev/waveterm/pkg/wshutil"
@@ -39,6 +40,7 @@ func sendRpcRequestCallHelper[T any](w *wshutil.WshRpc, command string, data int
 
 func rtnErr[T any](ch chan wshrpc.RespOrErrorUnion[T], err error) {
 	go func() {
+		defer panichandler.PanicHandler("wshclientutil:rtnErr")
 		ch <- wshrpc.RespOrErrorUnion[T]{Error: err}
 		close(ch)
 	}()
@@ -63,6 +65,7 @@ func sendRpcRequestResponseStreamHelper[T any](w *wshutil.WshRpc, command string
 		reqHandler.SendCancel()
 	}
 	go func() {
+		defer panichandler.PanicHandler("sendRpcRequestResponseStreamHelper")
 		defer close(respChan)
 		for {
 			if reqHandler.ResponseDone() {

pkg/wshrpc/wshserver/wshserver.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/wavetermdev/waveterm/pkg/blockcontroller"
 	"github.com/wavetermdev/waveterm/pkg/filestore"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/remote"
 	"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
 	"github.com/wavetermdev/waveterm/pkg/telemetry"
@@ -45,11 +46,7 @@ func (*WshServer) WshServerImpl() {}
 var WshServerImpl = WshServer{}
 
 func (ws *WshServer) TestCommand(ctx context.Context, data string) error {
-	defer func() {
+	defer panichandler.PanicHandler("TestCommand")
-		if r := recover(); r != nil {
-			log.Printf("panic in TestCommand: %v", r)
-		}
-	}()
 	rpcSource := wshutil.GetRpcSourceFromContext(ctx)
 	log.Printf("TEST src:%s | %s\n", rpcSource, data)
 	return nil
@@ -65,6 +62,7 @@ func (ws *WshServer) MessageCommand(ctx context.Context, data wshrpc.CommandMess
 func (ws *WshServer) StreamTestCommand(ctx context.Context) chan wshrpc.RespOrErrorUnion[int] {
 	rtn := make(chan wshrpc.RespOrErrorUnion[int])
 	go func() {
+		defer panichandler.PanicHandler("StreamTestCommand")
 		for i := 1; i <= 5; i++ {
 			rtn <- wshrpc.RespOrErrorUnion[int]{Response: i}
 			time.Sleep(1 * time.Second)

pkg/wshutil/wshadapter.go

@@ -8,6 +8,7 @@ import (
 	"reflect"
 	"strings"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 )
@@ -131,6 +132,7 @@ func serverImplAdapter(impl any) func(*RpcResponseHandler) bool {
 				return true
 			}
 			go func() {
+				defer panichandler.PanicHandler("serverImplAdapter:responseStream")
 				defer handler.Finalize()
 				// must use reflection here because we don't know the generic type of RespOrErrorUnion
 				for {

pkg/wshutil/wshmultiproxy.go

@@ -9,6 +9,7 @@ import (
 	"sync"
 
 	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
 )
 
@@ -112,6 +113,7 @@ func (p *WshRpcMultiProxy) handleUnauthMessage(msgBytes []byte) {
 		p.setRouteInfo(routeInfo.AuthToken, routeInfo)
 		p.sendAuthResponse(msg, routeId, routeInfo.AuthToken)
 		go func() {
+			defer panichandler.PanicHandler("WshRpcMultiProxy:handleUnauthMessage")
 			for msgBytes := range routeInfo.Proxy.ToRemoteCh {
 				p.ToRemoteCh <- msgBytes
 			}

pkg/wshutil/wshrouter.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
 	"github.com/wavetermdev/waveterm/pkg/wps"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
@@ -91,6 +92,7 @@ func noRouteErr(routeId string) error {
 }
 
 func (router *WshRouter) SendEvent(routeId string, event wps.WaveEvent) {
+	defer panichandler.PanicHandler("WshRouter.SendEvent")
 	rpc := router.GetRpc(routeId)
 	if rpc == nil {
 		return
@@ -296,6 +298,7 @@ func (router *WshRouter) RegisterRoute(routeId string, rpc AbstractRpcClient, sh
 	}
 	router.RouteMap[routeId] = rpc
 	go func() {
+		defer panichandler.PanicHandler("WshRouter:registerRoute:recvloop")
 		// announce
 		if shouldAnnounce && !alreadyExists && router.GetUpstreamClient() != nil {
 			announceMsg := RpcMessage{Command: wshrpc.Command_RouteAnnounce, Source: routeId}
@@ -341,6 +344,7 @@ func (router *WshRouter) UnregisterRoute(routeId string) {
 		}
 	}
 	go func() {
+		defer panichandler.PanicHandler("WshRouter:unregisterRoute:routegone")
 		wps.Broker.UnsubscribeAll(routeId)
 		wps.Broker.Publish(wps.WaveEvent{Event: wps.Event_RouteGone, Scopes: []string{routeId}})
 	}()

pkg/wshutil/wshrpc.go

@@ -10,12 +10,12 @@ import (
 	"fmt"
 	"log"
 	"reflect"
-	"runtime/debug"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
 	"github.com/wavetermdev/waveterm/pkg/wps"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
@@ -281,15 +281,6 @@ func (w *WshRpc) handleRequest(req *RpcMessage) {
 	}
 
 	var respHandler *RpcResponseHandler
-	defer func() {
-		if r := recover(); r != nil {
-			log.Printf("panic in handleRequest: %v\n", r)
-			debug.PrintStack()
-			if respHandler != nil {
-				respHandler.SendResponseError(fmt.Errorf("panic: %v", r))
-			}
-		}
-	}()
 	timeoutMs := req.Timeout
 	if timeoutMs <= 0 {
 		timeoutMs = DefaultTimeoutMs
@@ -313,13 +304,13 @@ func (w *WshRpc) handleRequest(req *RpcMessage) {
 	w.registerResponseHandler(req.ReqId, respHandler)
 	isAsync := false
 	defer func() {
-		if r := recover(); r != nil {
+		panicErr := panichandler.PanicHandler("handleRequest")
-			log.Printf("panic in handleRequest: %v\n", r)
+		if panicErr != nil {
-			debug.PrintStack()
+			respHandler.SendResponseError(panicErr)
-			respHandler.SendResponseError(fmt.Errorf("panic: %v", r))
 		}
 		if isAsync {
 			go func() {
+				defer panichandler.PanicHandler("handleRequest:finalize")
 				<-ctx.Done()
 				respHandler.Finalize()
 			}()
@@ -394,6 +385,7 @@ func (w *WshRpc) registerRpc(ctx context.Context, reqId string) chan *RpcMessage
 		Ctx:   ctx,
 	}
 	go func() {
+		defer panichandler.PanicHandler("registerRpc:timeout")
 		<-ctx.Done()
 		w.unregisterRpc(reqId, fmt.Errorf("EC-TIME: timeout waiting for response"))
 	}()
@@ -463,12 +455,7 @@ func (handler *RpcRequestHandler) Context() context.Context {
 }
 
 func (handler *RpcRequestHandler) SendCancel() {
-	defer func() {
+	defer panichandler.PanicHandler("SendCancel")
-		if r := recover(); r != nil {
-			// this is likely a write to closed channel
-			log.Printf("panic in SendCancel: %v\n", r)
-		}
-	}()
 	msg := &RpcMessage{
 		Cancel:    true,
 		ReqId:     handler.reqId,
@@ -573,13 +560,7 @@ func (handler *RpcResponseHandler) SendMessage(msg string) {
 }
 
 func (handler *RpcResponseHandler) SendResponse(data any, done bool) error {
-	defer func() {
+	defer panichandler.PanicHandler("SendResponse")
-		if r := recover(); r != nil {
-			// this is likely a write to closed channel
-			log.Printf("panic in SendResponse: %v\n", r)
-			handler.close()
-		}
-	}()
 	if handler.reqId == "" {
 		return nil // no response expected
 	}
@@ -604,13 +585,7 @@ func (handler *RpcResponseHandler) SendResponse(data any, done bool) error {
 }
 
 func (handler *RpcResponseHandler) SendResponseError(err error) {
-	defer func() {
+	defer panichandler.PanicHandler("SendResponseError")
-		if r := recover(); r != nil {
-			// this is likely a write to closed channel
-			log.Printf("panic in SendResponseError: %v\n", r)
-			handler.close()
-		}
-	}()
 	if handler.reqId == "" || handler.done.Load() {
 		return
 	}
@@ -659,12 +634,7 @@ func (w *WshRpc) SendComplexRequest(command string, data any, opts *wshrpc.RpcOp
 	if timeoutMs <= 0 {
 		timeoutMs = DefaultTimeoutMs
 	}
-	defer func() {
+	defer panichandler.PanicHandler("SendComplexRequest")
-		if r := recover(); r != nil {
-			log.Printf("panic in SendComplexRequest: %v\n", r)
-			rtnErr = fmt.Errorf("panic: %v", r)
-		}
-	}()
 	if command == "" {
 		return nil, fmt.Errorf("command cannot be empty")
 	}

pkg/wshutil/wshutil.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/golang-jwt/jwt/v5"
 	"github.com/google/uuid"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/packetparser"
 	"github.com/wavetermdev/waveterm/pkg/wavebase"
 	"github.com/wavetermdev/waveterm/pkg/wshrpc"
@@ -150,6 +151,7 @@ func installShutdownSignalHandlers(quiet bool) {
 	sigCh := make(chan os.Signal, 1)
 	signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
 	go func() {
+		defer panichandler.PanicHandlerNoTelemetry("installShutdownSignalHandlers")
 		for sig := range sigCh {
 			DoShutdown(fmt.Sprintf("got signal %v", sig), 1, quiet)
 			break
@@ -198,6 +200,7 @@ func SetupTerminalRpcClient(serverImpl ServerImpl) (*WshRpc, io.Reader) {
 	ptyBuf := MakePtyBuffer(WaveServerOSCPrefix, os.Stdin, messageCh)
 	rpcClient := MakeWshRpc(messageCh, outputCh, wshrpc.RpcContext{}, serverImpl)
 	go func() {
+		defer panichandler.PanicHandler("SetupTerminalRpcClient")
 		for msg := range outputCh {
 			barr, err := EncodeWaveOSCBytes(WaveOSC, msg)
 			if err != nil {
@@ -218,6 +221,7 @@ func SetupPacketRpcClient(input io.Reader, output io.Writer, serverImpl ServerIm
 	rpcClient := MakeWshRpc(messageCh, outputCh, wshrpc.RpcContext{}, serverImpl)
 	go packetparser.Parse(input, messageCh, rawCh)
 	go func() {
+		defer panichandler.PanicHandler("SetupPacketRpcClient:outputloop")
 		for msg := range outputCh {
 			packetparser.WritePacket(output, msg)
 		}
@@ -230,6 +234,7 @@ func SetupConnRpcClient(conn net.Conn, serverImpl ServerImpl) (*WshRpc, chan err
 	outputCh := make(chan []byte, DefaultOutputChSize)
 	writeErrCh := make(chan error, 1)
 	go func() {
+		defer panichandler.PanicHandler("SetupConnRpcClient:AdaptOutputChToStream")
 		writeErr := AdaptOutputChToStream(outputCh, conn)
 		if writeErr != nil {
 			writeErrCh <- writeErr
@@ -237,6 +242,7 @@ func SetupConnRpcClient(conn net.Conn, serverImpl ServerImpl) (*WshRpc, chan err
 		}
 	}()
 	go func() {
+		defer panichandler.PanicHandler("SetupConnRpcClient:AdaptStreamToMsgCh")
 		// when input is closed, close the connection
 		defer conn.Close()
 		AdaptStreamToMsgCh(conn, inputCh)
@@ -264,6 +270,7 @@ func SetupDomainSocketRpcClient(sockName string, serverImpl ServerImpl) (*WshRpc
 	}
 	rtn, errCh, err := SetupConnRpcClient(conn, serverImpl)
 	go func() {
+		defer panichandler.PanicHandler("SetupDomainSocketRpcClient:closeConn")
 		defer conn.Close()
 		err := <-errCh
 		if err != nil && err != io.EOF {
@@ -410,9 +417,11 @@ func HandleStdIOClient(logName string, input io.Reader, output io.Writer) {
 		proxy.DisposeRoutes()
 	}
 	go func() {
+		defer panichandler.PanicHandler("HandleStdIOClient:RunUnauthLoop")
 		proxy.RunUnauthLoop()
 	}()
 	go func() {
+		defer panichandler.PanicHandler("HandleStdIOClient:ToRemoteChLoop")
 		defer closeDoneCh()
 		for msg := range proxy.ToRemoteCh {
 			err := packetparser.WritePacket(output, msg)
@@ -423,6 +432,7 @@ func HandleStdIOClient(logName string, input io.Reader, output io.Writer) {
 		}
 	}()
 	go func() {
+		defer panichandler.PanicHandler("HandleStdIOClient:RawChLoop")
 		defer closeDoneCh()
 		for msg := range rawCh {
 			log.Printf("[%s:stdout] %s", logName, msg)
@@ -435,6 +445,7 @@ func handleDomainSocketClient(conn net.Conn) {
 	var routeIdContainer atomic.Pointer[string]
 	proxy := MakeRpcProxy()
 	go func() {
+		defer panichandler.PanicHandler("handleDomainSocketClient:AdaptOutputChToStream")
 		writeErr := AdaptOutputChToStream(proxy.ToRemoteCh, conn)
 		if writeErr != nil {
 			log.Printf("error writing to domain socket: %v\n", writeErr)
@@ -442,6 +453,7 @@ func handleDomainSocketClient(conn net.Conn) {
 	}()
 	go func() {
 		// when input is closed, close the connection
+		defer panichandler.PanicHandler("handleDomainSocketClient:AdaptStreamToMsgCh")
 		defer func() {
 			conn.Close()
 			routeIdPtr := routeIdContainer.Load()

pkg/wsl/wsl-util.go

@@ -15,6 +15,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 )
 
 func DetectShell(ctx context.Context, client *Distro) (string, error) {
@@ -233,6 +235,7 @@ func CpHostToRemote(ctx context.Context, client *Distro, sourcePath string, dest
 		return fmt.Errorf("cannot open local file %s to send to host: %v", sourcePath, err)
 	}
 	go func() {
+		defer panichandler.PanicHandler("wslutil:cpHostToRemote:catStdin")
 		io.Copy(catStdin, input)
 		installStepCmds["cat"].Cancel()
 

pkg/wsl/wsl.go

@@ -13,6 +13,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/telemetry"
 	"github.com/wavetermdev/waveterm/pkg/userinput"
 	"github.com/wavetermdev/waveterm/pkg/util/shellutil"
@@ -234,6 +235,7 @@ func (conn *WslConn) StartConnServer() error {
 	})
 	// service the I/O
 	go func() {
+		defer panichandler.PanicHandler("wsl:StartConnServer:wait")
 		// wait for termination, clear the controller
 		defer conn.WithLock(func() {
 			conn.ConnController = nil
@@ -242,6 +244,7 @@ func (conn *WslConn) StartConnServer() error {
 		log.Printf("conn controller (%q) terminated: %v", conn.GetName(), waitErr)
 	}()
 	go func() {
+		defer panichandler.PanicHandler("wsl:StartConnServer:handleStdIOClient")
 		logName := fmt.Sprintf("conncontroller:%s", conn.GetName())
 		wshutil.HandleStdIOClient(logName, pipeRead, inputPipeWrite)
 	}()

pkg/wstore/wstore_dbops.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/wavetermdev/waveterm/pkg/filestore"
+	"github.com/wavetermdev/waveterm/pkg/panichandler"
 	"github.com/wavetermdev/waveterm/pkg/util/dbutil"
 	"github.com/wavetermdev/waveterm/pkg/waveobj"
 )
@@ -211,6 +212,7 @@ func DBDelete(ctx context.Context, otype string, id string) error {
 		return err
 	}
 	go func() {
+		defer panichandler.PanicHandler("DBDelete:filestore.DeleteZone")
 		// we spawn a go routine here because we don't want to reuse the DB connection
 		// since DBDelete is called in a transaction from DeleteTab
 		deleteCtx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)