Upstream/waveterm
1
0
Fork 0
mirror of https://github.com/wavetermdev/waveterm.git synced 2025-01-21 21:32:13 +01:00
Code Issues Packages Projects Releases Wiki Activity

panic handlers everywhere (#1327)

Browse Source
This commit is contained in:
Mike Sawka 2024-11-20 18:05:13 -08:00 committed by GitHub
parent 812732673c
commit 360964d4ba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 173 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
cmd/server/main-server.go
View File

@ -9,7 +9,6 @@ import (
"log"
"os"
"os/signal"
"runtime/debug"
"runtime"
"sync"
@ -19,6 +18,7 @@ import (
"github.com/wavetermdev/waveterm/pkg/authkey"
"github.com/wavetermdev/waveterm/pkg/blockcontroller"
"github.com/wavetermdev/waveterm/pkg/filestore"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
"github.com/wavetermdev/waveterm/pkg/service"
"github.com/wavetermdev/waveterm/pkg/telemetry"
@ -111,15 +111,16 @@ func telemetryLoop() {
}
}
func panicTelemetryHandler() {
activity := telemetry.ActivityUpdate{NumPanics: 1}
err := telemetry.UpdateActivity(context.Background(), activity)
if err != nil {
log.Printf("error updating activity (panicTelemetryHandler): %v\n", err)
}
}
func sendTelemetryWrapper() {
defer func() {
r := recover()
if r == nil {
return
}
log.Printf("[error] in sendTelemetryWrapper: %v\n", r)
debug.PrintStack()
}()
defer panichandler.PanicHandler("sendTelemetryWrapper")
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
defer cancelFn()
beforeSendActivityUpdate(ctx)
@ -254,7 +255,9 @@ func main() {
log.Printf("error initializing wstore: %v\n", err)
return
}
panichandler.PanicTelemetryHandler = panicTelemetryHandler
go func() {
defer panichandler.PanicHandler("InitCustomShellStartupFiles")
err := shellutil.InitCustomShellStartupFiles()
if err != nil {
log.Printf("error initializing wsh and shell-integration files: %v\n", err)

4
cmd/wsh/cmd/wshcmd-connserver.go
View File

@ -14,6 +14,7 @@ import (
"time"
"github.com/spf13/cobra"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/packetparser"
"github.com/wavetermdev/waveterm/pkg/wavebase"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
@ -53,6 +54,7 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) {
var routeIdContainer atomic.Pointer[string]
proxy := wshutil.MakeRpcProxy()
go func() {
defer panichandler.PanicHandler("handleNewListenerConn:AdaptOutputChToStream")
writeErr := wshutil.AdaptOutputChToStream(proxy.ToRemoteCh, conn)
if writeErr != nil {
log.Printf("error writing to domain socket: %v\n", writeErr)
@ -60,6 +62,7 @@ func handleNewListenerConn(conn net.Conn, router *wshutil.WshRouter) {
}()
go func() {
// when input is closed, close the connection
defer panichandler.PanicHandler("handleNewListenerConn:AdaptStreamToMsgCh")
defer func() {
conn.Close()
routeIdPtr := routeIdContainer.Load()
@ -136,6 +139,7 @@ func serverRunRouter() error {
rawCh := make(chan []byte, wshutil.DefaultOutputChSize)
go packetparser.Parse(os.Stdin, termProxy.FromRemoteCh, rawCh)
go func() {
defer panichandler.PanicHandler("serverRunRouter:WritePackets")
for msg := range termProxy.ToRemoteCh {
packetparser.WritePacket(os.Stdout, msg)
}

6
pkg/blockcontroller/blockcontroller.go
View File

@ -16,6 +16,7 @@ import (
"time"
"github.com/wavetermdev/waveterm/pkg/filestore"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/remote"
"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
"github.com/wavetermdev/waveterm/pkg/shellexec"
@ -354,6 +355,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
ptyBuffer := wshutil.MakePtyBuffer(wshutil.WaveOSCPrefix, bc.ShellProc.Cmd, wshProxy.FromRemoteCh)
go func() {
// handles regular output from the pty (goes to the blockfile and xterm)
defer panichandler.PanicHandler("blockcontroller:shellproc-pty-read-loop")
defer func() {
log.Printf("[shellproc] pty-read loop done\n")
bc.ShellProc.Close()
@ -386,6 +388,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
go func() {
// handles input from the shellInputCh, sent to pty
// use shellInputCh instead of bc.ShellInputCh (because we want to be attached to *this* ch. bc.ShellInputCh can be updated)
defer panichandler.PanicHandler("blockcontroller:shellproc-input-loop")
for ic := range shellInputCh {
if len(ic.InputData) > 0 {
bc.ShellProc.Cmd.Write(ic.InputData)
@ -403,6 +406,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
}
}()
go func() {
defer panichandler.PanicHandler("blockcontroller:shellproc-output-loop")
// handles outputCh -> shellInputCh
for msg := range wshProxy.ToRemoteCh {
encodedMsg, err := wshutil.EncodeWaveOSCBytes(wshutil.WaveServerOSC, msg)
@ -413,6 +417,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts, blockMeta waveobj
}
}()
go func() {
defer panichandler.PanicHandler("blockcontroller:shellproc-wait-loop")
// wait for the shell to finish
defer func() {
wshutil.DefaultRouter.UnregisterRoute(wshutil.MakeControllerRouteId(bc.BlockId))
@ -484,6 +489,7 @@ func (bc *BlockController) run(bdata *waveobj.Block, blockMeta map[string]any, r
runOnStart := getBoolFromMeta(blockMeta, waveobj.MetaKey_CmdRunOnStart, true)
if runOnStart {
go func() {
defer panichandler.PanicHandler("blockcontroller:run-shell-command")
var termSize waveobj.TermSize
if rtOpts != nil {
termSize = rtOpts.TermSize

9
pkg/filestore/blockstore.go
View File

@ -12,12 +12,12 @@ import (
"fmt"
"io/fs"
"log"
"runtime/debug"
"sync"
"sync/atomic"
"time"
"github.com/wavetermdev/waveterm/pkg/ijson"
"github.com/wavetermdev/waveterm/pkg/panichandler"
)
const (
@ -514,12 +514,7 @@ func (s *FileStore) runFlushWithNewContext() (FlushStats, error) {
}
func (s *FileStore) runFlusher() {
defer func() {
if r := recover(); r != nil {
log.Printf("panic in filestore flusher: %v\n", r)
debug.PrintStack()
}
}()
defer panichandler.PanicHandler("filestore flusher")
for {
stats, err := s.runFlushWithNewContext()
if err != nil || stats.NumDirtyEntries > 0 {

43
pkg/panichandler/panichandler.go Normal file
View File

@ -0,0 +1,43 @@
// Copyright 2024, Command Line Inc.
// SPDX-License-Identifier: Apache-2.0
package panichandler
import (
"fmt"
"log"
"runtime/debug"
)
// to log NumPanics into the local telemetry system
// gets around import cycles
var PanicTelemetryHandler func()
func PanicHandlerNoTelemetry(debugStr string) {
r := recover()
if r == nil {
return
}
log.Printf("[panic] in %s: %v\n", debugStr, r)
debug.PrintStack()
}
// returns an error (wrapping the panic) if a panic occurred
func PanicHandler(debugStr string) error {
r := recover()
if r == nil {
return nil
}
log.Printf("[panic] in %s: %v\n", debugStr, r)
debug.PrintStack()
if PanicTelemetryHandler != nil {
go func() {
defer PanicHandlerNoTelemetry("PanicTelemetryHandler")
PanicTelemetryHandler()
}()
}
if err, ok := r.(error); ok {
return fmt.Errorf("panic in %s: %w", debugStr, err)
}
return fmt.Errorf("panic in %s: %v", debugStr, r)
}

4
pkg/remote/conncontroller/conncontroller.go
View File

@ -20,6 +20,7 @@ import (
"github.com/kevinburke/ssh_config"
"github.com/skeema/knownhosts"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/remote"
"github.com/wavetermdev/waveterm/pkg/telemetry"
"github.com/wavetermdev/waveterm/pkg/userinput"
@ -193,6 +194,7 @@ func (conn *SSHConn) OpenDomainSocketListener() error {
conn.DomainSockListener = listener
})
go func() {
defer panichandler.PanicHandler("conncontroller:OpenDomainSocketListener")
defer conn.WithLock(func() {
conn.DomainSockListener = nil
conn.SockName = ""
@ -252,6 +254,7 @@ func (conn *SSHConn) StartConnServer() error {
})
// service the I/O
go func() {
defer panichandler.PanicHandler("conncontroller:sshSession.Wait")
// wait for termination, clear the controller
defer conn.WithLock(func() {
conn.ConnController = nil
@ -260,6 +263,7 @@ func (conn *SSHConn) StartConnServer() error {
log.Printf("conn controller (%q) terminated: %v", conn.GetName(), waitErr)
}()
go func() {
defer panichandler.PanicHandler("conncontroller:sshSession-output")
readErr := wshutil.StreamToLines(pipeRead, func(line []byte) {
lineStr := string(line)
if !strings.HasSuffix(lineStr, "\n") {

2
pkg/remote/connutil.go
View File

@ -16,6 +16,7 @@ import (
"strconv"
"strings"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"golang.org/x/crypto/ssh"
)
@ -288,6 +289,7 @@ func CpHostToRemote(client *ssh.Client, sourcePath string, destPath string) erro
}
go func() {
defer panichandler.PanicHandler("connutil:CpHostToRemote")
io.Copy(installStdin, input)
session.Close() // this allows the command to complete for reasons i don't fully understand
}()

3
pkg/service/objectservice/objectservice.go
View File

@ -9,6 +9,7 @@ import (
"strings"
"time"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta"
"github.com/wavetermdev/waveterm/pkg/waveobj"
"github.com/wavetermdev/waveterm/pkg/wcore"
@ -94,6 +95,7 @@ func (svc *ObjectService) AddTabToWorkspace(windowId string, tabName string, act
}
updates := waveobj.ContextGetUpdatesRtn(ctx)
go func() {
defer panichandler.PanicHandler("ObjectService:AddTabToWorkspace:SendUpdateEvents")
wps.Broker.SendUpdateEvents(updates)
}()
return tabId, updates, nil
@ -142,6 +144,7 @@ func (svc *ObjectService) SetActiveTab(windowId string, tabId string) (waveobj.U
}
updates := waveobj.ContextGetUpdatesRtn(ctx)
go func() {
defer panichandler.PanicHandler("ObjectService:SetActiveTab:SendUpdateEvents")
wps.Broker.SendUpdateEvents(updates)
}()
var extraUpdates waveobj.UpdatesRtnType

3
pkg/service/windowservice/windowservice.go
View File

@ -11,6 +11,7 @@ import (
"github.com/wavetermdev/waveterm/pkg/blockcontroller"
"github.com/wavetermdev/waveterm/pkg/eventbus"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta"
"github.com/wavetermdev/waveterm/pkg/util/utilfn"
"github.com/wavetermdev/waveterm/pkg/waveobj"
@ -75,6 +76,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s
}
}
go func() {
defer panichandler.PanicHandler("WindowService:CloseTab:StopBlockControllers")
for _, blockId := range tab.BlockIds {
blockcontroller.StopBlockController(blockId)
}
@ -107,6 +109,7 @@ func (svc *WindowService) CloseTab(ctx context.Context, windowId string, tabId s
}
updates := waveobj.ContextGetUpdatesRtn(ctx)
go func() {
defer panichandler.PanicHandler("WindowService:CloseTab:SendUpdateEvents")
wps.Broker.SendUpdateEvents(updates)
}()
return rtn, updates, nil

3
pkg/shellexec/conninterface.go
View File

@ -9,6 +9,7 @@ import (
"time"
"github.com/creack/pty"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/wsl"
"golang.org/x/crypto/ssh"
)
@ -51,6 +52,7 @@ func (cw CmdWrap) KillGraceful(timeout time.Duration) {
cw.Cmd.Process.Signal(syscall.SIGTERM)
}
go func() {
defer panichandler.PanicHandler("KillGraceful:Kill")
time.Sleep(timeout)
if cw.Cmd.ProcessState == nil || !cw.Cmd.ProcessState.Exited() {
cw.Cmd.Process.Kill() // force kill if it is already not exited
@ -159,6 +161,7 @@ func (wcw WslCmdWrap) KillGraceful(timeout time.Duration) {
}
process.Signal(os.Interrupt)
go func() {
defer panichandler.PanicHandler("KillGraceful-wsl:Kill")
time.Sleep(timeout)
process := wcw.WslCmd.GetProcess()
processState := wcw.WslCmd.GetProcessState()

3
pkg/shellexec/shellexec.go
View File

@ -19,6 +19,7 @@ import (
"time"
"github.com/creack/pty"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/remote"
"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
"github.com/wavetermdev/waveterm/pkg/util/shellutil"
@ -51,6 +52,7 @@ type ShellProc struct {
func (sp *ShellProc) Close() {
sp.Cmd.KillGraceful(DefaultGracefulKillWait)
go func() {
defer panichandler.PanicHandler("ShellProc.Close")
waitErr := sp.Cmd.Wait()
sp.SetWaitErrorAndSignalDone(waitErr)
@ -450,6 +452,7 @@ func RunSimpleCmdInPty(ecmd *exec.Cmd, termSize waveobj.TermSize) ([]byte, error
ioDone := make(chan bool)
var outputBuf bytes.Buffer
go func() {
panichandler.PanicHandler("RunSimpleCmdInPty:ioCopy")
// ignore error (/dev/ptmx has read error when process is done)
defer close(ioDone)
io.Copy(&outputBuf, cmdPty)

5
pkg/telemetry/telemetry.go
View File

@ -9,6 +9,7 @@ import (
"log"
"time"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/daystr"
"github.com/wavetermdev/waveterm/pkg/util/dbutil"
"github.com/wavetermdev/waveterm/pkg/wavebase"
@ -36,6 +37,7 @@ type ActivityUpdate struct {
NumSSHConn int `json:"numsshconn,omitempty"`
NumWSLConn int `json:"numwslconn,omitempty"`
NumMagnify int `json:"nummagnify,omitempty"`
NumPanics int `json:"numpanics,omitempty"`
Startup int `json:"startup,omitempty"`
Shutdown int `json:"shutdown,omitempty"`
SetTabTheme int `json:"settabtheme,omitempty"`
@ -71,6 +73,7 @@ type TelemetryData struct {
NewTab int `json:"newtab"`
NumStartup int `json:"numstartup,omitempty"`
NumShutdown int `json:"numshutdown,omitempty"`
NumPanics int `json:"numpanics,omitempty"`
SetTabTheme int `json:"settabtheme,omitempty"`
Displays []ActivityDisplayType `json:"displays,omitempty"`
Renderers map[string]int `json:"renderers,omitempty"`
@ -104,6 +107,7 @@ func AutoUpdateChannel() string {
// Wraps UpdateCurrentActivity, spawns goroutine, and logs errors
func GoUpdateActivityWrap(update ActivityUpdate, debugStr string) {
go func() {
defer panichandler.PanicHandlerNoTelemetry("GoUpdateActivityWrap")
ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
defer cancelFn()
err := UpdateActivity(ctx, update)
@ -138,6 +142,7 @@ func UpdateActivity(ctx context.Context, update ActivityUpdate) error {
tdata.NumShutdown += update.Shutdown
tdata.SetTabTheme += update.SetTabTheme
tdata.NumMagnify += update.NumMagnify
tdata.NumPanics += update.NumPanics
if update.NumTabs > 0 {
tdata.NumTabs = update.NumTabs
}

16
pkg/waveai/anthropicbackend.go
View File

@ -10,11 +10,10 @@ import (
"errors"
"fmt"
"io"
"log"
"net/http"
"runtime/debug"
"strings"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
)
@ -115,17 +114,10 @@ func (AnthropicBackend) StreamCompletion(ctx context.Context, request wshrpc.Ope
go func() {
defer func() {
if r := recover(); r != nil {
// Convert panic to error and send it
log.Printf("panic: %v\n", r)
debug.PrintStack()
err, ok := r.(error)
if !ok {
err = fmt.Errorf("anthropic backend panic: %v", r)
}
rtn <- makeAIError(err)
panicErr := panichandler.PanicHandler("AnthropicBackend.StreamCompletion")
if panicErr != nil {
rtn <- makeAIError(panicErr)
}
// Always close the channel
close(rtn)
}()

9
pkg/waveai/cloudbackend.go
View File

@ -11,6 +11,7 @@ import (
"log"
"github.com/gorilla/websocket"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/wcloud"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
)
@ -37,7 +38,13 @@ func (WaveAICloudBackend) StreamCompletion(ctx context.Context, request wshrpc.O
rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.OpenAIPacketType])
wsEndpoint := wcloud.GetWSEndpoint()
go func() {
defer close(rtn)
defer func() {
panicErr := panichandler.PanicHandler("WaveAICloudBackend.StreamCompletion")
if panicErr != nil {
rtn <- makeAIError(panicErr)
}
close(rtn)
}()
if wsEndpoint == "" {
rtn <- makeAIError(fmt.Errorf("no cloud ws endpoint found"))
return

16
pkg/waveai/openaibackend.go
View File

@ -8,12 +8,11 @@ import (
"errors"
"fmt"
"io"
"log"
"regexp"
"runtime/debug"
"strings"
openaiapi "github.com/sashabaranov/go-openai"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
)
@ -75,17 +74,10 @@ func (OpenAIBackend) StreamCompletion(ctx context.Context, request wshrpc.OpenAi
rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.OpenAIPacketType])
go func() {
defer func() {
if r := recover(); r != nil {
// Convert panic to error and send it
log.Printf("panic: %v\n", r)
debug.PrintStack()
err, ok := r.(error)
if !ok {
err = fmt.Errorf("openai backend panic: %v", r)
}
rtn <- makeAIError(err)
panicErr := panichandler.PanicHandler("OpenAIBackend.StreamCompletion")
if panicErr != nil {
rtn <- makeAIError(panicErr)
}
// Always close the channel
close(rtn)
}()
if request.Opts == nil {

14
pkg/waveapp/waveappserverimpl.go
View File

@ -10,6 +10,7 @@ import (
"log"
"net/http"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/vdom"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
)
@ -23,12 +24,11 @@ func (*WaveAppServerImpl) WshServerImpl() {}
func (impl *WaveAppServerImpl) VDomRenderCommand(ctx context.Context, feUpdate vdom.VDomFrontendUpdate) chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate] {
respChan := make(chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate], 5)
defer func() {
if r := recover(); r != nil {
log.Printf("panic in VDomRenderCommand: %v\n", r)
panicErr := panichandler.PanicHandler("VDomRenderCommand")
if panicErr != nil {
respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{
Error: fmt.Errorf("internal error: %v", r),
Error: panicErr,
}
close(respChan)
}
@ -88,6 +88,7 @@ func (impl *WaveAppServerImpl) VDomRenderCommand(ctx context.Context, feUpdate v
// Split the update into chunks and send them sequentially
updates := vdom.SplitBackendUpdate(update)
go func() {
defer panichandler.PanicHandler("VDomRenderCommand:splitUpdates")
defer close(respChan)
for _, splitUpdate := range updates {
respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{
@ -108,9 +109,10 @@ func (impl *WaveAppServerImpl) VDomUrlRequestCommand(ctx context.Context, data w
defer writer.Close() // Ensures writer is closed before the channel is closed
defer func() {
if r := recover(); r != nil {
panicErr := panichandler.PanicHandler("VDomUrlRequestCommand")
if panicErr != nil {
writer.WriteHeader(http.StatusInternalServerError)
writer.Write([]byte(fmt.Sprintf("internal server error: %v", r)))
writer.Write([]byte(fmt.Sprintf("internal server error: %v", panicErr)))
}
}()

2
pkg/wconfig/filewatcher.go
View File

@ -10,6 +10,7 @@ import (
"sync"
"github.com/fsnotify/fsnotify"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/wavebase"
"github.com/wavetermdev/waveterm/pkg/wps"
)
@ -64,6 +65,7 @@ func (w *Watcher) Start() {
w.sendInitialValues()
go func() {
defer panichandler.PanicHandler("filewatcher:Start")
for {
select {
case event, ok := <-w.watcher.Events:

2
pkg/wcore/wcore.go
View File

@ -12,6 +12,7 @@ import (
"github.com/google/uuid"
"github.com/wavetermdev/waveterm/pkg/blockcontroller"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/telemetry"
"github.com/wavetermdev/waveterm/pkg/waveobj"
"github.com/wavetermdev/waveterm/pkg/wps"
@ -267,6 +268,7 @@ func CreateBlock(ctx context.Context, tabId string, blockDef *waveobj.BlockDef,
return nil, fmt.Errorf("error creating block: %w", err)
}
go func() {
defer panichandler.PanicHandler("CreateBlock:telemetry")
blockView := blockDef.Meta.GetString(waveobj.MetaKey_View, "")
if blockView == "" {
return

11
pkg/web/web.go
View File

@ -15,7 +15,6 @@ import (
"net/http"
"os"
"path/filepath"
"runtime/debug"
"strconv"
"time"
@ -25,6 +24,7 @@ import (
"github.com/wavetermdev/waveterm/pkg/authkey"
"github.com/wavetermdev/waveterm/pkg/docsite"
"github.com/wavetermdev/waveterm/pkg/filestore"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/service"
"github.com/wavetermdev/waveterm/pkg/wavebase"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
@ -358,21 +358,18 @@ type ClientActiveState struct {
func WebFnWrap(opts WebFnOpts, fn WebFnType) WebFnType {
return func(w http.ResponseWriter, r *http.Request) {
defer func() {
recErr := recover()
recErr := panichandler.PanicHandler("WebFnWrap")
if recErr == nil {
return
}
panicStr := fmt.Sprintf("panic: %v", recErr)
log.Printf("panic: %v\n", recErr)
debug.PrintStack()
if opts.JsonErrors {
jsonRtn := marshalReturnValue(nil, fmt.Errorf(panicStr))
jsonRtn := marshalReturnValue(nil, recErr)
w.Header().Set(ContentTypeHeaderKey, ContentTypeJson)
w.Header().Set(ContentLengthHeaderKey, fmt.Sprintf("%d", len(jsonRtn)))
w.WriteHeader(http.StatusOK)
w.Write(jsonRtn)
} else {
http.Error(w, panicStr, http.StatusInternalServerError)
http.Error(w, recErr.Error(), http.StatusInternalServerError)
}
}()
if !opts.AllowCaching {

13
pkg/web/ws.go
View File

@ -9,7 +9,6 @@ import (
"log"
"net"
"net/http"
"runtime/debug"
"sync"
"time"
@ -18,6 +17,7 @@ import (
"github.com/gorilla/websocket"
"github.com/wavetermdev/waveterm/pkg/authkey"
"github.com/wavetermdev/waveterm/pkg/eventbus"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/web/webcmd"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
"github.com/wavetermdev/waveterm/pkg/wshutil"
@ -81,11 +81,9 @@ func getStringFromMap(jmsg map[string]any, key string) string {
func processWSCommand(jmsg map[string]any, outputCh chan any, rpcInputCh chan []byte) {
var rtnErr error
defer func() {
r := recover()
if r != nil {
rtnErr = fmt.Errorf("panic: %v", r)
log.Printf("[websocket] panic in processMessage: %v\n", r)
debug.PrintStack()
panicErr := panichandler.PanicHandler("processWSCommand")
if panicErr != nil {
rtnErr = panicErr
}
if rtnErr == nil {
return
@ -304,6 +302,7 @@ func HandleWsInternal(w http.ResponseWriter, r *http.Request) error {
wg := &sync.WaitGroup{}
wg.Add(2)
go func() {
defer panichandler.PanicHandler("HandleWsInternal:outputCh")
// no waitgroup add here
// move values from rpcOutputCh to outputCh
for msgBytes := range wproxy.ToRemoteCh {
@ -315,11 +314,13 @@ func HandleWsInternal(w http.ResponseWriter, r *http.Request) error {
}
}()
go func() {
defer panichandler.PanicHandler("HandleWsInternal:ReadLoop")
// read loop
defer wg.Done()
ReadLoop(conn, outputCh, closeCh, wproxy.FromRemoteCh, routeId)
}()
go func() {
defer panichandler.PanicHandler("HandleWsInternal:WriteLoop")
// write loop
defer wg.Done()
WriteLoop(conn, outputCh, closeCh, routeId)

3
pkg/wshrpc/wshclient/wshclientutil.go
View File

@ -6,6 +6,7 @@ package wshclient
import (
"errors"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/utilfn"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
"github.com/wavetermdev/waveterm/pkg/wshutil"
@ -39,6 +40,7 @@ func sendRpcRequestCallHelper[T any](w *wshutil.WshRpc, command string, data int
func rtnErr[T any](ch chan wshrpc.RespOrErrorUnion[T], err error) {
go func() {
defer panichandler.PanicHandler("wshclientutil:rtnErr")
ch <- wshrpc.RespOrErrorUnion[T]{Error: err}
close(ch)
}()
@ -63,6 +65,7 @@ func sendRpcRequestResponseStreamHelper[T any](w *wshutil.WshRpc, command string
reqHandler.SendCancel()
}
go func() {
defer panichandler.PanicHandler("sendRpcRequestResponseStreamHelper")
defer close(respChan)
for {
if reqHandler.ResponseDone() {

8
pkg/wshrpc/wshserver/wshserver.go
View File

@ -18,6 +18,7 @@ import (
"github.com/wavetermdev/waveterm/pkg/blockcontroller"
"github.com/wavetermdev/waveterm/pkg/filestore"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/remote"
"github.com/wavetermdev/waveterm/pkg/remote/conncontroller"
"github.com/wavetermdev/waveterm/pkg/telemetry"
@ -45,11 +46,7 @@ func (*WshServer) WshServerImpl() {}
var WshServerImpl = WshServer{}
func (ws *WshServer) TestCommand(ctx context.Context, data string) error {
defer func() {
if r := recover(); r != nil {
log.Printf("panic in TestCommand: %v", r)
}
}()
defer panichandler.PanicHandler("TestCommand")
rpcSource := wshutil.GetRpcSourceFromContext(ctx)
log.Printf("TEST src:%s | %s\n", rpcSource, data)
return nil
@ -65,6 +62,7 @@ func (ws *WshServer) MessageCommand(ctx context.Context, data wshrpc.CommandMess
func (ws *WshServer) StreamTestCommand(ctx context.Context) chan wshrpc.RespOrErrorUnion[int] {
rtn := make(chan wshrpc.RespOrErrorUnion[int])
go func() {
defer panichandler.PanicHandler("StreamTestCommand")
for i := 1; i <= 5; i++ {
rtn <- wshrpc.RespOrErrorUnion[int]{Response: i}
time.Sleep(1 * time.Second)

2
pkg/wshutil/wshadapter.go
View File

@ -8,6 +8,7 @@ import (
"reflect"
"strings"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/utilfn"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
)
@ -131,6 +132,7 @@ func serverImplAdapter(impl any) func(*RpcResponseHandler) bool {
return true
}
go func() {
defer panichandler.PanicHandler("serverImplAdapter:responseStream")
defer handler.Finalize()
// must use reflection here because we don't know the generic type of RespOrErrorUnion
for {

2
pkg/wshutil/wshmultiproxy.go
View File

@ -9,6 +9,7 @@ import (
"sync"
"github.com/google/uuid"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
)
@ -112,6 +113,7 @@ func (p *WshRpcMultiProxy) handleUnauthMessage(msgBytes []byte) {
p.setRouteInfo(routeInfo.AuthToken, routeInfo)
p.sendAuthResponse(msg, routeId, routeInfo.AuthToken)
go func() {
defer panichandler.PanicHandler("WshRpcMultiProxy:handleUnauthMessage")
for msgBytes := range routeInfo.Proxy.ToRemoteCh {
p.ToRemoteCh <- msgBytes
}

4
pkg/wshutil/wshrouter.go
View File

@ -13,6 +13,7 @@ import (
"time"
"github.com/google/uuid"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/utilfn"
"github.com/wavetermdev/waveterm/pkg/wps"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
@ -91,6 +92,7 @@ func noRouteErr(routeId string) error {
}
func (router *WshRouter) SendEvent(routeId string, event wps.WaveEvent) {
defer panichandler.PanicHandler("WshRouter.SendEvent")
rpc := router.GetRpc(routeId)
if rpc == nil {
return
@ -296,6 +298,7 @@ func (router *WshRouter) RegisterRoute(routeId string, rpc AbstractRpcClient, sh
}
router.RouteMap[routeId] = rpc
go func() {
defer panichandler.PanicHandler("WshRouter:registerRoute:recvloop")
// announce
if shouldAnnounce && !alreadyExists && router.GetUpstreamClient() != nil {
announceMsg := RpcMessage{Command: wshrpc.Command_RouteAnnounce, Source: routeId}
@ -341,6 +344,7 @@ func (router *WshRouter) UnregisterRoute(routeId string) {
}
}
go func() {
defer panichandler.PanicHandler("WshRouter:unregisterRoute:routegone")
wps.Broker.UnsubscribeAll(routeId)
wps.Broker.Publish(wps.WaveEvent{Event: wps.Event_RouteGone, Scopes: []string{routeId}})
}()

50
pkg/wshutil/wshrpc.go
View File

@ -10,12 +10,12 @@ import (
"fmt"
"log"
"reflect"
"runtime/debug"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/utilfn"
"github.com/wavetermdev/waveterm/pkg/wps"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
@ -281,15 +281,6 @@ func (w *WshRpc) handleRequest(req *RpcMessage) {
}
var respHandler *RpcResponseHandler
defer func() {
if r := recover(); r != nil {
log.Printf("panic in handleRequest: %v\n", r)
debug.PrintStack()
if respHandler != nil {
respHandler.SendResponseError(fmt.Errorf("panic: %v", r))
}
}
}()
timeoutMs := req.Timeout
if timeoutMs <= 0 {
timeoutMs = DefaultTimeoutMs
@ -313,13 +304,13 @@ func (w *WshRpc) handleRequest(req *RpcMessage) {
w.registerResponseHandler(req.ReqId, respHandler)
isAsync := false
defer func() {
if r := recover(); r != nil {
log.Printf("panic in handleRequest: %v\n", r)
debug.PrintStack()
respHandler.SendResponseError(fmt.Errorf("panic: %v", r))
panicErr := panichandler.PanicHandler("handleRequest")
if panicErr != nil {
respHandler.SendResponseError(panicErr)
}
if isAsync {
go func() {
defer panichandler.PanicHandler("handleRequest:finalize")
<-ctx.Done()
respHandler.Finalize()
}()
@ -394,6 +385,7 @@ func (w *WshRpc) registerRpc(ctx context.Context, reqId string) chan *RpcMessage
Ctx: ctx,
}
go func() {
defer panichandler.PanicHandler("registerRpc:timeout")
<-ctx.Done()
w.unregisterRpc(reqId, fmt.Errorf("EC-TIME: timeout waiting for response"))
}()
@ -463,12 +455,7 @@ func (handler *RpcRequestHandler) Context() context.Context {
}
func (handler *RpcRequestHandler) SendCancel() {
defer func() {
if r := recover(); r != nil {
// this is likely a write to closed channel
log.Printf("panic in SendCancel: %v\n", r)
}
}()
defer panichandler.PanicHandler("SendCancel")
msg := &RpcMessage{
Cancel: true,
ReqId: handler.reqId,
@ -573,13 +560,7 @@ func (handler *RpcResponseHandler) SendMessage(msg string) {
}
func (handler *RpcResponseHandler) SendResponse(data any, done bool) error {
defer func() {
if r := recover(); r != nil {
// this is likely a write to closed channel
log.Printf("panic in SendResponse: %v\n", r)
handler.close()
}
}()
defer panichandler.PanicHandler("SendResponse")
if handler.reqId == "" {
return nil // no response expected
}
@ -604,13 +585,7 @@ func (handler *RpcResponseHandler) SendResponse(data any, done bool) error {
}
func (handler *RpcResponseHandler) SendResponseError(err error) {
defer func() {
if r := recover(); r != nil {
// this is likely a write to closed channel
log.Printf("panic in SendResponseError: %v\n", r)
handler.close()
}
}()
defer panichandler.PanicHandler("SendResponseError")
if handler.reqId == "" || handler.done.Load() {
return
}
@ -659,12 +634,7 @@ func (w *WshRpc) SendComplexRequest(command string, data any, opts *wshrpc.RpcOp
if timeoutMs <= 0 {
timeoutMs = DefaultTimeoutMs
}
defer func() {
if r := recover(); r != nil {
log.Printf("panic in SendComplexRequest: %v\n", r)
rtnErr = fmt.Errorf("panic: %v", r)
}
}()
defer panichandler.PanicHandler("SendComplexRequest")
if command == "" {
return nil, fmt.Errorf("command cannot be empty")
}

12
pkg/wshutil/wshutil.go
View File

@ -19,6 +19,7 @@ import (
"github.com/golang-jwt/jwt/v5"
"github.com/google/uuid"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/packetparser"
"github.com/wavetermdev/waveterm/pkg/wavebase"
"github.com/wavetermdev/waveterm/pkg/wshrpc"
@ -150,6 +151,7 @@ func installShutdownSignalHandlers(quiet bool) {
sigCh := make(chan os.Signal, 1)
signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
go func() {
defer panichandler.PanicHandlerNoTelemetry("installShutdownSignalHandlers")
for sig := range sigCh {
DoShutdown(fmt.Sprintf("got signal %v", sig), 1, quiet)
break
@ -198,6 +200,7 @@ func SetupTerminalRpcClient(serverImpl ServerImpl) (*WshRpc, io.Reader) {
ptyBuf := MakePtyBuffer(WaveServerOSCPrefix, os.Stdin, messageCh)
rpcClient := MakeWshRpc(messageCh, outputCh, wshrpc.RpcContext{}, serverImpl)
go func() {
defer panichandler.PanicHandler("SetupTerminalRpcClient")
for msg := range outputCh {
barr, err := EncodeWaveOSCBytes(WaveOSC, msg)
if err != nil {
@ -218,6 +221,7 @@ func SetupPacketRpcClient(input io.Reader, output io.Writer, serverImpl ServerIm
rpcClient := MakeWshRpc(messageCh, outputCh, wshrpc.RpcContext{}, serverImpl)
go packetparser.Parse(input, messageCh, rawCh)
go func() {
defer panichandler.PanicHandler("SetupPacketRpcClient:outputloop")
for msg := range outputCh {
packetparser.WritePacket(output, msg)
}
@ -230,6 +234,7 @@ func SetupConnRpcClient(conn net.Conn, serverImpl ServerImpl) (*WshRpc, chan err
outputCh := make(chan []byte, DefaultOutputChSize)
writeErrCh := make(chan error, 1)
go func() {
defer panichandler.PanicHandler("SetupConnRpcClient:AdaptOutputChToStream")
writeErr := AdaptOutputChToStream(outputCh, conn)
if writeErr != nil {
writeErrCh <- writeErr
@ -237,6 +242,7 @@ func SetupConnRpcClient(conn net.Conn, serverImpl ServerImpl) (*WshRpc, chan err
}
}()
go func() {
defer panichandler.PanicHandler("SetupConnRpcClient:AdaptStreamToMsgCh")
// when input is closed, close the connection
defer conn.Close()
AdaptStreamToMsgCh(conn, inputCh)
@ -264,6 +270,7 @@ func SetupDomainSocketRpcClient(sockName string, serverImpl ServerImpl) (*WshRpc
}
rtn, errCh, err := SetupConnRpcClient(conn, serverImpl)
go func() {
defer panichandler.PanicHandler("SetupDomainSocketRpcClient:closeConn")
defer conn.Close()
err := <-errCh
if err != nil && err != io.EOF {
@ -410,9 +417,11 @@ func HandleStdIOClient(logName string, input io.Reader, output io.Writer) {
proxy.DisposeRoutes()
}
go func() {
defer panichandler.PanicHandler("HandleStdIOClient:RunUnauthLoop")
proxy.RunUnauthLoop()
}()
go func() {
defer panichandler.PanicHandler("HandleStdIOClient:ToRemoteChLoop")
defer closeDoneCh()
for msg := range proxy.ToRemoteCh {
err := packetparser.WritePacket(output, msg)
@ -423,6 +432,7 @@ func HandleStdIOClient(logName string, input io.Reader, output io.Writer) {
}
}()
go func() {
defer panichandler.PanicHandler("HandleStdIOClient:RawChLoop")
defer closeDoneCh()
for msg := range rawCh {
log.Printf("[%s:stdout] %s", logName, msg)
@ -435,6 +445,7 @@ func handleDomainSocketClient(conn net.Conn) {
var routeIdContainer atomic.Pointer[string]
proxy := MakeRpcProxy()
go func() {
defer panichandler.PanicHandler("handleDomainSocketClient:AdaptOutputChToStream")
writeErr := AdaptOutputChToStream(proxy.ToRemoteCh, conn)
if writeErr != nil {
log.Printf("error writing to domain socket: %v\n", writeErr)
@ -442,6 +453,7 @@ func handleDomainSocketClient(conn net.Conn) {
}()
go func() {
// when input is closed, close the connection
defer panichandler.PanicHandler("handleDomainSocketClient:AdaptStreamToMsgCh")
defer func() {
conn.Close()
routeIdPtr := routeIdContainer.Load()

3
pkg/wsl/wsl-util.go
View File

@ -15,6 +15,8 @@ import (
"path/filepath"
"strings"
"time"
"github.com/wavetermdev/waveterm/pkg/panichandler"
)
func DetectShell(ctx context.Context, client *Distro) (string, error) {
@ -233,6 +235,7 @@ func CpHostToRemote(ctx context.Context, client *Distro, sourcePath string, dest
return fmt.Errorf("cannot open local file %s to send to host: %v", sourcePath, err)
}
go func() {
defer panichandler.PanicHandler("wslutil:cpHostToRemote:catStdin")
io.Copy(catStdin, input)
installStepCmds["cat"].Cancel()

3
pkg/wsl/wsl.go
View File

@ -13,6 +13,7 @@ import (
"sync/atomic"
"time"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/telemetry"
"github.com/wavetermdev/waveterm/pkg/userinput"
"github.com/wavetermdev/waveterm/pkg/util/shellutil"
@ -234,6 +235,7 @@ func (conn *WslConn) StartConnServer() error {
})
// service the I/O
go func() {
defer panichandler.PanicHandler("wsl:StartConnServer:wait")
// wait for termination, clear the controller
defer conn.WithLock(func() {
conn.ConnController = nil
@ -242,6 +244,7 @@ func (conn *WslConn) StartConnServer() error {
log.Printf("conn controller (%q) terminated: %v", conn.GetName(), waitErr)
}()
go func() {
defer panichandler.PanicHandler("wsl:StartConnServer:handleStdIOClient")
logName := fmt.Sprintf("conncontroller:%s", conn.GetName())
wshutil.HandleStdIOClient(logName, pipeRead, inputPipeWrite)
}()

2
pkg/wstore/wstore_dbops.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/wavetermdev/waveterm/pkg/filestore"
"github.com/wavetermdev/waveterm/pkg/panichandler"
"github.com/wavetermdev/waveterm/pkg/util/dbutil"
"github.com/wavetermdev/waveterm/pkg/waveobj"
)
@ -211,6 +212,7 @@ func DBDelete(ctx context.Context, otype string, id string) error {
return err
}
go func() {
defer panichandler.PanicHandler("DBDelete:filestore.DeleteZone")
// we spawn a go routine here because we don't want to reuse the DB connection
// since DBDelete is called in a transaction from DeleteTab
deleteCtx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second)