From 8e3540f754cbd587ffb64121f4a7d8947f66e0f7 Mon Sep 17 00:00:00 2001 From: Mike Sawka Date: Thu, 13 Jun 2024 23:54:04 -0700 Subject: [PATCH] working on ijson and wsh magic (#53) --- cmd/wsh/main-wsh.go | 66 ++++++++- emain/emain.ts | 5 + frontend/app/block/block.tsx | 2 +- frontend/app/store/global.ts | 30 +++++ frontend/app/view/ijson.tsx | 117 ++++++++++++++++ frontend/app/view/term.tsx | 141 ++++++++++++++++++-- frontend/app/view/view.less | 6 + frontend/types/gotypes.d.ts | 37 +++++- go.mod | 1 + go.sum | 2 + package.json | 1 + pkg/blockcontroller/blockcommand.go | 109 --------------- pkg/blockcontroller/blockcontroller.go | 98 +++++++++++--- pkg/blockcontroller/ptybuffer.go | 10 +- pkg/eventbus/eventbus.go | 11 ++ pkg/filestore/blockstore.go | 109 ++++++++++++++- pkg/filestore/blockstore_test.go | 128 ++++++++++++++++++ pkg/ijson/ijson.go | 162 ++++++++++++++++++++--- pkg/service/blockservice/blockservice.go | 3 +- pkg/service/service.go | 6 +- pkg/tsgen/tsgen.go | 5 +- pkg/web/ws.go | 6 +- pkg/wshutil/wshcommands.go | 131 +++++++++++++----- pkg/wshutil/wshutil.go | 8 +- pkg/wstore/wstore_dbops.go | 12 +- pkg/wstore/wstore_types.go | 1 + yarn.lock | 12 ++ 27 files changed, 996 insertions(+), 223 deletions(-) create mode 100644 frontend/app/view/ijson.tsx delete mode 100644 pkg/blockcontroller/blockcommand.go diff --git a/cmd/wsh/main-wsh.go b/cmd/wsh/main-wsh.go index 927af8c97..a8bb8c4f1 100644 --- a/cmd/wsh/main-wsh.go +++ b/cmd/wsh/main-wsh.go @@ -5,14 +5,74 @@ package main import ( "fmt" + "log" "os" + "os/signal" + "sync" + "syscall" + + "github.com/wavetermdev/thenextwave/pkg/wshutil" + "golang.org/x/term" ) +var shutdownOnce sync.Once +var origTermState *term.State + +func doShutdown(reason string, exitCode int) { + shutdownOnce.Do(func() { + defer os.Exit(exitCode) + log.Printf("shutting down: %s\r\n", reason) + cmd := &wshutil.BlockSetMetaCommand{ + Command: wshutil.BlockCommand_SetMeta, + Meta: map[string]any{"term:mode": nil}, + } + barr, _ := wshutil.EncodeWaveOSCMessage(cmd) + if origTermState != nil { + term.Restore(int(os.Stdin.Fd()), origTermState) + } + os.Stdout.Write(barr) + }) +} + +func installShutdownSignalHandlers() { + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) + go func() { + for sig := range sigCh { + doShutdown(fmt.Sprintf("got signal %v", sig), 1) + break + } + }() +} + func main() { - barr, err := os.ReadFile("/Users/mike/Downloads/2.png") + installShutdownSignalHandlers() + defer doShutdown("normal exit", 0) + origState, err := term.MakeRaw(int(os.Stdin.Fd())) if err != nil { - fmt.Println("error reading file:", err) + fmt.Fprintf(os.Stderr, "Error setting raw mode: %v\n", err) return } - fmt.Println("file size:", len(barr)) + origTermState = origState + cmd := &wshutil.BlockSetMetaCommand{ + Command: wshutil.BlockCommand_SetMeta, + Meta: map[string]any{"term:mode": "html"}, + } + barr, _ := wshutil.EncodeWaveOSCMessage(cmd) + os.Stdout.Write(barr) + for { + var buf [1]byte + _, err := os.Stdin.Read(buf[:]) + if err != nil { + doShutdown(fmt.Sprintf("stdin closed/error (%v)", err), 1) + } + if buf[0] == 0x03 { + doShutdown("read Ctrl-C from stdin", 1) + break + } + if buf[0] == 'x' { + doShutdown("read 'x' from stdin", 0) + break + } + } } diff --git a/emain/emain.ts b/emain/emain.ts index fa22f2e30..fa44c48de 100644 --- a/emain/emain.ts +++ b/emain/emain.ts @@ -124,6 +124,11 @@ function mainResizeHandler(_: any, win: Electron.BrowserWindow) { } function shNavHandler(event: Electron.Event, url: string) { + if (url.startsWith("http://localhost:5173/index.html")) { + // this is a dev-mode hot-reload, ignore it + console.log("allowing hot-reload of index.html"); + return; + } event.preventDefault(); if (url.startsWith("https://") || url.startsWith("http://") || url.startsWith("file://")) { console.log("open external, shNav", url); diff --git a/frontend/app/block/block.tsx b/frontend/app/block/block.tsx index 1a839a070..3b7fa5d05 100644 --- a/frontend/app/block/block.tsx +++ b/frontend/app/block/block.tsx @@ -49,7 +49,7 @@ const Block = ({ blockId, onClose }: BlockProps) => { } else if (blockData.view === "plot") { blockElem = ; } else if (blockData.view === "codeedit") { - blockElem = ; + blockElem = ; } return (
diff --git a/frontend/app/store/global.ts b/frontend/app/store/global.ts index c9a20e9a8..7b408c0ff 100644 --- a/frontend/app/store/global.ts +++ b/frontend/app/store/global.ts @@ -66,6 +66,7 @@ type SubjectWithRef = rxjs.Subject & { refCount: number; release: () => vo // key is "eventType" or "eventType|oref" const eventSubjects = new Map>(); +const fileSubjects = new Map>(); function getSubjectInternal(subjectKey: string): SubjectWithRef { let subject = eventSubjects.get(subjectKey); @@ -93,6 +94,25 @@ function getEventORefSubject(eventType: string, oref: string): SubjectWithRef { + const subjectKey = zoneId + "|" + fileName; + let subject = fileSubjects.get(subjectKey); + if (subject == null) { + subject = new rxjs.Subject() as any; + subject.refCount = 0; + subject.release = () => { + subject.refCount--; + if (subject.refCount === 0) { + subject.complete(); + fileSubjects.delete(subjectKey); + } + }; + fileSubjects.set(subjectKey, subject); + } + subject.refCount++; + return subject; +} + const blockCache = new Map>(); function useBlockCache(blockId: string, name: string, makeFn: () => T): T { @@ -142,6 +162,15 @@ function handleWSEventMessage(msg: WSEventType) { console.log("unsupported event", msg); return; } + if (msg.eventtype == "blockfile") { + const fileData: WSFileEventData = msg.data; + const fileSubject = getFileSubject(fileData.zoneid, fileData.filename); + if (fileSubject != null) { + fileSubject.next(fileData); + } + return; + } + // we send to two subjects just eventType and eventType|oref // we don't use getORefSubject here because we don't want to create a new subject const eventSubject = eventSubjects.get(msg.eventtype); @@ -193,6 +222,7 @@ export { getBackendHostPort, getEventORefSubject, getEventSubject, + getFileSubject, globalStore, globalWS, initWS, diff --git a/frontend/app/view/ijson.tsx b/frontend/app/view/ijson.tsx new file mode 100644 index 000000000..b91fea4eb --- /dev/null +++ b/frontend/app/view/ijson.tsx @@ -0,0 +1,117 @@ +// Copyright 2024, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +import * as React from "react"; +import Frame from "react-frame-component"; + +type IJsonNode = { + tag: string; + props?: Record; + children?: (IJsonNode | string)[]; +}; + +const TagMap: Record> = {}; + +function convertNodeToTag(node: IJsonNode | string, idx?: number): JSX.Element | string { + if (node == null) { + return null; + } + if (idx == null) { + idx = 0; + } + if (typeof node === "string") { + return node; + } + let key = node.props?.key ?? "child-" + idx; + let TagComp = TagMap[node.tag]; + if (!TagComp) { + return
Unknown tag:{node.tag}
; + } + return ; +} + +function IJsonHtmlTag({ node }: { node: IJsonNode }) { + let { tag, props, children } = node; + let divProps = {}; + if (props != null) { + for (let [key, val] of Object.entries(props)) { + if (key.startsWith("on")) { + divProps[key] = (e: any) => { + console.log("handler", key, val); + }; + } else { + divProps[key] = val; + } + } + } + let childrenComps: (string | JSX.Element)[] = []; + if (children != null) { + for (let idx = 0; idx < children.length; idx++) { + let comp = convertNodeToTag(children[idx], idx); + if (comp != null) { + childrenComps.push(comp); + } + } + } + return React.createElement(tag, divProps, childrenComps); +} + +TagMap["div"] = IJsonHtmlTag; +TagMap["b"] = IJsonHtmlTag; +TagMap["i"] = IJsonHtmlTag; +TagMap["p"] = IJsonHtmlTag; +TagMap["s"] = IJsonHtmlTag; +TagMap["span"] = IJsonHtmlTag; +TagMap["a"] = IJsonHtmlTag; +TagMap["img"] = IJsonHtmlTag; +TagMap["h1"] = IJsonHtmlTag; +TagMap["h2"] = IJsonHtmlTag; +TagMap["h3"] = IJsonHtmlTag; +TagMap["h4"] = IJsonHtmlTag; +TagMap["h5"] = IJsonHtmlTag; +TagMap["h6"] = IJsonHtmlTag; +TagMap["ul"] = IJsonHtmlTag; +TagMap["ol"] = IJsonHtmlTag; +TagMap["li"] = IJsonHtmlTag; +TagMap["input"] = IJsonHtmlTag; +TagMap["button"] = IJsonHtmlTag; +TagMap["textarea"] = IJsonHtmlTag; +TagMap["select"] = IJsonHtmlTag; +TagMap["option"] = IJsonHtmlTag; +TagMap["form"] = IJsonHtmlTag; + +function IJsonView({ rootNode }: { rootNode: IJsonNode }) { + // TODO fix this huge inline style + return ( +
+ + + {convertNodeToTag(rootNode)} + +
+ ); +} + +export { IJsonView }; diff --git a/frontend/app/view/term.tsx b/frontend/app/view/term.tsx index 60b343614..45f7f0aae 100644 --- a/frontend/app/view/term.tsx +++ b/frontend/app/view/term.tsx @@ -1,14 +1,25 @@ // Copyright 2024, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 -import { WOS, getBackendHostPort, getEventORefSubject, sendWSCommand } from "@/store/global"; +import { + WOS, + atoms, + getBackendHostPort, + getFileSubject, + globalStore, + sendWSCommand, + useBlockAtom, +} from "@/store/global"; import * as services from "@/store/services"; import { base64ToArray } from "@/util/util"; import { FitAddon } from "@xterm/addon-fit"; import type { ITheme } from "@xterm/xterm"; import { Terminal } from "@xterm/xterm"; import clsx from "clsx"; +import { produce } from "immer"; +import * as jotai from "jotai"; import * as React from "react"; +import { IJsonView } from "./ijson"; import "public/xterm.css"; import { debounce } from "throttle-debounce"; @@ -59,17 +70,96 @@ function handleResize(fitAddon: FitAddon, blockId: string, term: Terminal) { } } +const keyMap = { + Enter: "\r", + Backspace: "\x7f", + Tab: "\t", + Escape: "\x1b", + ArrowUp: "\x1b[A", + ArrowDown: "\x1b[B", + ArrowRight: "\x1b[C", + ArrowLeft: "\x1b[D", + Insert: "\x1b[2~", + Delete: "\x1b[3~", + Home: "\x1b[1~", + End: "\x1b[4~", + PageUp: "\x1b[5~", + PageDown: "\x1b[6~", +}; + +function keyboardEventToASCII(event: React.KeyboardEvent): string { + // check modifiers + // if no modifiers are set, just send the key + if (!event.altKey && !event.ctrlKey && !event.metaKey) { + if (event.key == null || event.key == "") { + return ""; + } + if (keyMap[event.key] != null) { + return keyMap[event.key]; + } + if (event.key.length == 1) { + return event.key; + } else { + console.log("not sending keyboard event", event.key, event); + } + } + // if meta or alt is set, there is no ASCII representation + if (event.metaKey || event.altKey) { + return ""; + } + // if ctrl is set, if it is a letter, subtract 64 from the uppercase value to get the ASCII value + if (event.ctrlKey) { + if ( + (event.key.length === 1 && event.key >= "A" && event.key <= "Z") || + (event.key >= "a" && event.key <= "z") + ) { + const key = event.key.toUpperCase(); + return String.fromCharCode(key.charCodeAt(0) - 64); + } + } + return ""; +} + type InitialLoadDataType = { loaded: boolean; heldData: Uint8Array[]; }; +const IJSONConst = { + tag: "div", + children: [ + { + tag: "h1", + children: ["Hello World"], + }, + { + tag: "p", + children: ["This is a paragraph"], + }, + ], +}; + +function setBlockFocus(blockId: string) { + let winData = globalStore.get(atoms.waveWindow); + winData = produce(winData, (draft) => { + draft.activeblockid = blockId; + }); + WOS.setObjectValue(winData, globalStore.set, true); +} + const TerminalView = ({ blockId }: { blockId: string }) => { const connectElemRef = React.useRef(null); const termRef = React.useRef(null); const initialLoadRef = React.useRef({ loaded: false, heldData: [] }); const htmlElemFocusRef = React.useRef(null); const [blockData] = WOS.useWaveObjectValue(WOS.makeORef("block", blockId)); + const isFocusedAtom = useBlockAtom(blockId, "isFocused", () => { + return jotai.atom((get) => { + const winData = get(atoms.waveWindow); + return winData.activeblockid === blockId; + }); + }); + const isFocused = jotai.useAtomValue(isFocusedAtom); React.useEffect(() => { console.log("terminal created"); const newTerm = new Terminal({ @@ -95,13 +185,16 @@ const TerminalView = ({ blockId }: { blockId: string }) => { const inputCmd: BlockInputCommand = { command: "controller:input", inputdata64: b64data }; services.BlockService.SendCommand(blockId, inputCmd); }); - - // block subject - const blockSubject = getEventORefSubject("block:ptydata", WOS.makeORef("block", blockId)); - blockSubject.subscribe((msg: WSEventType) => { - // base64 decode - const data = msg.data; - const decodedData = base64ToArray(data.ptydata); + newTerm.textarea.addEventListener("focus", () => { + setBlockFocus(blockId); + }); + const mainFileSubject = getFileSubject(blockId, "main"); + mainFileSubject.subscribe((msg: WSFileEventData) => { + if (msg.fileop != "append") { + console.log("bad fileop for terminal", msg); + return; + } + const decodedData = base64ToArray(msg.data64); if (initialLoadRef.current.loaded) { newTerm.write(decodedData); } else { @@ -146,7 +239,7 @@ const TerminalView = ({ blockId }: { blockId: string }) => { return () => { newTerm.dispose(); - blockSubject.release(); + mainFileSubject.release(); }; }, []); @@ -157,6 +250,13 @@ const TerminalView = ({ blockId }: { blockId: string }) => { services.BlockService.SendCommand(blockId, metaCmd); return false; } + const asciiVal = keyboardEventToASCII(event); + if (asciiVal.length == 0) { + return false; + } + const b64data = btoa(asciiVal); + const inputCmd: BlockInputCommand = { command: "controller:input", inputdata64: b64data }; + services.BlockService.SendCommand(blockId, inputCmd); return true; }; @@ -164,8 +264,18 @@ const TerminalView = ({ blockId }: { blockId: string }) => { if (termMode != "term" && termMode != "html") { termMode = "term"; } + + React.useEffect(() => { + if (isFocused && termMode == "term") { + termRef.current?.focus(); + } + if (isFocused && termMode == "html") { + htmlElemFocusRef.current?.focus(); + } + }); + return ( -
+
{ if (htmlElemFocusRef.current != null) { htmlElemFocusRef.current.focus(); } + setBlockFocus(blockId); }} >
- + {}} + />
- HTML MODE +
diff --git a/frontend/app/view/view.less b/frontend/app/view/view.less index bec72a389..b2048758f 100644 --- a/frontend/app/view/view.less +++ b/frontend/app/view/view.less @@ -77,6 +77,12 @@ .term-htmlelem { display: flex; } + + .ijson iframe { + width: 100%; + height: 100%; + border: none; + } } } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 90e9bbf52..27bb7c88b 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -14,9 +14,23 @@ declare global { meta: MetaType; }; + // wshutil.BlockAppendFileCommand + type BlockAppendFileCommand = { + command: "blockfile:append"; + filename: string; + data: number[]; + }; + + // wshutil.BlockAppendIJsonCommand + type BlockAppendIJsonCommand = { + command: "blockfile:appendijson"; + filename: string; + data: MetaType; + }; + type BlockCommand = { command: string; - } & ( BlockInputCommand | BlockSetViewCommand | BlockSetMetaCommand ); + } & ( BlockAppendIJsonCommand | BlockInputCommand | BlockSetViewCommand | BlockSetMetaCommand | BlockMessageCommand | BlockAppendFileCommand ); // wstore.BlockDef type BlockDef = { @@ -26,7 +40,7 @@ declare global { meta?: MetaType; }; - // blockcontroller.BlockInputCommand + // wshutil.BlockInputCommand type BlockInputCommand = { command: "controller:input"; inputdata64?: string; @@ -34,13 +48,19 @@ declare global { termsize?: TermSize; }; - // blockcontroller.BlockSetMetaCommand + // wshutil.BlockMessageCommand + type BlockMessageCommand = { + command: "message"; + message: string; + }; + + // wshutil.BlockSetMetaCommand type BlockSetMetaCommand = { command: "setmeta"; meta: MetaType; }; - // blockcontroller.BlockSetViewCommand + // wshutil.BlockSetViewCommand type BlockSetViewCommand = { command: "setview"; view: string; @@ -149,6 +169,14 @@ declare global { data: any; }; + // eventbus.WSFileEventData + type WSFileEventData = { + zoneid: string; + filename: string; + fileop: string; + data64: string; + }; + // waveobj.WaveObj type WaveObj = { otype: string; @@ -190,6 +218,7 @@ declare global { type WaveWindow = WaveObj & { workspaceid: string; activetabid: string; + activeblockid?: string; activeblockmap: {[key: string]: string}; pos: Point; winsize: WinSize; diff --git a/go.mod b/go.mod index 59496553c..6501a537b 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/sawka/txwrap v0.2.0 github.com/wavetermdev/waveterm/wavesrv v0.0.0-20240508181017-d07068c09d94 golang.org/x/sys v0.20.0 + golang.org/x/term v0.17.0 ) require ( diff --git a/go.sum b/go.sum index 04d5b4c91..4b6ea66ac 100644 --- a/go.sum +++ b/go.sum @@ -46,5 +46,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/package.json b/package.json index cf41368e0..8a3659673 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", "react-dom": "^18.3.1", + "react-frame-component": "^5.2.7", "react-markdown": "^9.0.1", "remark-gfm": "^4.0.0", "rxjs": "^7.8.1", diff --git a/pkg/blockcontroller/blockcommand.go b/pkg/blockcontroller/blockcommand.go deleted file mode 100644 index 3e8bb2ec9..000000000 --- a/pkg/blockcontroller/blockcommand.go +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright 2024, Command Line Inc. -// SPDX-License-Identifier: Apache-2.0 - -package blockcontroller - -import ( - "encoding/json" - "fmt" - "reflect" - - "github.com/wavetermdev/thenextwave/pkg/shellexec" - "github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta" -) - -const CommandKey = "command" - -const ( - BlockCommand_Message = "message" - BlockCommand_SetView = "setview" - BlockCommand_SetMeta = "setmeta" - BlockCommand_Input = "controller:input" -) - -var CommandToTypeMap = map[string]reflect.Type{ - BlockCommand_Input: reflect.TypeOf(BlockInputCommand{}), - BlockCommand_SetView: reflect.TypeOf(BlockSetViewCommand{}), - BlockCommand_SetMeta: reflect.TypeOf(BlockSetMetaCommand{}), - BlockCommand_Message: reflect.TypeOf(BlockMessageCommand{}), -} - -func CommandTypeUnionMeta() tsgenmeta.TypeUnionMeta { - return tsgenmeta.TypeUnionMeta{ - BaseType: reflect.TypeOf((*BlockCommand)(nil)).Elem(), - TypeFieldName: "command", - Types: []reflect.Type{ - reflect.TypeOf(BlockInputCommand{}), - reflect.TypeOf(BlockSetViewCommand{}), - reflect.TypeOf(BlockSetMetaCommand{}), - reflect.TypeOf(BlockMessageCommand{}), - }, - } -} - -type BlockCommand interface { - GetCommand() string -} - -type BlockCommandWrapper struct { - BlockCommand -} - -func ParseCmdMap(cmdMap map[string]any) (BlockCommand, error) { - cmdType, ok := cmdMap[CommandKey].(string) - if !ok { - return nil, fmt.Errorf("no %s field in command map", CommandKey) - } - mapJson, err := json.Marshal(cmdMap) - if err != nil { - return nil, fmt.Errorf("error marshalling command map: %w", err) - } - rtype := CommandToTypeMap[cmdType] - if rtype == nil { - return nil, fmt.Errorf("unknown command type %q", cmdType) - } - cmd := reflect.New(rtype).Interface() - err = json.Unmarshal(mapJson, cmd) - if err != nil { - return nil, fmt.Errorf("error unmarshalling command: %w", err) - } - return cmd.(BlockCommand), nil -} - -type BlockInputCommand struct { - Command string `json:"command" tstype:"\"controller:input\""` - InputData64 string `json:"inputdata64,omitempty"` - SigName string `json:"signame,omitempty"` - TermSize *shellexec.TermSize `json:"termsize,omitempty"` -} - -func (ic *BlockInputCommand) GetCommand() string { - return BlockCommand_Input -} - -type BlockSetViewCommand struct { - Command string `json:"command" tstype:"\"setview\""` - View string `json:"view"` -} - -func (svc *BlockSetViewCommand) GetCommand() string { - return BlockCommand_SetView -} - -type BlockSetMetaCommand struct { - Command string `json:"command" tstype:"\"setmeta\""` - Meta map[string]any `json:"meta"` -} - -func (smc *BlockSetMetaCommand) GetCommand() string { - return BlockCommand_SetMeta -} - -type BlockMessageCommand struct { - Command string `json:"command" tstype:"\"message\""` - Message string `json:"message"` -} - -func (bmc *BlockMessageCommand) GetCommand() string { - return BlockCommand_Message -} diff --git a/pkg/blockcontroller/blockcontroller.go b/pkg/blockcontroller/blockcontroller.go index a5dc37727..115ef072f 100644 --- a/pkg/blockcontroller/blockcontroller.go +++ b/pkg/blockcontroller/blockcontroller.go @@ -20,6 +20,7 @@ import ( "github.com/wavetermdev/thenextwave/pkg/filestore" "github.com/wavetermdev/thenextwave/pkg/shellexec" "github.com/wavetermdev/thenextwave/pkg/waveobj" + "github.com/wavetermdev/thenextwave/pkg/wshutil" "github.com/wavetermdev/thenextwave/pkg/wstore" ) @@ -28,21 +29,27 @@ const ( BlockController_Cmd = "cmd" ) +const ( + BlockFile_Main = "main" // used for main pty output + BlockFile_Html = "html" // used for alt html layout +) + const DefaultTimeout = 2 * time.Second var globalLock = &sync.Mutex{} var blockControllerMap = make(map[string]*BlockController) type BlockController struct { - Lock *sync.Mutex - BlockId string - BlockDef *wstore.BlockDef - InputCh chan BlockCommand - Status string + Lock *sync.Mutex + BlockId string + BlockDef *wstore.BlockDef + InputCh chan wshutil.BlockCommand + Status string + CreatedHtmlFile bool PtyBuffer *PtyBuffer ShellProc *shellexec.ShellProc - ShellInputCh chan *BlockInputCommand + ShellInputCh chan *wshutil.BlockInputCommand } func (bc *BlockController) WithLock(f func()) { @@ -91,21 +98,49 @@ func (bc *BlockController) Close() { } const DefaultTermMaxFileSize = 256 * 1024 +const DefaultHtmlMaxFileSize = 256 * 1024 -func (bc *BlockController) handleShellProcData(data []byte) error { +func handleAppendBlockFile(blockId string, blockFile string, data []byte) error { ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout) defer cancelFn() - err := filestore.WFS.AppendData(ctx, bc.BlockId, "main", data) + err := filestore.WFS.AppendData(ctx, blockId, blockFile, data) if err != nil { return fmt.Errorf("error appending to blockfile: %w", err) } eventbus.SendEvent(eventbus.WSEventType{ - EventType: "block:ptydata", - ORef: waveobj.MakeORef(wstore.OType_Block, bc.BlockId).String(), - Data: map[string]any{ - "blockid": bc.BlockId, - "blockfile": "main", - "ptydata": base64.StdEncoding.EncodeToString(data), + EventType: "blockfile", + ORef: waveobj.MakeORef(wstore.OType_Block, blockId).String(), + Data: &eventbus.WSFileEventData{ + ZoneId: blockId, + FileName: blockFile, + FileOp: eventbus.FileOp_Append, + Data64: base64.StdEncoding.EncodeToString(data), + }, + }) + return nil +} + +func handleAppendIJsonFile(blockId string, blockFile string, cmd map[string]any, tryCreate bool) error { + ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout) + defer cancelFn() + if blockFile == BlockFile_Html && tryCreate { + err := filestore.WFS.MakeFile(ctx, blockId, blockFile, nil, filestore.FileOptsType{MaxSize: DefaultHtmlMaxFileSize, IJson: true}) + if err != nil && err != filestore.ErrAlreadyExists { + return fmt.Errorf("error creating blockfile[html]: %w", err) + } + } + err := filestore.WFS.AppendIJson(ctx, blockId, blockFile, cmd) + if err != nil { + return fmt.Errorf("error appending to blockfile(ijson): %w", err) + } + eventbus.SendEvent(eventbus.WSEventType{ + EventType: "blockfile", + ORef: waveobj.MakeORef(wstore.OType_Block, blockId).String(), + Data: &eventbus.WSFileEventData{ + ZoneId: blockId, + FileName: blockFile, + FileOp: eventbus.FileOp_Append, + Data64: base64.StdEncoding.EncodeToString([]byte("{}")), }, }) return nil @@ -150,7 +185,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts) error { bc.ShellProc.Close() return err } - shellInputCh := make(chan *BlockInputCommand) + shellInputCh := make(chan *wshutil.BlockInputCommand) bc.ShellInputCh = shellInputCh go func() { defer func() { @@ -233,7 +268,7 @@ func (bc *BlockController) Run(bdata *wstore.Block) { for genCmd := range bc.InputCh { switch cmd := genCmd.(type) { - case *BlockInputCommand: + case *wshutil.BlockInputCommand: log.Printf("INPUT: %s | %q\n", bc.BlockId, cmd.InputData64) if bc.ShellInputCh != nil { bc.ShellInputCh <- cmd @@ -266,9 +301,11 @@ func StartBlockController(ctx context.Context, blockId string) error { Lock: &sync.Mutex{}, BlockId: blockId, Status: "init", - InputCh: make(chan BlockCommand), + InputCh: make(chan wshutil.BlockCommand), } - ptyBuffer := MakePtyBuffer(bc.handleShellProcData, func(cmd BlockCommand) error { + ptyBuffer := MakePtyBuffer(func(fileName string, data []byte) error { + return handleAppendBlockFile(blockId, fileName, data) + }, func(cmd wshutil.BlockCommand) error { if strings.HasPrefix(cmd.GetCommand(), "controller:") { bc.InputCh <- cmd } else { @@ -297,11 +334,11 @@ func GetBlockController(blockId string) *BlockController { return blockControllerMap[blockId] } -func ProcessStaticCommand(blockId string, cmdGen BlockCommand) error { +func ProcessStaticCommand(blockId string, cmdGen wshutil.BlockCommand) error { ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout) defer cancelFn() switch cmd := cmdGen.(type) { - case *BlockSetViewCommand: + case *wshutil.BlockSetViewCommand: log.Printf("SETVIEW: %s | %q\n", blockId, cmd.View) block, err := wstore.DBGet[*wstore.Block](ctx, blockId) if err != nil { @@ -328,7 +365,7 @@ func ProcessStaticCommand(blockId string, cmdGen BlockCommand) error { }, }) return nil - case *BlockSetMetaCommand: + case *wshutil.BlockSetMetaCommand: log.Printf("SETMETA: %s | %v\n", blockId, cmd.Meta) block, err := wstore.DBGet[*wstore.Block](ctx, blockId) if err != nil { @@ -367,9 +404,26 @@ func ProcessStaticCommand(blockId string, cmdGen BlockCommand) error { }, }) return nil - case *BlockMessageCommand: + case *wshutil.BlockMessageCommand: log.Printf("MESSAGE: %s | %q\n", blockId, cmd.Message) return nil + + case *wshutil.BlockAppendFileCommand: + log.Printf("APPENDFILE: %s | %q | len:%d\n", blockId, cmd.FileName, len(cmd.Data)) + err := handleAppendBlockFile(blockId, cmd.FileName, cmd.Data) + if err != nil { + return fmt.Errorf("error appending blockfile: %w", err) + } + return nil + + case *wshutil.BlockAppendIJsonCommand: + log.Printf("APPENDIJSON: %s | %q\n", blockId, cmd.FileName) + err := handleAppendIJsonFile(blockId, cmd.FileName, cmd.Data, true) + if err != nil { + return fmt.Errorf("error appending blockfile(ijson): %w", err) + } + return nil + default: return fmt.Errorf("unknown command type %T", cmdGen) } diff --git a/pkg/blockcontroller/ptybuffer.go b/pkg/blockcontroller/ptybuffer.go index f093e13f3..fb20a96c4 100644 --- a/pkg/blockcontroller/ptybuffer.go +++ b/pkg/blockcontroller/ptybuffer.go @@ -19,12 +19,12 @@ const ( type PtyBuffer struct { Mode string EscSeqBuf []byte - DataOutputFn func([]byte) error - CommandOutputFn func(BlockCommand) error + DataOutputFn func(string, []byte) error + CommandOutputFn func(wshutil.BlockCommand) error Err error } -func MakePtyBuffer(dataOutputFn func([]byte) error, commandOutputFn func(BlockCommand) error) *PtyBuffer { +func MakePtyBuffer(dataOutputFn func(string, []byte) error, commandOutputFn func(wshutil.BlockCommand) error) *PtyBuffer { return &PtyBuffer{ Mode: Mode_Normal, DataOutputFn: dataOutputFn, @@ -45,7 +45,7 @@ func (b *PtyBuffer) processWaveEscSeq(escSeq []byte) { b.setErr(fmt.Errorf("error unmarshalling Wave OSC sequence data: %w", err)) return } - cmd, err := ParseCmdMap(jmsg) + cmd, err := wshutil.ParseCmdMap(jmsg) if err != nil { b.setErr(fmt.Errorf("error parsing Wave OSC command: %w", err)) return @@ -111,7 +111,7 @@ func (b *PtyBuffer) AppendData(data []byte) { outputBuf = append(outputBuf, ch) } if len(outputBuf) > 0 { - err := b.DataOutputFn(outputBuf) + err := b.DataOutputFn(BlockFile_Main, outputBuf) if err != nil { b.setErr(fmt.Errorf("error processing data output: %w", err)) } diff --git a/pkg/eventbus/eventbus.go b/pkg/eventbus/eventbus.go index 34cea81c6..f2da20b68 100644 --- a/pkg/eventbus/eventbus.go +++ b/pkg/eventbus/eventbus.go @@ -15,6 +15,17 @@ type WSEventType struct { Data any `json:"data"` } +const ( + FileOp_Append = "append" +) + +type WSFileEventData struct { + ZoneId string `json:"zoneid"` + FileName string `json:"filename"` + FileOp string `json:"fileop"` + Data64 string `json:"data64"` +} + type WindowWatchData struct { WindowWSCh chan any WaveWindowId string diff --git a/pkg/filestore/blockstore.go b/pkg/filestore/blockstore.go index 121314035..d7760c9b7 100644 --- a/pkg/filestore/blockstore.go +++ b/pkg/filestore/blockstore.go @@ -16,6 +16,21 @@ import ( "sync" "sync/atomic" "time" + + "github.com/wavetermdev/thenextwave/pkg/ijson" +) + +const ( + // ijson meta keys + IJsonNumCommands = "ijson:numcmds" + IJsonIncrementalBytes = "ijson:incbytes" +) + +const ( + IJsonHighCommands = 100 + IJsonHighRatio = 3 + IJsonLowRatio = 1 + IJsonLowCommands = 10 ) const DefaultPartDataSize = 64 * 1024 @@ -35,9 +50,10 @@ var WFS *FileStore = &FileStore{ } type FileOptsType struct { - MaxSize int64 `json:"maxsize,omitempty"` - Circular bool `json:"circular,omitempty"` - IJson bool `json:"ijson,omitempty"` + MaxSize int64 `json:"maxsize,omitempty"` + Circular bool `json:"circular,omitempty"` + IJson bool `json:"ijson,omitempty"` + IJsonBudget int `json:"ijsonbudget,omitempty"` } type FileMeta = map[string]any @@ -118,6 +134,12 @@ func (s *FileStore) MakeFile(ctx context.Context, zoneId string, name string, me opts.MaxSize = (opts.MaxSize/partDataSize + 1) * partDataSize } } + if opts.IJsonBudget > 0 && !opts.IJson { + return fmt.Errorf("ijson budget requires ijson") + } + if opts.IJsonBudget < 0 { + return fmt.Errorf("ijson budget must be non-negative") + } return withLock(s, zoneId, name, func(entry *CacheEntry) error { if entry.File != nil { return fs.ErrExist @@ -262,6 +284,87 @@ func (s *FileStore) AppendData(ctx context.Context, zoneId string, name string, }) } +func metaIncrement(file *WaveFile, key string, amount int) int { + if file.Meta == nil { + file.Meta = make(FileMeta) + } + val, ok := file.Meta[key].(int) + if !ok { + val = 0 + } + newVal := val + amount + file.Meta[key] = newVal + return newVal +} + +func (s *FileStore) compactIJson(ctx context.Context, entry *CacheEntry) error { + // we don't need to lock the entry because we have the lock on the filestore + _, fullData, err := entry.readAt(ctx, 0, 0, true) + if err != nil { + return err + } + newBytes, err := ijson.CompactIJson(fullData, entry.File.Opts.IJsonBudget) + if err != nil { + return err + } + entry.writeAt(0, newBytes, true) + return nil +} + +func (s *FileStore) CompactIJson(ctx context.Context, zoneId string, name string) error { + return withLock(s, zoneId, name, func(entry *CacheEntry) error { + err := entry.loadFileIntoCache(ctx) + if err != nil { + return err + } + if !entry.File.Opts.IJson { + return fmt.Errorf("file %s:%s is not an ijson file", zoneId, name) + } + return s.compactIJson(ctx, entry) + }) +} + +func (s *FileStore) AppendIJson(ctx context.Context, zoneId string, name string, command map[string]any) error { + data, err := ijson.ValidateAndMarshalCommand(command) + if err != nil { + return err + } + return withLock(s, zoneId, name, func(entry *CacheEntry) error { + err := entry.loadFileIntoCache(ctx) + if err != nil { + return err + } + if !entry.File.Opts.IJson { + return fmt.Errorf("file %s:%s is not an ijson file", zoneId, name) + } + partMap := entry.File.computePartMap(entry.File.Size, int64(len(data))) + incompleteParts := incompletePartsFromMap(partMap) + if len(incompleteParts) > 0 { + err = entry.loadDataPartsIntoCache(ctx, incompleteParts) + if err != nil { + return err + } + } + oldSize := entry.File.Size + entry.writeAt(entry.File.Size, data, false) + entry.writeAt(entry.File.Size, []byte("\n"), false) + if oldSize == 0 { + return nil + } + // check if we should compact + numCmds := metaIncrement(entry.File, IJsonNumCommands, 1) + numBytes := metaIncrement(entry.File, IJsonIncrementalBytes, len(data)+1) + incRatio := float64(numBytes) / float64(entry.File.Size) + if numCmds > IJsonHighCommands || incRatio >= IJsonHighRatio || (numCmds > IJsonLowCommands && incRatio >= IJsonLowRatio) { + err := s.compactIJson(ctx, entry) + if err != nil { + return err + } + } + return nil + }) +} + func (s *FileStore) GetAllZoneIds(ctx context.Context) ([]string, error) { return dbGetAllZoneIds(ctx) } diff --git a/pkg/filestore/blockstore_test.go b/pkg/filestore/blockstore_test.go index e5ba928ad..7743de430 100644 --- a/pkg/filestore/blockstore_test.go +++ b/pkg/filestore/blockstore_test.go @@ -10,12 +10,14 @@ import ( "fmt" "io/fs" "log" + "reflect" "sync" "sync/atomic" "testing" "time" "github.com/google/uuid" + "github.com/wavetermdev/thenextwave/pkg/ijson" ) func initDb(t *testing.T) { @@ -620,3 +622,129 @@ func TestConcurrentAppend(t *testing.T) { checkFileByteCount(t, ctx, zoneId, fileName, 'a', 100) checkFileByteCount(t, ctx, zoneId, fileName, 'e', 100) } + +func jsonDeepEqual(d1 any, d2 any) bool { + if d1 == nil && d2 == nil { + return true + } + if d1 == nil || d2 == nil { + return false + } + t1 := reflect.TypeOf(d1) + t2 := reflect.TypeOf(d2) + if t1 != t2 { + return false + } + switch d1.(type) { + case float64: + return d1.(float64) == d2.(float64) + case string: + return d1.(string) == d2.(string) + case bool: + return d1.(bool) == d2.(bool) + case []any: + a1 := d1.([]any) + a2 := d2.([]any) + if len(a1) != len(a2) { + return false + } + for i := 0; i < len(a1); i++ { + if !jsonDeepEqual(a1[i], a2[i]) { + return false + } + } + return true + case map[string]any: + m1 := d1.(map[string]any) + m2 := d2.(map[string]any) + if len(m1) != len(m2) { + return false + } + for k, v := range m1 { + if !jsonDeepEqual(v, m2[k]) { + return false + } + } + return true + default: + return false + } +} + +func TestIJson(t *testing.T) { + initDb(t) + defer cleanupDb(t) + ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelFn() + zoneId := uuid.NewString() + fileName := "ij1" + err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{IJson: true}) + if err != nil { + t.Fatalf("error creating file: %v", err) + } + rootSet := ijson.MakeSetCommand(nil, map[string]any{"tag": "div", "class": "root"}) + err = WFS.AppendIJson(ctx, zoneId, fileName, rootSet) + if err != nil { + t.Fatalf("error appending ijson: %v", err) + } + _, fullData, err := WFS.ReadFile(ctx, zoneId, fileName) + if err != nil { + t.Fatalf("error reading file: %v", err) + } + cmds, err := ijson.ParseIJson(fullData) + if err != nil { + t.Fatalf("error parsing ijson: %v", err) + } + outData, err := ijson.ApplyCommands(nil, cmds, 0) + if err != nil { + t.Fatalf("error applying ijson: %v", err) + } + if !jsonDeepEqual(rootSet["data"], outData) { + t.Errorf("data mismatch: expected %v, got %v", rootSet["data"], outData) + } + childrenAppend := ijson.MakeAppendCommand(ijson.Path{"children"}, map[string]any{"tag": "div", "class": "child"}) + err = WFS.AppendIJson(ctx, zoneId, fileName, childrenAppend) + if err != nil { + t.Fatalf("error appending ijson: %v", err) + } + _, fullData, err = WFS.ReadFile(ctx, zoneId, fileName) + if err != nil { + t.Fatalf("error reading file: %v", err) + } + cmds, err = ijson.ParseIJson(fullData) + if err != nil { + t.Fatalf("error parsing ijson: %v", err) + } + if len(cmds) != 2 { + t.Fatalf("command count mismatch: expected 2, got %d", len(cmds)) + } + outData, err = ijson.ApplyCommands(nil, cmds, 0) + if err != nil { + t.Fatalf("error applying ijson: %v", err) + } + if !jsonDeepEqual(ijson.M{"tag": "div", "class": "root", "children": ijson.A{ijson.M{"tag": "div", "class": "child"}}}, outData) { + t.Errorf("data mismatch: expected %v, got %v", rootSet["data"], outData) + } + err = WFS.CompactIJson(ctx, zoneId, fileName) + if err != nil { + t.Fatalf("error compacting ijson: %v", err) + } + _, fullData, err = WFS.ReadFile(ctx, zoneId, fileName) + if err != nil { + t.Fatalf("error reading file: %v", err) + } + cmds, err = ijson.ParseIJson(fullData) + if err != nil { + t.Fatalf("error parsing ijson: %v", err) + } + if len(cmds) != 1 { + t.Fatalf("command count mismatch: expected 1, got %d", len(cmds)) + } + outData, err = ijson.ApplyCommands(nil, cmds, 0) + if err != nil { + t.Fatalf("error applying ijson: %v", err) + } + if !jsonDeepEqual(ijson.M{"tag": "div", "class": "root", "children": ijson.A{ijson.M{"tag": "div", "class": "child"}}}, outData) { + t.Errorf("data mismatch: expected %v, got %v", rootSet["data"], outData) + } +} diff --git a/pkg/ijson/ijson.go b/pkg/ijson/ijson.go index bda7345b2..bb442f3ad 100644 --- a/pkg/ijson/ijson.go +++ b/pkg/ijson/ijson.go @@ -6,6 +6,7 @@ package ijson import ( "bytes" + "encoding/json" "fmt" "regexp" "strconv" @@ -22,11 +23,39 @@ const ( AppendCommandStr = "append" ) +type Command = map[string]any +type Path = []any +type M = map[string]any +type A = []any + // instead of defining structs for commands, we just define a command shape // set: type, path, value // del: type, path // arrayappend: type, path, value +func MakeSetCommand(path Path, value any) Command { + return Command{ + "type": SetCommandStr, + "path": path, + "data": value, + } +} + +func MakeDelCommand(path Path) Command { + return Command{ + "type": DelCommandStr, + "path": path, + } +} + +func MakeAppendCommand(path Path, value any) Command { + return Command{ + "type": AppendCommandStr, + "path": path, + "data": value, + } +} + type PathError struct { Err string } @@ -35,11 +64,11 @@ func (e PathError) Error() string { return "PathError: " + e.Err } -func MakePathTypeError(path []any, index int) error { +func MakePathTypeError(path Path, index int) error { return PathError{fmt.Sprintf("invalid path element type:%T at index:%d (%s)", path[index], index, FormatPath(path))} } -func MakePathError(errStr string, path []any, index int) error { +func MakePathError(errStr string, path Path, index int) error { return PathError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))} } @@ -51,7 +80,7 @@ func (e SetTypeError) Error() string { return "SetTypeError: " + e.Err } -func MakeSetTypeError(errStr string, path []any, index int) error { +func MakeSetTypeError(errStr string, path Path, index int) error { return SetTypeError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))} } @@ -63,13 +92,13 @@ func (e BudgetError) Error() string { return "BudgetError: " + e.Err } -func MakeBudgetError(errStr string, path []any, index int) error { +func MakeBudgetError(errStr string, path Path, index int) error { return BudgetError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))} } var simplePathStrRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`) -func FormatPath(path []any) string { +func FormatPath(path Path) string { if len(path) == 0 { return "$" } @@ -99,7 +128,7 @@ func FormatPath(path []any) string { } type pathWithPos struct { - Path []any + Path Path Index int } @@ -152,12 +181,12 @@ type SetPathOpts struct { CombineFn CombiningFunc } -func SetPathNoErr(data any, path []any, value any, opts *SetPathOpts) any { +func SetPathNoErr(data any, path Path, value any, opts *SetPathOpts) any { ret, _ := SetPath(data, path, value, opts) return ret } -func SetPath(data any, path []any, value any, opts *SetPathOpts) (any, error) { +func SetPath(data any, path Path, value any, opts *SetPathOpts) (any, error) { if opts == nil { opts = &SetPathOpts{} } @@ -464,7 +493,7 @@ func DeepEqual(v1 any, v2 any) bool { } } -func getCommandType(command map[string]any) string { +func getCommandType(command Command) string { typeVal, ok := command["type"] if !ok { return "" @@ -476,7 +505,7 @@ func getCommandType(command map[string]any) string { return typeStr } -func getCommandPath(command map[string]any) []any { +func getCommandPath(command Command) []any { pathVal, ok := command["path"] if !ok { return nil @@ -488,26 +517,119 @@ func getCommandPath(command map[string]any) []any { return path } -func ApplyCommand(data any, command any, budget int) (any, error) { - mapVal, ok := command.(map[string]any) - if !ok { - return nil, fmt.Errorf("ApplyCommand: expected map, but got %T", command) +func ValidatePath(path any) error { + if path == nil { + // nil path is allowed (sets the root) + return nil } - commandType := getCommandType(mapVal) + pathArr, ok := path.([]any) + if !ok { + return fmt.Errorf("path is not an array") + } + for idx, elem := range pathArr { + switch elem.(type) { + case string, int: + continue + default: + return fmt.Errorf("path element %d is not a string or int", idx) + } + } + return nil +} + +func ValidateAndMarshalCommand(command Command) ([]byte, error) { + cmdType := getCommandType(command) + if cmdType != SetCommandStr && cmdType != DelCommandStr && cmdType != AppendCommandStr { + return nil, fmt.Errorf("unknown ijson command type %q", cmdType) + } + path := getCommandPath(command) + err := ValidatePath(path) + if err != nil { + return nil, err + } + barr, err := json.Marshal(command) + if err != nil { + return nil, fmt.Errorf("error marshalling ijson command to json: %w", err) + } + return barr, nil +} + +func ApplyCommand(data any, command Command, budget int) (any, error) { + commandType := getCommandType(command) if commandType == "" { return nil, fmt.Errorf("ApplyCommand: missing type field") } switch commandType { case SetCommandStr: - path := getCommandPath(mapVal) - return SetPath(data, path, mapVal["data"], &SetPathOpts{Budget: budget}) + path := getCommandPath(command) + return SetPath(data, path, command["data"], &SetPathOpts{Budget: budget}) case DelCommandStr: - path := getCommandPath(mapVal) + path := getCommandPath(command) return SetPath(data, path, nil, &SetPathOpts{Remove: true, Budget: budget}) case AppendCommandStr: - path := getCommandPath(mapVal) - return SetPath(data, path, mapVal["data"], &SetPathOpts{CombineFn: CombineFn_ArrayAppend, Budget: budget}) + path := getCommandPath(command) + return SetPath(data, path, command["data"], &SetPathOpts{CombineFn: CombineFn_ArrayAppend, Budget: budget}) default: return nil, fmt.Errorf("ApplyCommand: unknown command type %q", commandType) } } + +func ApplyCommands(data any, commands []Command, budget int) (any, error) { + for _, command := range commands { + var err error + data, err = ApplyCommand(data, command, budget) + if err != nil { + return nil, err + } + } + return data, nil +} + +func CompactIJson(fullData []byte, budget int) ([]byte, error) { + var newData any + for len(fullData) > 0 { + nlIdx := bytes.IndexByte(fullData, '\n') + var cmdData []byte + if nlIdx == -1 { + cmdData = fullData + fullData = nil + } else { + cmdData = fullData[:nlIdx] + fullData = fullData[nlIdx+1:] + } + var cmdMap Command + err := json.Unmarshal(cmdData, &cmdMap) + if err != nil { + return nil, fmt.Errorf("error unmarshalling ijson command: %w", err) + } + newData, err = ApplyCommand(newData, cmdMap, budget) + if err != nil { + return nil, fmt.Errorf("error applying ijson command: %w", err) + } + } + newRootCmd := MakeSetCommand(nil, newData) + return json.Marshal(newRootCmd) +} + +// returns a list of commands +func ParseIJson(fullData []byte) ([]Command, error) { + var commands []Command + for len(fullData) > 0 { + nlIdx := bytes.IndexByte(fullData, '\n') + var cmdData []byte + if nlIdx == -1 { + cmdData = fullData + fullData = nil + } else { + cmdData = fullData[:nlIdx] + fullData = fullData[nlIdx+1:] + } + var cmdMap Command + err := json.Unmarshal(cmdData, &cmdMap) + if err != nil { + return nil, fmt.Errorf("error unmarshalling ijson command: %w", err) + } + commands = append(commands, cmdMap) + } + return commands, nil +} diff --git a/pkg/service/blockservice/blockservice.go b/pkg/service/blockservice/blockservice.go index 102555175..58b25e0e2 100644 --- a/pkg/service/blockservice/blockservice.go +++ b/pkg/service/blockservice/blockservice.go @@ -10,6 +10,7 @@ import ( "github.com/wavetermdev/thenextwave/pkg/blockcontroller" "github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta" + "github.com/wavetermdev/thenextwave/pkg/wshutil" ) type BlockService struct{} @@ -25,7 +26,7 @@ func (bs *BlockService) SendCommand_Meta() tsgenmeta.MethodMeta { } } -func (bs *BlockService) SendCommand(blockId string, cmd blockcontroller.BlockCommand) error { +func (bs *BlockService) SendCommand(blockId string, cmd wshutil.BlockCommand) error { if strings.HasPrefix(cmd.GetCommand(), "controller:") { bc := blockcontroller.GetBlockController(blockId) if bc == nil { diff --git a/pkg/service/service.go b/pkg/service/service.go index 936010f56..fc72b15ce 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -9,7 +9,6 @@ import ( "reflect" "strings" - "github.com/wavetermdev/thenextwave/pkg/blockcontroller" "github.com/wavetermdev/thenextwave/pkg/service/blockservice" "github.com/wavetermdev/thenextwave/pkg/service/clientservice" "github.com/wavetermdev/thenextwave/pkg/service/fileservice" @@ -17,6 +16,7 @@ import ( "github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta" "github.com/wavetermdev/thenextwave/pkg/waveobj" "github.com/wavetermdev/thenextwave/pkg/web/webcmd" + "github.com/wavetermdev/thenextwave/pkg/wshutil" "github.com/wavetermdev/thenextwave/pkg/wstore" ) @@ -36,7 +36,7 @@ var waveObjMapRType = reflect.TypeOf(map[string]waveobj.WaveObj{}) var methodMetaRType = reflect.TypeOf(tsgenmeta.MethodMeta{}) var waveObjUpdateRType = reflect.TypeOf(wstore.WaveObjUpdate{}) var uiContextRType = reflect.TypeOf((*wstore.UIContext)(nil)).Elem() -var blockCommandRType = reflect.TypeOf((*blockcontroller.BlockCommand)(nil)).Elem() +var blockCommandRType = reflect.TypeOf((*wshutil.BlockCommand)(nil)).Elem() var wsCommandRType = reflect.TypeOf((*webcmd.WSCommandType)(nil)).Elem() type WebCallType struct { @@ -100,7 +100,7 @@ func convertBlockCommand(argType reflect.Type, jsonArg any) (any, error) { if _, ok := jsonArg.(map[string]any); !ok { return nil, fmt.Errorf("cannot convert %T to %s", jsonArg, argType) } - cmd, err := blockcontroller.ParseCmdMap(jsonArg.(map[string]any)) + cmd, err := wshutil.ParseCmdMap(jsonArg.(map[string]any)) if err != nil { return nil, fmt.Errorf("error parsing command map: %w", err) } diff --git a/pkg/tsgen/tsgen.go b/pkg/tsgen/tsgen.go index 5669f7b4b..cbfdc883c 100644 --- a/pkg/tsgen/tsgen.go +++ b/pkg/tsgen/tsgen.go @@ -10,12 +10,12 @@ import ( "reflect" "strings" - "github.com/wavetermdev/thenextwave/pkg/blockcontroller" "github.com/wavetermdev/thenextwave/pkg/eventbus" "github.com/wavetermdev/thenextwave/pkg/service" "github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta" "github.com/wavetermdev/thenextwave/pkg/waveobj" "github.com/wavetermdev/thenextwave/pkg/web/webcmd" + "github.com/wavetermdev/thenextwave/pkg/wshutil" "github.com/wavetermdev/thenextwave/pkg/wstore" ) @@ -354,7 +354,7 @@ func GenerateServiceClass(serviceName string, serviceObj any, tsTypesMap map[ref } func GenerateWaveObjTypes(tsTypesMap map[reflect.Type]string) { - GenerateTSTypeUnion(blockcontroller.CommandTypeUnionMeta(), tsTypesMap) + GenerateTSTypeUnion(wshutil.CommandTypeUnionMeta(), tsTypesMap) GenerateTSTypeUnion(webcmd.WSCommandTypeUnionMeta(), tsTypesMap) GenerateTSType(reflect.TypeOf(waveobj.ORef{}), tsTypesMap) GenerateTSType(reflect.TypeOf((*waveobj.WaveObj)(nil)).Elem(), tsTypesMap) @@ -363,6 +363,7 @@ func GenerateWaveObjTypes(tsTypesMap map[reflect.Type]string) { GenerateTSType(reflect.TypeOf(service.WebReturnType{}), tsTypesMap) GenerateTSType(reflect.TypeOf(wstore.UIContext{}), tsTypesMap) GenerateTSType(reflect.TypeOf(eventbus.WSEventType{}), tsTypesMap) + GenerateTSType(reflect.TypeOf(eventbus.WSFileEventData{}), tsTypesMap) for _, rtype := range wstore.AllWaveObjTypes() { GenerateTSType(rtype, tsTypesMap) } diff --git a/pkg/web/ws.go b/pkg/web/ws.go index 86c156646..012b33562 100644 --- a/pkg/web/ws.go +++ b/pkg/web/ws.go @@ -15,10 +15,10 @@ import ( "github.com/google/uuid" "github.com/gorilla/mux" "github.com/gorilla/websocket" - "github.com/wavetermdev/thenextwave/pkg/blockcontroller" "github.com/wavetermdev/thenextwave/pkg/eventbus" "github.com/wavetermdev/thenextwave/pkg/service/blockservice" "github.com/wavetermdev/thenextwave/pkg/web/webcmd" + "github.com/wavetermdev/thenextwave/pkg/wshutil" ) const wsReadWaitTimeout = 15 * time.Second @@ -95,8 +95,8 @@ func processWSCommand(jmsg map[string]any, outputCh chan any) { } switch cmd := wsCommand.(type) { case *webcmd.SetBlockTermSizeWSCommand: - blockCmd := &blockcontroller.BlockInputCommand{ - Command: blockcontroller.BlockCommand_Input, + blockCmd := &wshutil.BlockInputCommand{ + Command: wshutil.BlockCommand_Input, TermSize: &cmd.TermSize, } blockservice.BlockServiceInstance.SendCommand(cmd.BlockId, blockCmd) diff --git a/pkg/wshutil/wshcommands.go b/pkg/wshutil/wshcommands.go index 53fc7b8f7..32d4da2b1 100644 --- a/pkg/wshutil/wshcommands.go +++ b/pkg/wshutil/wshcommands.go @@ -3,64 +3,135 @@ package wshutil -import "reflect" +import ( + "encoding/json" + "fmt" + "reflect" + + "github.com/wavetermdev/thenextwave/pkg/ijson" + "github.com/wavetermdev/thenextwave/pkg/shellexec" + "github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta" +) + +const CommandKey = "command" const ( - CommandSetView = "setview" - CommandSetMeta = "setmeta" - CommandBlockFileAppend = "blockfile:append" - CommandStreamFile = "streamfile" + BlockCommand_Message = "message" + BlockCommand_SetView = "setview" + BlockCommand_SetMeta = "setmeta" + BlockCommand_Input = "controller:input" + BlockCommand_AppendBlockFile = "blockfile:append" + BlockCommand_AppendIJson = "blockfile:appendijson" ) var CommandToTypeMap = map[string]reflect.Type{ - CommandSetView: reflect.TypeOf(SetViewCommand{}), - CommandSetMeta: reflect.TypeOf(SetMetaCommand{}), + BlockCommand_Input: reflect.TypeOf(BlockInputCommand{}), + BlockCommand_SetView: reflect.TypeOf(BlockSetViewCommand{}), + BlockCommand_SetMeta: reflect.TypeOf(BlockSetMetaCommand{}), + BlockCommand_Message: reflect.TypeOf(BlockMessageCommand{}), + BlockCommand_AppendBlockFile: reflect.TypeOf(BlockAppendFileCommand{}), + BlockCommand_AppendIJson: reflect.TypeOf(BlockAppendIJsonCommand{}), } -type Command interface { +func CommandTypeUnionMeta() tsgenmeta.TypeUnionMeta { + var rtypes []reflect.Type + for _, rtype := range CommandToTypeMap { + rtypes = append(rtypes, rtype) + } + return tsgenmeta.TypeUnionMeta{ + BaseType: reflect.TypeOf((*BlockCommand)(nil)).Elem(), + TypeFieldName: "command", + Types: rtypes, + } +} + +type baseCommand struct { + Command string `json:"command"` +} + +type BlockCommand interface { GetCommand() string } -// for unmarshalling -type baseCommand struct { - Command string `json:"command"` - RpcID string `json:"rpcid"` - RpcType string `json:"rpctype"` +type BlockCommandWrapper struct { + BlockCommand } -type SetViewCommand struct { - Command string `json:"command"` +func ParseCmdMap(cmdMap map[string]any) (BlockCommand, error) { + cmdType, ok := cmdMap[CommandKey].(string) + if !ok { + return nil, fmt.Errorf("no %s field in command map", CommandKey) + } + mapJson, err := json.Marshal(cmdMap) + if err != nil { + return nil, fmt.Errorf("error marshalling command map: %w", err) + } + rtype := CommandToTypeMap[cmdType] + if rtype == nil { + return nil, fmt.Errorf("unknown command type %q", cmdType) + } + cmd := reflect.New(rtype).Interface() + err = json.Unmarshal(mapJson, cmd) + if err != nil { + return nil, fmt.Errorf("error unmarshalling command: %w", err) + } + return cmd.(BlockCommand), nil +} + +type BlockInputCommand struct { + Command string `json:"command" tstype:"\"controller:input\""` + InputData64 string `json:"inputdata64,omitempty"` + SigName string `json:"signame,omitempty"` + TermSize *shellexec.TermSize `json:"termsize,omitempty"` +} + +func (ic *BlockInputCommand) GetCommand() string { + return BlockCommand_Input +} + +type BlockSetViewCommand struct { + Command string `json:"command" tstype:"\"setview\""` View string `json:"view"` } -func (svc *SetViewCommand) GetCommand() string { - return CommandSetView +func (svc *BlockSetViewCommand) GetCommand() string { + return BlockCommand_SetView } -type SetMetaCommand struct { - Command string `json:"command"` +type BlockSetMetaCommand struct { + Command string `json:"command" tstype:"\"setmeta\""` Meta map[string]any `json:"meta"` } -func (smc *SetMetaCommand) GetCommand() string { - return CommandSetMeta +func (smc *BlockSetMetaCommand) GetCommand() string { + return BlockCommand_SetMeta } -type BlockFileAppendCommand struct { - Command string `json:"command"` +type BlockMessageCommand struct { + Command string `json:"command" tstype:"\"message\""` + Message string `json:"message"` +} + +func (bmc *BlockMessageCommand) GetCommand() string { + return BlockCommand_Message +} + +type BlockAppendFileCommand struct { + Command string `json:"command" tstype:"\"blockfile:append\""` FileName string `json:"filename"` Data []byte `json:"data"` } -func (bfac *BlockFileAppendCommand) GetCommand() string { - return CommandBlockFileAppend +func (bwc *BlockAppendFileCommand) GetCommand() string { + return BlockCommand_AppendBlockFile } -type StreamFileCommand struct { - Command string `json:"command"` - FileName string `json:"filename"` +type BlockAppendIJsonCommand struct { + Command string `json:"command" tstype:"\"blockfile:appendijson\""` + FileName string `json:"filename"` + Data ijson.Command `json:"data"` } -func (c *StreamFileCommand) GetCommand() string { - return CommandStreamFile +func (bwc *BlockAppendIJsonCommand) GetCommand() string { + return BlockCommand_AppendIJson } diff --git a/pkg/wshutil/wshutil.go b/pkg/wshutil/wshutil.go index 615ccfdab..36eab91bf 100644 --- a/pkg/wshutil/wshutil.go +++ b/pkg/wshutil/wshutil.go @@ -25,7 +25,7 @@ var WaveOSCPrefixBytes = []byte(WaveOSCPrefix) // JSON = must escape all ASCII control characters ([\x00-\x1F\x7F]) // we can tell the difference between JSON and base64-JSON by the first character: '{' or not -func EncodeWaveOSCMessage(cmd Command) ([]byte, error) { +func EncodeWaveOSCMessage(cmd BlockCommand) ([]byte, error) { if cmd.GetCommand() == "" { return nil, fmt.Errorf("Command field not set in struct") } @@ -74,7 +74,7 @@ func EncodeWaveOSCMessage(cmd Command) ([]byte, error) { return buf.Bytes(), nil } -func decodeWaveOSCMessage(data []byte) (Command, error) { +func decodeWaveOSCMessage(data []byte) (BlockCommand, error) { var baseCmd baseCommand err := json.Unmarshal(data, &baseCmd) if err != nil { @@ -85,12 +85,12 @@ func decodeWaveOSCMessage(data []byte) (Command, error) { if err != nil { return nil, fmt.Errorf("error unmarshalling json: %w", err) } - return rtnCmd.(Command), nil + return rtnCmd.(BlockCommand), nil } // data does not contain the escape sequence, just the innards // this function implements the switch between JSON and base64-JSON -func DecodeWaveOSCMessage(data []byte) (Command, error) { +func DecodeWaveOSCMessage(data []byte) (BlockCommand, error) { if len(data) == 0 { return nil, fmt.Errorf("empty data") } diff --git a/pkg/wstore/wstore_dbops.go b/pkg/wstore/wstore_dbops.go index 43e619845..5ab036027 100644 --- a/pkg/wstore/wstore_dbops.go +++ b/pkg/wstore/wstore_dbops.go @@ -6,7 +6,9 @@ package wstore import ( "context" "fmt" + "log" + "github.com/wavetermdev/thenextwave/pkg/filestore" "github.com/wavetermdev/thenextwave/pkg/waveobj" "github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil" ) @@ -167,13 +169,21 @@ func DBSelectMap[T waveobj.WaveObj](ctx context.Context, ids []string) (map[stri } func DBDelete(ctx context.Context, otype string, id string) error { - return WithTx(ctx, func(tx *TxWrap) error { + err := WithTx(ctx, func(tx *TxWrap) error { table := tableNameFromOType(otype) query := fmt.Sprintf("DELETE FROM %s WHERE oid = ?", table) tx.Exec(query, id) ContextAddUpdate(ctx, WaveObjUpdate{UpdateType: UpdateType_Delete, OType: otype, OID: id}) return nil }) + if err != nil { + return err + } + err = filestore.WFS.DeleteZone(ctx, id) + if err != nil { + log.Printf("error deleting filestore zone (after deleting block): %v", err) + } + return nil } func DBUpdate(ctx context.Context, val waveobj.WaveObj) error { diff --git a/pkg/wstore/wstore_types.go b/pkg/wstore/wstore_types.go index d408f81ee..d1bed55b2 100644 --- a/pkg/wstore/wstore_types.go +++ b/pkg/wstore/wstore_types.go @@ -130,6 +130,7 @@ type Window struct { Version int `json:"version"` WorkspaceId string `json:"workspaceid"` ActiveTabId string `json:"activetabid"` + ActiveBlockId string `json:"activeblockid,omitempty"` ActiveBlockMap map[string]string `json:"activeblockmap"` // map from tabid to blockid Pos Point `json:"pos"` WinSize WinSize `json:"winsize"` diff --git a/yarn.lock b/yarn.lock index 043afb141..21835f500 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11015,6 +11015,17 @@ __metadata: languageName: node linkType: hard +"react-frame-component@npm:^5.2.7": + version: 5.2.7 + resolution: "react-frame-component@npm:5.2.7" + peerDependencies: + prop-types: ^15.5.9 + react: ">= 16.3" + react-dom: ">= 16.3" + checksum: 10c0/e138602aa98557c021ae825f51468026c53b9939140c5961d5371b65ad07ff9a5adaf2cd4e4a8a77414a05ae0f95a842939c8e102aa576ef21ff096368b905a3 + languageName: node + linkType: hard + "react-is@npm:18.1.0": version: 18.1.0 resolution: "react-is@npm:18.1.0" @@ -12277,6 +12288,7 @@ __metadata: react-dnd: "npm:^16.0.1" react-dnd-html5-backend: "npm:^16.0.1" react-dom: "npm:^18.3.1" + react-frame-component: "npm:^5.2.7" react-markdown: "npm:^9.0.1" remark-gfm: "npm:^4.0.0" rxjs: "npm:^7.8.1"