working on ijson and wsh magic (#53)

cmd/wsh/main-wsh.go

@@ -5,14 +5,74 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
+	"golang.org/x/term"
 )
 
+var shutdownOnce sync.Once
+var origTermState *term.State
+
+func doShutdown(reason string, exitCode int) {
+	shutdownOnce.Do(func() {
+		defer os.Exit(exitCode)
+		log.Printf("shutting down: %s\r\n", reason)
+		cmd := &wshutil.BlockSetMetaCommand{
+			Command: wshutil.BlockCommand_SetMeta,
+			Meta:    map[string]any{"term:mode": nil},
+		}
+		barr, _ := wshutil.EncodeWaveOSCMessage(cmd)
+		if origTermState != nil {
+			term.Restore(int(os.Stdin.Fd()), origTermState)
+		}
+		os.Stdout.Write(barr)
+	})
+}
+
+func installShutdownSignalHandlers() {
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT)
+	go func() {
+		for sig := range sigCh {
+			doShutdown(fmt.Sprintf("got signal %v", sig), 1)
+			break
+		}
+	}()
+}
+
 func main() {
-	barr, err := os.ReadFile("/Users/mike/Downloads/2.png")
+	installShutdownSignalHandlers()
+	defer doShutdown("normal exit", 0)
+	origState, err := term.MakeRaw(int(os.Stdin.Fd()))
 	if err != nil {
-		fmt.Println("error reading file:", err)
+		fmt.Fprintf(os.Stderr, "Error setting raw mode: %v\n", err)
 		return
 	}
-	fmt.Println("file size:", len(barr))
+	origTermState = origState
+	cmd := &wshutil.BlockSetMetaCommand{
+		Command: wshutil.BlockCommand_SetMeta,
+		Meta:    map[string]any{"term:mode": "html"},
+	}
+	barr, _ := wshutil.EncodeWaveOSCMessage(cmd)
+	os.Stdout.Write(barr)
+	for {
+		var buf [1]byte
+		_, err := os.Stdin.Read(buf[:])
+		if err != nil {
+			doShutdown(fmt.Sprintf("stdin closed/error (%v)", err), 1)
+		}
+		if buf[0] == 0x03 {
+			doShutdown("read Ctrl-C from stdin", 1)
+			break
+		}
+		if buf[0] == 'x' {
+			doShutdown("read 'x' from stdin", 0)
+			break
+		}
+	}
 }

emain/emain.ts

@@ -124,6 +124,11 @@ function mainResizeHandler(_: any, win: Electron.BrowserWindow) {
 }
 
 function shNavHandler(event: Electron.Event<Electron.WebContentsWillNavigateEventParams>, url: string) {
+    if (url.startsWith("http://localhost:5173/index.html")) {
+        // this is a dev-mode hot-reload, ignore it
+        console.log("allowing hot-reload of index.html");
+        return;
+    }
     event.preventDefault();
     if (url.startsWith("https://") || url.startsWith("http://") || url.startsWith("file://")) {
         console.log("open external, shNav", url);

frontend/app/block/block.tsx

@@ -49,7 +49,7 @@ const Block = ({ blockId, onClose }: BlockProps) => {
     } else if (blockData.view === "plot") {
         blockElem = <PlotView />;
     } else if (blockData.view === "codeedit") {
-        blockElem = <CodeEdit text={null} />;
+        blockElem = <CodeEdit text={null} filename={null} />;
     }
     return (
         <div className="block" ref={blockRef}>

frontend/app/store/global.ts

@@ -66,6 +66,7 @@ type SubjectWithRef<T> = rxjs.Subject<T> & { refCount: number; release: () => vo
 
 // key is "eventType" or "eventType|oref"
 const eventSubjects = new Map<string, SubjectWithRef<WSEventType>>();
+const fileSubjects = new Map<string, SubjectWithRef<WSFileEventData>>();
 
 function getSubjectInternal(subjectKey: string): SubjectWithRef<WSEventType> {
     let subject = eventSubjects.get(subjectKey);
@@ -93,6 +94,25 @@ function getEventORefSubject(eventType: string, oref: string): SubjectWithRef<WS
     return getSubjectInternal(eventType + "|" + oref);
 }
 
+function getFileSubject(zoneId: string, fileName: string): SubjectWithRef<WSFileEventData> {
+    const subjectKey = zoneId + "|" + fileName;
+    let subject = fileSubjects.get(subjectKey);
+    if (subject == null) {
+        subject = new rxjs.Subject<any>() as any;
+        subject.refCount = 0;
+        subject.release = () => {
+            subject.refCount--;
+            if (subject.refCount === 0) {
+                subject.complete();
+                fileSubjects.delete(subjectKey);
+            }
+        };
+        fileSubjects.set(subjectKey, subject);
+    }
+    subject.refCount++;
+    return subject;
+}
+
 const blockCache = new Map<string, Map<string, any>>();
 
 function useBlockCache<T>(blockId: string, name: string, makeFn: () => T): T {
@@ -142,6 +162,15 @@ function handleWSEventMessage(msg: WSEventType) {
         console.log("unsupported event", msg);
         return;
     }
+    if (msg.eventtype == "blockfile") {
+        const fileData: WSFileEventData = msg.data;
+        const fileSubject = getFileSubject(fileData.zoneid, fileData.filename);
+        if (fileSubject != null) {
+            fileSubject.next(fileData);
+        }
+        return;
+    }
+
     // we send to two subjects just eventType and eventType|oref
     // we don't use getORefSubject here because we don't want to create a new subject
     const eventSubject = eventSubjects.get(msg.eventtype);
@@ -193,6 +222,7 @@ export {
     getBackendHostPort,
     getEventORefSubject,
     getEventSubject,
+    getFileSubject,
     globalStore,
     globalWS,
     initWS,

frontend/app/view/ijson.tsx

@@ -0,0 +1,117 @@
+// Copyright 2024, Command Line Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+import * as React from "react";
+import Frame from "react-frame-component";
+
+type IJsonNode = {
+    tag: string;
+    props?: Record<string, any>;
+    children?: (IJsonNode | string)[];
+};
+
+const TagMap: Record<string, React.ComponentType<{ node: IJsonNode }>> = {};
+
+function convertNodeToTag(node: IJsonNode | string, idx?: number): JSX.Element | string {
+    if (node == null) {
+        return null;
+    }
+    if (idx == null) {
+        idx = 0;
+    }
+    if (typeof node === "string") {
+        return node;
+    }
+    let key = node.props?.key ?? "child-" + idx;
+    let TagComp = TagMap[node.tag];
+    if (!TagComp) {
+        return <div key={key}>Unknown tag:{node.tag}</div>;
+    }
+    return <TagComp key={key} node={node} />;
+}
+
+function IJsonHtmlTag({ node }: { node: IJsonNode }) {
+    let { tag, props, children } = node;
+    let divProps = {};
+    if (props != null) {
+        for (let [key, val] of Object.entries(props)) {
+            if (key.startsWith("on")) {
+                divProps[key] = (e: any) => {
+                    console.log("handler", key, val);
+                };
+            } else {
+                divProps[key] = val;
+            }
+        }
+    }
+    let childrenComps: (string | JSX.Element)[] = [];
+    if (children != null) {
+        for (let idx = 0; idx < children.length; idx++) {
+            let comp = convertNodeToTag(children[idx], idx);
+            if (comp != null) {
+                childrenComps.push(comp);
+            }
+        }
+    }
+    return React.createElement(tag, divProps, childrenComps);
+}
+
+TagMap["div"] = IJsonHtmlTag;
+TagMap["b"] = IJsonHtmlTag;
+TagMap["i"] = IJsonHtmlTag;
+TagMap["p"] = IJsonHtmlTag;
+TagMap["s"] = IJsonHtmlTag;
+TagMap["span"] = IJsonHtmlTag;
+TagMap["a"] = IJsonHtmlTag;
+TagMap["img"] = IJsonHtmlTag;
+TagMap["h1"] = IJsonHtmlTag;
+TagMap["h2"] = IJsonHtmlTag;
+TagMap["h3"] = IJsonHtmlTag;
+TagMap["h4"] = IJsonHtmlTag;
+TagMap["h5"] = IJsonHtmlTag;
+TagMap["h6"] = IJsonHtmlTag;
+TagMap["ul"] = IJsonHtmlTag;
+TagMap["ol"] = IJsonHtmlTag;
+TagMap["li"] = IJsonHtmlTag;
+TagMap["input"] = IJsonHtmlTag;
+TagMap["button"] = IJsonHtmlTag;
+TagMap["textarea"] = IJsonHtmlTag;
+TagMap["select"] = IJsonHtmlTag;
+TagMap["option"] = IJsonHtmlTag;
+TagMap["form"] = IJsonHtmlTag;
+
+function IJsonView({ rootNode }: { rootNode: IJsonNode }) {
+    // TODO fix this huge inline style
+    return (
+        <div className="ijson">
+            <Frame>
+                <style>
+                    {`
+*::before, *::after { box-sizing: border-box; }
+* { margin: 0; }
+body { line-height: 1.2; -webkit-font-smoothing: antialiased; }
+img, picture, video, canvas, sgv { display: block; }
+input, button, textarea, select { font: inherit; }
+
+body {
+	display: flex;
+	flex-direction: column;
+	width: 100vw;
+	height: 100vh;
+	background-color: #000;
+	color: #fff;
+	font: normal 15px / normal "Lato", sans-serif;
+}
+
+.fixed-font {
+	normal 12px / normal "Hack", monospace;
+}
+					`}
+                </style>
+                {convertNodeToTag(rootNode)}
+            </Frame>
+        </div>
+    );
+}
+
+export { IJsonView };

frontend/app/view/term.tsx

@@ -1,14 +1,25 @@
 // Copyright 2024, Command Line Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-import { WOS, getBackendHostPort, getEventORefSubject, sendWSCommand } from "@/store/global";
+import {
+    WOS,
+    atoms,
+    getBackendHostPort,
+    getFileSubject,
+    globalStore,
+    sendWSCommand,
+    useBlockAtom,
+} from "@/store/global";
 import * as services from "@/store/services";
 import { base64ToArray } from "@/util/util";
 import { FitAddon } from "@xterm/addon-fit";
 import type { ITheme } from "@xterm/xterm";
 import { Terminal } from "@xterm/xterm";
 import clsx from "clsx";
+import { produce } from "immer";
+import * as jotai from "jotai";
 import * as React from "react";
+import { IJsonView } from "./ijson";
 
 import "public/xterm.css";
 import { debounce } from "throttle-debounce";
@@ -59,17 +70,96 @@ function handleResize(fitAddon: FitAddon, blockId: string, term: Terminal) {
     }
 }
 
+const keyMap = {
+    Enter: "\r",
+    Backspace: "\x7f",
+    Tab: "\t",
+    Escape: "\x1b",
+    ArrowUp: "\x1b[A",
+    ArrowDown: "\x1b[B",
+    ArrowRight: "\x1b[C",
+    ArrowLeft: "\x1b[D",
+    Insert: "\x1b[2~",
+    Delete: "\x1b[3~",
+    Home: "\x1b[1~",
+    End: "\x1b[4~",
+    PageUp: "\x1b[5~",
+    PageDown: "\x1b[6~",
+};
+
+function keyboardEventToASCII(event: React.KeyboardEvent<HTMLInputElement>): string {
+    // check modifiers
+    // if no modifiers are set, just send the key
+    if (!event.altKey && !event.ctrlKey && !event.metaKey) {
+        if (event.key == null || event.key == "") {
+            return "";
+        }
+        if (keyMap[event.key] != null) {
+            return keyMap[event.key];
+        }
+        if (event.key.length == 1) {
+            return event.key;
+        } else {
+            console.log("not sending keyboard event", event.key, event);
+        }
+    }
+    // if meta or alt is set, there is no ASCII representation
+    if (event.metaKey || event.altKey) {
+        return "";
+    }
+    // if ctrl is set, if it is a letter, subtract 64 from the uppercase value to get the ASCII value
+    if (event.ctrlKey) {
+        if (
+            (event.key.length === 1 && event.key >= "A" && event.key <= "Z") ||
+            (event.key >= "a" && event.key <= "z")
+        ) {
+            const key = event.key.toUpperCase();
+            return String.fromCharCode(key.charCodeAt(0) - 64);
+        }
+    }
+    return "";
+}
+
 type InitialLoadDataType = {
     loaded: boolean;
     heldData: Uint8Array[];
 };
 
+const IJSONConst = {
+    tag: "div",
+    children: [
+        {
+            tag: "h1",
+            children: ["Hello World"],
+        },
+        {
+            tag: "p",
+            children: ["This is a paragraph"],
+        },
+    ],
+};
+
+function setBlockFocus(blockId: string) {
+    let winData = globalStore.get(atoms.waveWindow);
+    winData = produce(winData, (draft) => {
+        draft.activeblockid = blockId;
+    });
+    WOS.setObjectValue(winData, globalStore.set, true);
+}
+
 const TerminalView = ({ blockId }: { blockId: string }) => {
     const connectElemRef = React.useRef<HTMLDivElement>(null);
     const termRef = React.useRef<Terminal>(null);
     const initialLoadRef = React.useRef<InitialLoadDataType>({ loaded: false, heldData: [] });
     const htmlElemFocusRef = React.useRef<HTMLInputElement>(null);
     const [blockData] = WOS.useWaveObjectValue<Block>(WOS.makeORef("block", blockId));
+    const isFocusedAtom = useBlockAtom<boolean>(blockId, "isFocused", () => {
+        return jotai.atom((get) => {
+            const winData = get(atoms.waveWindow);
+            return winData.activeblockid === blockId;
+        });
+    });
+    const isFocused = jotai.useAtomValue(isFocusedAtom);
     React.useEffect(() => {
         console.log("terminal created");
         const newTerm = new Terminal({
@@ -95,13 +185,16 @@ const TerminalView = ({ blockId }: { blockId: string }) => {
             const inputCmd: BlockInputCommand = { command: "controller:input", inputdata64: b64data };
             services.BlockService.SendCommand(blockId, inputCmd);
         });
-
+        newTerm.textarea.addEventListener("focus", () => {
-        // block subject
+            setBlockFocus(blockId);
-        const blockSubject = getEventORefSubject("block:ptydata", WOS.makeORef("block", blockId));
+        });
-        blockSubject.subscribe((msg: WSEventType) => {
+        const mainFileSubject = getFileSubject(blockId, "main");
-            // base64 decode
+        mainFileSubject.subscribe((msg: WSFileEventData) => {
-            const data = msg.data;
+            if (msg.fileop != "append") {
-            const decodedData = base64ToArray(data.ptydata);
+                console.log("bad fileop for terminal", msg);
+                return;
+            }
+            const decodedData = base64ToArray(msg.data64);
             if (initialLoadRef.current.loaded) {
                 newTerm.write(decodedData);
             } else {
@@ -146,7 +239,7 @@ const TerminalView = ({ blockId }: { blockId: string }) => {
 
         return () => {
             newTerm.dispose();
-            blockSubject.release();
+            mainFileSubject.release();
         };
     }, []);
 
@@ -157,6 +250,13 @@ const TerminalView = ({ blockId }: { blockId: string }) => {
             services.BlockService.SendCommand(blockId, metaCmd);
             return false;
         }
+        const asciiVal = keyboardEventToASCII(event);
+        if (asciiVal.length == 0) {
+            return false;
+        }
+        const b64data = btoa(asciiVal);
+        const inputCmd: BlockInputCommand = { command: "controller:input", inputdata64: b64data };
+        services.BlockService.SendCommand(blockId, inputCmd);
         return true;
     };
 
@@ -164,8 +264,18 @@ const TerminalView = ({ blockId }: { blockId: string }) => {
     if (termMode != "term" && termMode != "html") {
         termMode = "term";
     }
+
+    React.useEffect(() => {
+        if (isFocused && termMode == "term") {
+            termRef.current?.focus();
+        }
+        if (isFocused && termMode == "html") {
+            htmlElemFocusRef.current?.focus();
+        }
+    });
+
     return (
-        <div className={clsx("view-term", "term-mode-" + termMode)}>
+        <div className={clsx("view-term", "term-mode-" + termMode, isFocused ? "is-focused" : null)}>
             <div key="conntectElem" className="term-connectelem" ref={connectElemRef}></div>
             <div
                 key="htmlElem"
@@ -174,13 +284,20 @@ const TerminalView = ({ blockId }: { blockId: string }) => {
                     if (htmlElemFocusRef.current != null) {
                         htmlElemFocusRef.current.focus();
                     }
+                    setBlockFocus(blockId);
                 }}
             >
                 <div key="htmlElemFocus" className="term-htmlelem-focus">
-                    <input type="text" ref={htmlElemFocusRef} onKeyDown={handleKeyDown} />
+                    <input
+                        type="text"
+                        value={""}
+                        ref={htmlElemFocusRef}
+                        onKeyDown={handleKeyDown}
+                        onChange={() => {}}
+                    />
                 </div>
                 <div key="htmlElemContent" className="term-htmlelem-content">
-                    HTML MODE
+                    <IJsonView rootNode={IJSONConst} />
                 </div>
             </div>
         </div>

frontend/app/view/view.less

@@ -77,6 +77,12 @@
         .term-htmlelem {
             display: flex;
         }
+
+        .ijson iframe {
+            width: 100%;
+            height: 100%;
+            border: none;
+        }
     }
 }
 

frontend/types/gotypes.d.ts

@@ -14,9 +14,23 @@ declare global {
         meta: MetaType;
     };
 
+    // wshutil.BlockAppendFileCommand
+    type BlockAppendFileCommand = {
+        command: "blockfile:append";
+        filename: string;
+        data: number[];
+    };
+
+    // wshutil.BlockAppendIJsonCommand
+    type BlockAppendIJsonCommand = {
+        command: "blockfile:appendijson";
+        filename: string;
+        data: MetaType;
+    };
+
     type BlockCommand = {
         command: string;
-    } & ( BlockInputCommand | BlockSetViewCommand | BlockSetMetaCommand );
+    } & ( BlockAppendIJsonCommand | BlockInputCommand | BlockSetViewCommand | BlockSetMetaCommand | BlockMessageCommand | BlockAppendFileCommand );
 
     // wstore.BlockDef
     type BlockDef = {
@@ -26,7 +40,7 @@ declare global {
         meta?: MetaType;
     };
 
-    // blockcontroller.BlockInputCommand
+    // wshutil.BlockInputCommand
     type BlockInputCommand = {
         command: "controller:input";
         inputdata64?: string;
@@ -34,13 +48,19 @@ declare global {
         termsize?: TermSize;
     };
 
-    // blockcontroller.BlockSetMetaCommand
+    // wshutil.BlockMessageCommand
+    type BlockMessageCommand = {
+        command: "message";
+        message: string;
+    };
+
+    // wshutil.BlockSetMetaCommand
     type BlockSetMetaCommand = {
         command: "setmeta";
         meta: MetaType;
     };
 
-    // blockcontroller.BlockSetViewCommand
+    // wshutil.BlockSetViewCommand
     type BlockSetViewCommand = {
         command: "setview";
         view: string;
@@ -149,6 +169,14 @@ declare global {
         data: any;
     };
 
+    // eventbus.WSFileEventData
+    type WSFileEventData = {
+        zoneid: string;
+        filename: string;
+        fileop: string;
+        data64: string;
+    };
+
     // waveobj.WaveObj
     type WaveObj = {
         otype: string;
@@ -190,6 +218,7 @@ declare global {
     type WaveWindow = WaveObj & {
         workspaceid: string;
         activetabid: string;
+        activeblockid?: string;
         activeblockmap: {[key: string]: string};
         pos: Point;
         winsize: WinSize;

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/sawka/txwrap v0.2.0
 	github.com/wavetermdev/waveterm/wavesrv v0.0.0-20240508181017-d07068c09d94
 	golang.org/x/sys v0.20.0
+	golang.org/x/term v0.17.0
 )
 
 require (

go.sum

@@ -46,5 +46,7 @@ go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

package.json

@@ -68,6 +68,7 @@
         "react-dnd": "^16.0.1",
         "react-dnd-html5-backend": "^16.0.1",
         "react-dom": "^18.3.1",
+        "react-frame-component": "^5.2.7",
         "react-markdown": "^9.0.1",
         "remark-gfm": "^4.0.0",
         "rxjs": "^7.8.1",

pkg/blockcontroller/blockcommand.go

@@ -1,109 +0,0 @@
-// Copyright 2024, Command Line Inc.
-// SPDX-License-Identifier: Apache-2.0
-
-package blockcontroller
-
-import (
-	"encoding/json"
-	"fmt"
-	"reflect"
-
-	"github.com/wavetermdev/thenextwave/pkg/shellexec"
-	"github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta"
-)
-
-const CommandKey = "command"
-
-const (
-	BlockCommand_Message = "message"
-	BlockCommand_SetView = "setview"
-	BlockCommand_SetMeta = "setmeta"
-	BlockCommand_Input   = "controller:input"
-)
-
-var CommandToTypeMap = map[string]reflect.Type{
-	BlockCommand_Input:   reflect.TypeOf(BlockInputCommand{}),
-	BlockCommand_SetView: reflect.TypeOf(BlockSetViewCommand{}),
-	BlockCommand_SetMeta: reflect.TypeOf(BlockSetMetaCommand{}),
-	BlockCommand_Message: reflect.TypeOf(BlockMessageCommand{}),
-}
-
-func CommandTypeUnionMeta() tsgenmeta.TypeUnionMeta {
-	return tsgenmeta.TypeUnionMeta{
-		BaseType:      reflect.TypeOf((*BlockCommand)(nil)).Elem(),
-		TypeFieldName: "command",
-		Types: []reflect.Type{
-			reflect.TypeOf(BlockInputCommand{}),
-			reflect.TypeOf(BlockSetViewCommand{}),
-			reflect.TypeOf(BlockSetMetaCommand{}),
-			reflect.TypeOf(BlockMessageCommand{}),
-		},
-	}
-}
-
-type BlockCommand interface {
-	GetCommand() string
-}
-
-type BlockCommandWrapper struct {
-	BlockCommand
-}
-
-func ParseCmdMap(cmdMap map[string]any) (BlockCommand, error) {
-	cmdType, ok := cmdMap[CommandKey].(string)
-	if !ok {
-		return nil, fmt.Errorf("no %s field in command map", CommandKey)
-	}
-	mapJson, err := json.Marshal(cmdMap)
-	if err != nil {
-		return nil, fmt.Errorf("error marshalling command map: %w", err)
-	}
-	rtype := CommandToTypeMap[cmdType]
-	if rtype == nil {
-		return nil, fmt.Errorf("unknown command type %q", cmdType)
-	}
-	cmd := reflect.New(rtype).Interface()
-	err = json.Unmarshal(mapJson, cmd)
-	if err != nil {
-		return nil, fmt.Errorf("error unmarshalling command: %w", err)
-	}
-	return cmd.(BlockCommand), nil
-}
-
-type BlockInputCommand struct {
-	Command     string              `json:"command" tstype:"\"controller:input\""`
-	InputData64 string              `json:"inputdata64,omitempty"`
-	SigName     string              `json:"signame,omitempty"`
-	TermSize    *shellexec.TermSize `json:"termsize,omitempty"`
-}
-
-func (ic *BlockInputCommand) GetCommand() string {
-	return BlockCommand_Input
-}
-
-type BlockSetViewCommand struct {
-	Command string `json:"command" tstype:"\"setview\""`
-	View    string `json:"view"`
-}
-
-func (svc *BlockSetViewCommand) GetCommand() string {
-	return BlockCommand_SetView
-}
-
-type BlockSetMetaCommand struct {
-	Command string         `json:"command" tstype:"\"setmeta\""`
-	Meta    map[string]any `json:"meta"`
-}
-
-func (smc *BlockSetMetaCommand) GetCommand() string {
-	return BlockCommand_SetMeta
-}
-
-type BlockMessageCommand struct {
-	Command string `json:"command" tstype:"\"message\""`
-	Message string `json:"message"`
-}
-
-func (bmc *BlockMessageCommand) GetCommand() string {
-	return BlockCommand_Message
-}

pkg/blockcontroller/blockcontroller.go

@@ -20,6 +20,7 @@ import (
 	"github.com/wavetermdev/thenextwave/pkg/filestore"
 	"github.com/wavetermdev/thenextwave/pkg/shellexec"
 	"github.com/wavetermdev/thenextwave/pkg/waveobj"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
 	"github.com/wavetermdev/thenextwave/pkg/wstore"
 )
 
@@ -28,6 +29,11 @@ const (
 	BlockController_Cmd   = "cmd"
 )
 
+const (
+	BlockFile_Main = "main" // used for main pty output
+	BlockFile_Html = "html" // used for alt html layout
+)
+
 const DefaultTimeout = 2 * time.Second
 
 var globalLock = &sync.Mutex{}
@@ -37,12 +43,13 @@ type BlockController struct {
 	Lock            *sync.Mutex
 	BlockId         string
 	BlockDef        *wstore.BlockDef
-	InputCh  chan BlockCommand
+	InputCh         chan wshutil.BlockCommand
 	Status          string
+	CreatedHtmlFile bool
 
 	PtyBuffer    *PtyBuffer
 	ShellProc    *shellexec.ShellProc
-	ShellInputCh chan *BlockInputCommand
+	ShellInputCh chan *wshutil.BlockInputCommand
 }
 
 func (bc *BlockController) WithLock(f func()) {
@@ -91,21 +98,49 @@ func (bc *BlockController) Close() {
 }
 
 const DefaultTermMaxFileSize = 256 * 1024
+const DefaultHtmlMaxFileSize = 256 * 1024
 
-func (bc *BlockController) handleShellProcData(data []byte) error {
+func handleAppendBlockFile(blockId string, blockFile string, data []byte) error {
 	ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout)
 	defer cancelFn()
-	err := filestore.WFS.AppendData(ctx, bc.BlockId, "main", data)
+	err := filestore.WFS.AppendData(ctx, blockId, blockFile, data)
 	if err != nil {
 		return fmt.Errorf("error appending to blockfile: %w", err)
 	}
 	eventbus.SendEvent(eventbus.WSEventType{
-		EventType: "block:ptydata",
+		EventType: "blockfile",
-		ORef:      waveobj.MakeORef(wstore.OType_Block, bc.BlockId).String(),
+		ORef:      waveobj.MakeORef(wstore.OType_Block, blockId).String(),
-		Data: map[string]any{
+		Data: &eventbus.WSFileEventData{
-			"blockid":   bc.BlockId,
+			ZoneId:   blockId,
-			"blockfile": "main",
+			FileName: blockFile,
-			"ptydata":   base64.StdEncoding.EncodeToString(data),
+			FileOp:   eventbus.FileOp_Append,
+			Data64:   base64.StdEncoding.EncodeToString(data),
+		},
+	})
+	return nil
+}
+
+func handleAppendIJsonFile(blockId string, blockFile string, cmd map[string]any, tryCreate bool) error {
+	ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout)
+	defer cancelFn()
+	if blockFile == BlockFile_Html && tryCreate {
+		err := filestore.WFS.MakeFile(ctx, blockId, blockFile, nil, filestore.FileOptsType{MaxSize: DefaultHtmlMaxFileSize, IJson: true})
+		if err != nil && err != filestore.ErrAlreadyExists {
+			return fmt.Errorf("error creating blockfile[html]: %w", err)
+		}
+	}
+	err := filestore.WFS.AppendIJson(ctx, blockId, blockFile, cmd)
+	if err != nil {
+		return fmt.Errorf("error appending to blockfile(ijson): %w", err)
+	}
+	eventbus.SendEvent(eventbus.WSEventType{
+		EventType: "blockfile",
+		ORef:      waveobj.MakeORef(wstore.OType_Block, blockId).String(),
+		Data: &eventbus.WSFileEventData{
+			ZoneId:   blockId,
+			FileName: blockFile,
+			FileOp:   eventbus.FileOp_Append,
+			Data64:   base64.StdEncoding.EncodeToString([]byte("{}")),
 		},
 	})
 	return nil
@@ -150,7 +185,7 @@ func (bc *BlockController) DoRunShellCommand(rc *RunShellOpts) error {
 		bc.ShellProc.Close()
 		return err
 	}
-	shellInputCh := make(chan *BlockInputCommand)
+	shellInputCh := make(chan *wshutil.BlockInputCommand)
 	bc.ShellInputCh = shellInputCh
 	go func() {
 		defer func() {
@@ -233,7 +268,7 @@ func (bc *BlockController) Run(bdata *wstore.Block) {
 
 	for genCmd := range bc.InputCh {
 		switch cmd := genCmd.(type) {
-		case *BlockInputCommand:
+		case *wshutil.BlockInputCommand:
 			log.Printf("INPUT: %s | %q\n", bc.BlockId, cmd.InputData64)
 			if bc.ShellInputCh != nil {
 				bc.ShellInputCh <- cmd
@@ -266,9 +301,11 @@ func StartBlockController(ctx context.Context, blockId string) error {
 		Lock:    &sync.Mutex{},
 		BlockId: blockId,
 		Status:  "init",
-		InputCh: make(chan BlockCommand),
+		InputCh: make(chan wshutil.BlockCommand),
 	}
-	ptyBuffer := MakePtyBuffer(bc.handleShellProcData, func(cmd BlockCommand) error {
+	ptyBuffer := MakePtyBuffer(func(fileName string, data []byte) error {
+		return handleAppendBlockFile(blockId, fileName, data)
+	}, func(cmd wshutil.BlockCommand) error {
 		if strings.HasPrefix(cmd.GetCommand(), "controller:") {
 			bc.InputCh <- cmd
 		} else {
@@ -297,11 +334,11 @@ func GetBlockController(blockId string) *BlockController {
 	return blockControllerMap[blockId]
 }
 
-func ProcessStaticCommand(blockId string, cmdGen BlockCommand) error {
+func ProcessStaticCommand(blockId string, cmdGen wshutil.BlockCommand) error {
 	ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout)
 	defer cancelFn()
 	switch cmd := cmdGen.(type) {
-	case *BlockSetViewCommand:
+	case *wshutil.BlockSetViewCommand:
 		log.Printf("SETVIEW: %s | %q\n", blockId, cmd.View)
 		block, err := wstore.DBGet[*wstore.Block](ctx, blockId)
 		if err != nil {
@@ -328,7 +365,7 @@ func ProcessStaticCommand(blockId string, cmdGen BlockCommand) error {
 			},
 		})
 		return nil
-	case *BlockSetMetaCommand:
+	case *wshutil.BlockSetMetaCommand:
 		log.Printf("SETMETA: %s | %v\n", blockId, cmd.Meta)
 		block, err := wstore.DBGet[*wstore.Block](ctx, blockId)
 		if err != nil {
@@ -367,9 +404,26 @@ func ProcessStaticCommand(blockId string, cmdGen BlockCommand) error {
 			},
 		})
 		return nil
-	case *BlockMessageCommand:
+	case *wshutil.BlockMessageCommand:
 		log.Printf("MESSAGE: %s | %q\n", blockId, cmd.Message)
 		return nil
+
+	case *wshutil.BlockAppendFileCommand:
+		log.Printf("APPENDFILE: %s | %q | len:%d\n", blockId, cmd.FileName, len(cmd.Data))
+		err := handleAppendBlockFile(blockId, cmd.FileName, cmd.Data)
+		if err != nil {
+			return fmt.Errorf("error appending blockfile: %w", err)
+		}
+		return nil
+
+	case *wshutil.BlockAppendIJsonCommand:
+		log.Printf("APPENDIJSON: %s | %q\n", blockId, cmd.FileName)
+		err := handleAppendIJsonFile(blockId, cmd.FileName, cmd.Data, true)
+		if err != nil {
+			return fmt.Errorf("error appending blockfile(ijson): %w", err)
+		}
+		return nil
+
 	default:
 		return fmt.Errorf("unknown command type %T", cmdGen)
 	}

pkg/blockcontroller/ptybuffer.go

@@ -19,12 +19,12 @@ const (
 type PtyBuffer struct {
 	Mode            string
 	EscSeqBuf       []byte
-	DataOutputFn    func([]byte) error
+	DataOutputFn    func(string, []byte) error
-	CommandOutputFn func(BlockCommand) error
+	CommandOutputFn func(wshutil.BlockCommand) error
 	Err             error
 }
 
-func MakePtyBuffer(dataOutputFn func([]byte) error, commandOutputFn func(BlockCommand) error) *PtyBuffer {
+func MakePtyBuffer(dataOutputFn func(string, []byte) error, commandOutputFn func(wshutil.BlockCommand) error) *PtyBuffer {
 	return &PtyBuffer{
 		Mode:            Mode_Normal,
 		DataOutputFn:    dataOutputFn,
@@ -45,7 +45,7 @@ func (b *PtyBuffer) processWaveEscSeq(escSeq []byte) {
 		b.setErr(fmt.Errorf("error unmarshalling Wave OSC sequence data: %w", err))
 		return
 	}
-	cmd, err := ParseCmdMap(jmsg)
+	cmd, err := wshutil.ParseCmdMap(jmsg)
 	if err != nil {
 		b.setErr(fmt.Errorf("error parsing Wave OSC command: %w", err))
 		return
@@ -111,7 +111,7 @@ func (b *PtyBuffer) AppendData(data []byte) {
 		outputBuf = append(outputBuf, ch)
 	}
 	if len(outputBuf) > 0 {
-		err := b.DataOutputFn(outputBuf)
+		err := b.DataOutputFn(BlockFile_Main, outputBuf)
 		if err != nil {
 			b.setErr(fmt.Errorf("error processing data output: %w", err))
 		}

pkg/eventbus/eventbus.go

@@ -15,6 +15,17 @@ type WSEventType struct {
 	Data      any    `json:"data"`
 }
 
+const (
+	FileOp_Append = "append"
+)
+
+type WSFileEventData struct {
+	ZoneId   string `json:"zoneid"`
+	FileName string `json:"filename"`
+	FileOp   string `json:"fileop"`
+	Data64   string `json:"data64"`
+}
+
 type WindowWatchData struct {
 	WindowWSCh   chan any
 	WaveWindowId string

pkg/filestore/blockstore.go

@@ -16,6 +16,21 @@ import (
 	"sync"
 	"sync/atomic"
 	"time"
+
+	"github.com/wavetermdev/thenextwave/pkg/ijson"
+)
+
+const (
+	// ijson meta keys
+	IJsonNumCommands      = "ijson:numcmds"
+	IJsonIncrementalBytes = "ijson:incbytes"
+)
+
+const (
+	IJsonHighCommands = 100
+	IJsonHighRatio    = 3
+	IJsonLowRatio     = 1
+	IJsonLowCommands  = 10
 )
 
 const DefaultPartDataSize = 64 * 1024
@@ -38,6 +53,7 @@ type FileOptsType struct {
 	MaxSize     int64 `json:"maxsize,omitempty"`
 	Circular    bool  `json:"circular,omitempty"`
 	IJson       bool  `json:"ijson,omitempty"`
+	IJsonBudget int   `json:"ijsonbudget,omitempty"`
 }
 
 type FileMeta = map[string]any
@@ -118,6 +134,12 @@ func (s *FileStore) MakeFile(ctx context.Context, zoneId string, name string, me
 			opts.MaxSize = (opts.MaxSize/partDataSize + 1) * partDataSize
 		}
 	}
+	if opts.IJsonBudget > 0 && !opts.IJson {
+		return fmt.Errorf("ijson budget requires ijson")
+	}
+	if opts.IJsonBudget < 0 {
+		return fmt.Errorf("ijson budget must be non-negative")
+	}
 	return withLock(s, zoneId, name, func(entry *CacheEntry) error {
 		if entry.File != nil {
 			return fs.ErrExist
@@ -262,6 +284,87 @@ func (s *FileStore) AppendData(ctx context.Context, zoneId string, name string,
 	})
 }
 
+func metaIncrement(file *WaveFile, key string, amount int) int {
+	if file.Meta == nil {
+		file.Meta = make(FileMeta)
+	}
+	val, ok := file.Meta[key].(int)
+	if !ok {
+		val = 0
+	}
+	newVal := val + amount
+	file.Meta[key] = newVal
+	return newVal
+}
+
+func (s *FileStore) compactIJson(ctx context.Context, entry *CacheEntry) error {
+	// we don't need to lock the entry because we have the lock on the filestore
+	_, fullData, err := entry.readAt(ctx, 0, 0, true)
+	if err != nil {
+		return err
+	}
+	newBytes, err := ijson.CompactIJson(fullData, entry.File.Opts.IJsonBudget)
+	if err != nil {
+		return err
+	}
+	entry.writeAt(0, newBytes, true)
+	return nil
+}
+
+func (s *FileStore) CompactIJson(ctx context.Context, zoneId string, name string) error {
+	return withLock(s, zoneId, name, func(entry *CacheEntry) error {
+		err := entry.loadFileIntoCache(ctx)
+		if err != nil {
+			return err
+		}
+		if !entry.File.Opts.IJson {
+			return fmt.Errorf("file %s:%s is not an ijson file", zoneId, name)
+		}
+		return s.compactIJson(ctx, entry)
+	})
+}
+
+func (s *FileStore) AppendIJson(ctx context.Context, zoneId string, name string, command map[string]any) error {
+	data, err := ijson.ValidateAndMarshalCommand(command)
+	if err != nil {
+		return err
+	}
+	return withLock(s, zoneId, name, func(entry *CacheEntry) error {
+		err := entry.loadFileIntoCache(ctx)
+		if err != nil {
+			return err
+		}
+		if !entry.File.Opts.IJson {
+			return fmt.Errorf("file %s:%s is not an ijson file", zoneId, name)
+		}
+		partMap := entry.File.computePartMap(entry.File.Size, int64(len(data)))
+		incompleteParts := incompletePartsFromMap(partMap)
+		if len(incompleteParts) > 0 {
+			err = entry.loadDataPartsIntoCache(ctx, incompleteParts)
+			if err != nil {
+				return err
+			}
+		}
+		oldSize := entry.File.Size
+		entry.writeAt(entry.File.Size, data, false)
+		entry.writeAt(entry.File.Size, []byte("\n"), false)
+		if oldSize == 0 {
+			return nil
+		}
+		// check if we should compact
+		numCmds := metaIncrement(entry.File, IJsonNumCommands, 1)
+		numBytes := metaIncrement(entry.File, IJsonIncrementalBytes, len(data)+1)
+		incRatio := float64(numBytes) / float64(entry.File.Size)
+		if numCmds > IJsonHighCommands || incRatio >= IJsonHighRatio || (numCmds > IJsonLowCommands && incRatio >= IJsonLowRatio) {
+			err := s.compactIJson(ctx, entry)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+}
+
 func (s *FileStore) GetAllZoneIds(ctx context.Context) ([]string, error) {
 	return dbGetAllZoneIds(ctx)
 }

pkg/filestore/blockstore_test.go

@@ -10,12 +10,14 @@ import (
 	"fmt"
 	"io/fs"
 	"log"
+	"reflect"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/wavetermdev/thenextwave/pkg/ijson"
 )
 
 func initDb(t *testing.T) {
@@ -620,3 +622,129 @@ func TestConcurrentAppend(t *testing.T) {
 	checkFileByteCount(t, ctx, zoneId, fileName, 'a', 100)
 	checkFileByteCount(t, ctx, zoneId, fileName, 'e', 100)
 }
+
+func jsonDeepEqual(d1 any, d2 any) bool {
+	if d1 == nil && d2 == nil {
+		return true
+	}
+	if d1 == nil || d2 == nil {
+		return false
+	}
+	t1 := reflect.TypeOf(d1)
+	t2 := reflect.TypeOf(d2)
+	if t1 != t2 {
+		return false
+	}
+	switch d1.(type) {
+	case float64:
+		return d1.(float64) == d2.(float64)
+	case string:
+		return d1.(string) == d2.(string)
+	case bool:
+		return d1.(bool) == d2.(bool)
+	case []any:
+		a1 := d1.([]any)
+		a2 := d2.([]any)
+		if len(a1) != len(a2) {
+			return false
+		}
+		for i := 0; i < len(a1); i++ {
+			if !jsonDeepEqual(a1[i], a2[i]) {
+				return false
+			}
+		}
+		return true
+	case map[string]any:
+		m1 := d1.(map[string]any)
+		m2 := d2.(map[string]any)
+		if len(m1) != len(m2) {
+			return false
+		}
+		for k, v := range m1 {
+			if !jsonDeepEqual(v, m2[k]) {
+				return false
+			}
+		}
+		return true
+	default:
+		return false
+	}
+}
+
+func TestIJson(t *testing.T) {
+	initDb(t)
+	defer cleanupDb(t)
+	ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancelFn()
+	zoneId := uuid.NewString()
+	fileName := "ij1"
+	err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{IJson: true})
+	if err != nil {
+		t.Fatalf("error creating file: %v", err)
+	}
+	rootSet := ijson.MakeSetCommand(nil, map[string]any{"tag": "div", "class": "root"})
+	err = WFS.AppendIJson(ctx, zoneId, fileName, rootSet)
+	if err != nil {
+		t.Fatalf("error appending ijson: %v", err)
+	}
+	_, fullData, err := WFS.ReadFile(ctx, zoneId, fileName)
+	if err != nil {
+		t.Fatalf("error reading file: %v", err)
+	}
+	cmds, err := ijson.ParseIJson(fullData)
+	if err != nil {
+		t.Fatalf("error parsing ijson: %v", err)
+	}
+	outData, err := ijson.ApplyCommands(nil, cmds, 0)
+	if err != nil {
+		t.Fatalf("error applying ijson: %v", err)
+	}
+	if !jsonDeepEqual(rootSet["data"], outData) {
+		t.Errorf("data mismatch: expected %v, got %v", rootSet["data"], outData)
+	}
+	childrenAppend := ijson.MakeAppendCommand(ijson.Path{"children"}, map[string]any{"tag": "div", "class": "child"})
+	err = WFS.AppendIJson(ctx, zoneId, fileName, childrenAppend)
+	if err != nil {
+		t.Fatalf("error appending ijson: %v", err)
+	}
+	_, fullData, err = WFS.ReadFile(ctx, zoneId, fileName)
+	if err != nil {
+		t.Fatalf("error reading file: %v", err)
+	}
+	cmds, err = ijson.ParseIJson(fullData)
+	if err != nil {
+		t.Fatalf("error parsing ijson: %v", err)
+	}
+	if len(cmds) != 2 {
+		t.Fatalf("command count mismatch: expected 2, got %d", len(cmds))
+	}
+	outData, err = ijson.ApplyCommands(nil, cmds, 0)
+	if err != nil {
+		t.Fatalf("error applying ijson: %v", err)
+	}
+	if !jsonDeepEqual(ijson.M{"tag": "div", "class": "root", "children": ijson.A{ijson.M{"tag": "div", "class": "child"}}}, outData) {
+		t.Errorf("data mismatch: expected %v, got %v", rootSet["data"], outData)
+	}
+	err = WFS.CompactIJson(ctx, zoneId, fileName)
+	if err != nil {
+		t.Fatalf("error compacting ijson: %v", err)
+	}
+	_, fullData, err = WFS.ReadFile(ctx, zoneId, fileName)
+	if err != nil {
+		t.Fatalf("error reading file: %v", err)
+	}
+	cmds, err = ijson.ParseIJson(fullData)
+	if err != nil {
+		t.Fatalf("error parsing ijson: %v", err)
+	}
+	if len(cmds) != 1 {
+		t.Fatalf("command count mismatch: expected 1, got %d", len(cmds))
+	}
+	outData, err = ijson.ApplyCommands(nil, cmds, 0)
+	if err != nil {
+		t.Fatalf("error applying ijson: %v", err)
+	}
+	if !jsonDeepEqual(ijson.M{"tag": "div", "class": "root", "children": ijson.A{ijson.M{"tag": "div", "class": "child"}}}, outData) {
+		t.Errorf("data mismatch: expected %v, got %v", rootSet["data"], outData)
+	}
+}

pkg/ijson/ijson.go

@@ -6,6 +6,7 @@ package ijson
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"strconv"
@@ -22,11 +23,39 @@ const (
 	AppendCommandStr = "append"
 )
 
+type Command = map[string]any
+type Path = []any
+type M = map[string]any
+type A = []any
+
 // instead of defining structs for commands, we just define a command shape
 // set: type, path, value
 // del: type, path
 // arrayappend: type, path, value
 
+func MakeSetCommand(path Path, value any) Command {
+	return Command{
+		"type": SetCommandStr,
+		"path": path,
+		"data": value,
+	}
+}
+
+func MakeDelCommand(path Path) Command {
+	return Command{
+		"type": DelCommandStr,
+		"path": path,
+	}
+}
+
+func MakeAppendCommand(path Path, value any) Command {
+	return Command{
+		"type": AppendCommandStr,
+		"path": path,
+		"data": value,
+	}
+}
+
 type PathError struct {
 	Err string
 }
@@ -35,11 +64,11 @@ func (e PathError) Error() string {
 	return "PathError: " + e.Err
 }
 
-func MakePathTypeError(path []any, index int) error {
+func MakePathTypeError(path Path, index int) error {
 	return PathError{fmt.Sprintf("invalid path element type:%T at index:%d (%s)", path[index], index, FormatPath(path))}
 }
 
-func MakePathError(errStr string, path []any, index int) error {
+func MakePathError(errStr string, path Path, index int) error {
 	return PathError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))}
 }
 
@@ -51,7 +80,7 @@ func (e SetTypeError) Error() string {
 	return "SetTypeError: " + e.Err
 }
 
-func MakeSetTypeError(errStr string, path []any, index int) error {
+func MakeSetTypeError(errStr string, path Path, index int) error {
 	return SetTypeError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))}
 }
 
@@ -63,13 +92,13 @@ func (e BudgetError) Error() string {
 	return "BudgetError: " + e.Err
 }
 
-func MakeBudgetError(errStr string, path []any, index int) error {
+func MakeBudgetError(errStr string, path Path, index int) error {
 	return BudgetError{fmt.Sprintf("%s at index:%d (%s)", errStr, index, FormatPath(path))}
 }
 
 var simplePathStrRe = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*$`)
 
-func FormatPath(path []any) string {
+func FormatPath(path Path) string {
 	if len(path) == 0 {
 		return "$"
 	}
@@ -99,7 +128,7 @@ func FormatPath(path []any) string {
 }
 
 type pathWithPos struct {
-	Path  []any
+	Path  Path
 	Index int
 }
 
@@ -152,12 +181,12 @@ type SetPathOpts struct {
 	CombineFn CombiningFunc
 }
 
-func SetPathNoErr(data any, path []any, value any, opts *SetPathOpts) any {
+func SetPathNoErr(data any, path Path, value any, opts *SetPathOpts) any {
 	ret, _ := SetPath(data, path, value, opts)
 	return ret
 }
 
-func SetPath(data any, path []any, value any, opts *SetPathOpts) (any, error) {
+func SetPath(data any, path Path, value any, opts *SetPathOpts) (any, error) {
 	if opts == nil {
 		opts = &SetPathOpts{}
 	}
@@ -464,7 +493,7 @@ func DeepEqual(v1 any, v2 any) bool {
 	}
 }
 
-func getCommandType(command map[string]any) string {
+func getCommandType(command Command) string {
 	typeVal, ok := command["type"]
 	if !ok {
 		return ""
@@ -476,7 +505,7 @@ func getCommandType(command map[string]any) string {
 	return typeStr
 }
 
-func getCommandPath(command map[string]any) []any {
+func getCommandPath(command Command) []any {
 	pathVal, ok := command["path"]
 	if !ok {
 		return nil
@@ -488,26 +517,119 @@ func getCommandPath(command map[string]any) []any {
 	return path
 }
 
-func ApplyCommand(data any, command any, budget int) (any, error) {
+func ValidatePath(path any) error {
-	mapVal, ok := command.(map[string]any)
+	if path == nil {
-	if !ok {
+		// nil path is allowed (sets the root)
-		return nil, fmt.Errorf("ApplyCommand: expected map, but got %T", command)
+		return nil
 	}
-	commandType := getCommandType(mapVal)
+	pathArr, ok := path.([]any)
+	if !ok {
+		return fmt.Errorf("path is not an array")
+	}
+	for idx, elem := range pathArr {
+		switch elem.(type) {
+		case string, int:
+			continue
+		default:
+			return fmt.Errorf("path element %d is not a string or int", idx)
+		}
+	}
+	return nil
+}
+
+func ValidateAndMarshalCommand(command Command) ([]byte, error) {
+	cmdType := getCommandType(command)
+	if cmdType != SetCommandStr && cmdType != DelCommandStr && cmdType != AppendCommandStr {
+		return nil, fmt.Errorf("unknown ijson command type %q", cmdType)
+	}
+	path := getCommandPath(command)
+	err := ValidatePath(path)
+	if err != nil {
+		return nil, err
+	}
+	barr, err := json.Marshal(command)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling ijson command to json: %w", err)
+	}
+	return barr, nil
+}
+
+func ApplyCommand(data any, command Command, budget int) (any, error) {
+	commandType := getCommandType(command)
 	if commandType == "" {
 		return nil, fmt.Errorf("ApplyCommand: missing type field")
 	}
 	switch commandType {
 	case SetCommandStr:
-		path := getCommandPath(mapVal)
+		path := getCommandPath(command)
-		return SetPath(data, path, mapVal["data"], &SetPathOpts{Budget: budget})
+		return SetPath(data, path, command["data"], &SetPathOpts{Budget: budget})
 	case DelCommandStr:
-		path := getCommandPath(mapVal)
+		path := getCommandPath(command)
 		return SetPath(data, path, nil, &SetPathOpts{Remove: true, Budget: budget})
 	case AppendCommandStr:
-		path := getCommandPath(mapVal)
+		path := getCommandPath(command)
-		return SetPath(data, path, mapVal["data"], &SetPathOpts{CombineFn: CombineFn_ArrayAppend, Budget: budget})
+		return SetPath(data, path, command["data"], &SetPathOpts{CombineFn: CombineFn_ArrayAppend, Budget: budget})
 	default:
 		return nil, fmt.Errorf("ApplyCommand: unknown command type %q", commandType)
 	}
 }
+
+func ApplyCommands(data any, commands []Command, budget int) (any, error) {
+	for _, command := range commands {
+		var err error
+		data, err = ApplyCommand(data, command, budget)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return data, nil
+}
+
+func CompactIJson(fullData []byte, budget int) ([]byte, error) {
+	var newData any
+	for len(fullData) > 0 {
+		nlIdx := bytes.IndexByte(fullData, '\n')
+		var cmdData []byte
+		if nlIdx == -1 {
+			cmdData = fullData
+			fullData = nil
+		} else {
+			cmdData = fullData[:nlIdx]
+			fullData = fullData[nlIdx+1:]
+		}
+		var cmdMap Command
+		err := json.Unmarshal(cmdData, &cmdMap)
+		if err != nil {
+			return nil, fmt.Errorf("error unmarshalling ijson command: %w", err)
+		}
+		newData, err = ApplyCommand(newData, cmdMap, budget)
+		if err != nil {
+			return nil, fmt.Errorf("error applying ijson command: %w", err)
+		}
+	}
+	newRootCmd := MakeSetCommand(nil, newData)
+	return json.Marshal(newRootCmd)
+}
+
+// returns a list of commands
+func ParseIJson(fullData []byte) ([]Command, error) {
+	var commands []Command
+	for len(fullData) > 0 {
+		nlIdx := bytes.IndexByte(fullData, '\n')
+		var cmdData []byte
+		if nlIdx == -1 {
+			cmdData = fullData
+			fullData = nil
+		} else {
+			cmdData = fullData[:nlIdx]
+			fullData = fullData[nlIdx+1:]
+		}
+		var cmdMap Command
+		err := json.Unmarshal(cmdData, &cmdMap)
+		if err != nil {
+			return nil, fmt.Errorf("error unmarshalling ijson command: %w", err)
+		}
+		commands = append(commands, cmdMap)
+	}
+	return commands, nil
+}

pkg/service/blockservice/blockservice.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/wavetermdev/thenextwave/pkg/blockcontroller"
 	"github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
 )
 
 type BlockService struct{}
@@ -25,7 +26,7 @@ func (bs *BlockService) SendCommand_Meta() tsgenmeta.MethodMeta {
 	}
 }
 
-func (bs *BlockService) SendCommand(blockId string, cmd blockcontroller.BlockCommand) error {
+func (bs *BlockService) SendCommand(blockId string, cmd wshutil.BlockCommand) error {
 	if strings.HasPrefix(cmd.GetCommand(), "controller:") {
 		bc := blockcontroller.GetBlockController(blockId)
 		if bc == nil {

pkg/service/service.go

@@ -9,7 +9,6 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/wavetermdev/thenextwave/pkg/blockcontroller"
 	"github.com/wavetermdev/thenextwave/pkg/service/blockservice"
 	"github.com/wavetermdev/thenextwave/pkg/service/clientservice"
 	"github.com/wavetermdev/thenextwave/pkg/service/fileservice"
@@ -17,6 +16,7 @@ import (
 	"github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta"
 	"github.com/wavetermdev/thenextwave/pkg/waveobj"
 	"github.com/wavetermdev/thenextwave/pkg/web/webcmd"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
 	"github.com/wavetermdev/thenextwave/pkg/wstore"
 )
 
@@ -36,7 +36,7 @@ var waveObjMapRType = reflect.TypeOf(map[string]waveobj.WaveObj{})
 var methodMetaRType = reflect.TypeOf(tsgenmeta.MethodMeta{})
 var waveObjUpdateRType = reflect.TypeOf(wstore.WaveObjUpdate{})
 var uiContextRType = reflect.TypeOf((*wstore.UIContext)(nil)).Elem()
-var blockCommandRType = reflect.TypeOf((*blockcontroller.BlockCommand)(nil)).Elem()
+var blockCommandRType = reflect.TypeOf((*wshutil.BlockCommand)(nil)).Elem()
 var wsCommandRType = reflect.TypeOf((*webcmd.WSCommandType)(nil)).Elem()
 
 type WebCallType struct {
@@ -100,7 +100,7 @@ func convertBlockCommand(argType reflect.Type, jsonArg any) (any, error) {
 	if _, ok := jsonArg.(map[string]any); !ok {
 		return nil, fmt.Errorf("cannot convert %T to %s", jsonArg, argType)
 	}
-	cmd, err := blockcontroller.ParseCmdMap(jsonArg.(map[string]any))
+	cmd, err := wshutil.ParseCmdMap(jsonArg.(map[string]any))
 	if err != nil {
 		return nil, fmt.Errorf("error parsing command map: %w", err)
 	}

pkg/tsgen/tsgen.go

@@ -10,12 +10,12 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/wavetermdev/thenextwave/pkg/blockcontroller"
 	"github.com/wavetermdev/thenextwave/pkg/eventbus"
 	"github.com/wavetermdev/thenextwave/pkg/service"
 	"github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta"
 	"github.com/wavetermdev/thenextwave/pkg/waveobj"
 	"github.com/wavetermdev/thenextwave/pkg/web/webcmd"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
 	"github.com/wavetermdev/thenextwave/pkg/wstore"
 )
 
@@ -354,7 +354,7 @@ func GenerateServiceClass(serviceName string, serviceObj any, tsTypesMap map[ref
 }
 
 func GenerateWaveObjTypes(tsTypesMap map[reflect.Type]string) {
-	GenerateTSTypeUnion(blockcontroller.CommandTypeUnionMeta(), tsTypesMap)
+	GenerateTSTypeUnion(wshutil.CommandTypeUnionMeta(), tsTypesMap)
 	GenerateTSTypeUnion(webcmd.WSCommandTypeUnionMeta(), tsTypesMap)
 	GenerateTSType(reflect.TypeOf(waveobj.ORef{}), tsTypesMap)
 	GenerateTSType(reflect.TypeOf((*waveobj.WaveObj)(nil)).Elem(), tsTypesMap)
@@ -363,6 +363,7 @@ func GenerateWaveObjTypes(tsTypesMap map[reflect.Type]string) {
 	GenerateTSType(reflect.TypeOf(service.WebReturnType{}), tsTypesMap)
 	GenerateTSType(reflect.TypeOf(wstore.UIContext{}), tsTypesMap)
 	GenerateTSType(reflect.TypeOf(eventbus.WSEventType{}), tsTypesMap)
+	GenerateTSType(reflect.TypeOf(eventbus.WSFileEventData{}), tsTypesMap)
 	for _, rtype := range wstore.AllWaveObjTypes() {
 		GenerateTSType(rtype, tsTypesMap)
 	}

pkg/web/ws.go

@@ -15,10 +15,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/websocket"
-	"github.com/wavetermdev/thenextwave/pkg/blockcontroller"
 	"github.com/wavetermdev/thenextwave/pkg/eventbus"
 	"github.com/wavetermdev/thenextwave/pkg/service/blockservice"
 	"github.com/wavetermdev/thenextwave/pkg/web/webcmd"
+	"github.com/wavetermdev/thenextwave/pkg/wshutil"
 )
 
 const wsReadWaitTimeout = 15 * time.Second
@@ -95,8 +95,8 @@ func processWSCommand(jmsg map[string]any, outputCh chan any) {
 	}
 	switch cmd := wsCommand.(type) {
 	case *webcmd.SetBlockTermSizeWSCommand:
-		blockCmd := &blockcontroller.BlockInputCommand{
+		blockCmd := &wshutil.BlockInputCommand{
-			Command:  blockcontroller.BlockCommand_Input,
+			Command:  wshutil.BlockCommand_Input,
 			TermSize: &cmd.TermSize,
 		}
 		blockservice.BlockServiceInstance.SendCommand(cmd.BlockId, blockCmd)

pkg/wshutil/wshcommands.go

@@ -3,64 +3,135 @@
 
 package wshutil
 
-import "reflect"
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+
+	"github.com/wavetermdev/thenextwave/pkg/ijson"
+	"github.com/wavetermdev/thenextwave/pkg/shellexec"
+	"github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta"
+)
+
+const CommandKey = "command"
 
 const (
-	CommandSetView         = "setview"
+	BlockCommand_Message         = "message"
-	CommandSetMeta         = "setmeta"
+	BlockCommand_SetView         = "setview"
-	CommandBlockFileAppend = "blockfile:append"
+	BlockCommand_SetMeta         = "setmeta"
-	CommandStreamFile      = "streamfile"
+	BlockCommand_Input           = "controller:input"
+	BlockCommand_AppendBlockFile = "blockfile:append"
+	BlockCommand_AppendIJson     = "blockfile:appendijson"
 )
 
 var CommandToTypeMap = map[string]reflect.Type{
-	CommandSetView: reflect.TypeOf(SetViewCommand{}),
+	BlockCommand_Input:           reflect.TypeOf(BlockInputCommand{}),
-	CommandSetMeta: reflect.TypeOf(SetMetaCommand{}),
+	BlockCommand_SetView:         reflect.TypeOf(BlockSetViewCommand{}),
+	BlockCommand_SetMeta:         reflect.TypeOf(BlockSetMetaCommand{}),
+	BlockCommand_Message:         reflect.TypeOf(BlockMessageCommand{}),
+	BlockCommand_AppendBlockFile: reflect.TypeOf(BlockAppendFileCommand{}),
+	BlockCommand_AppendIJson:     reflect.TypeOf(BlockAppendIJsonCommand{}),
 }
 
-type Command interface {
+func CommandTypeUnionMeta() tsgenmeta.TypeUnionMeta {
+	var rtypes []reflect.Type
+	for _, rtype := range CommandToTypeMap {
+		rtypes = append(rtypes, rtype)
+	}
+	return tsgenmeta.TypeUnionMeta{
+		BaseType:      reflect.TypeOf((*BlockCommand)(nil)).Elem(),
+		TypeFieldName: "command",
+		Types:         rtypes,
+	}
+}
+
+type baseCommand struct {
+	Command string `json:"command"`
+}
+
+type BlockCommand interface {
 	GetCommand() string
 }
 
-// for unmarshalling
+type BlockCommandWrapper struct {
-type baseCommand struct {
+	BlockCommand
-	Command string `json:"command"`
-	RpcID   string `json:"rpcid"`
-	RpcType string `json:"rpctype"`
 }
 
-type SetViewCommand struct {
+func ParseCmdMap(cmdMap map[string]any) (BlockCommand, error) {
-	Command string `json:"command"`
+	cmdType, ok := cmdMap[CommandKey].(string)
+	if !ok {
+		return nil, fmt.Errorf("no %s field in command map", CommandKey)
+	}
+	mapJson, err := json.Marshal(cmdMap)
+	if err != nil {
+		return nil, fmt.Errorf("error marshalling command map: %w", err)
+	}
+	rtype := CommandToTypeMap[cmdType]
+	if rtype == nil {
+		return nil, fmt.Errorf("unknown command type %q", cmdType)
+	}
+	cmd := reflect.New(rtype).Interface()
+	err = json.Unmarshal(mapJson, cmd)
+	if err != nil {
+		return nil, fmt.Errorf("error unmarshalling command: %w", err)
+	}
+	return cmd.(BlockCommand), nil
+}
+
+type BlockInputCommand struct {
+	Command     string              `json:"command" tstype:"\"controller:input\""`
+	InputData64 string              `json:"inputdata64,omitempty"`
+	SigName     string              `json:"signame,omitempty"`
+	TermSize    *shellexec.TermSize `json:"termsize,omitempty"`
+}
+
+func (ic *BlockInputCommand) GetCommand() string {
+	return BlockCommand_Input
+}
+
+type BlockSetViewCommand struct {
+	Command string `json:"command" tstype:"\"setview\""`
 	View    string `json:"view"`
 }
 
-func (svc *SetViewCommand) GetCommand() string {
+func (svc *BlockSetViewCommand) GetCommand() string {
-	return CommandSetView
+	return BlockCommand_SetView
 }
 
-type SetMetaCommand struct {
+type BlockSetMetaCommand struct {
-	Command string         `json:"command"`
+	Command string         `json:"command" tstype:"\"setmeta\""`
 	Meta    map[string]any `json:"meta"`
 }
 
-func (smc *SetMetaCommand) GetCommand() string {
+func (smc *BlockSetMetaCommand) GetCommand() string {
-	return CommandSetMeta
+	return BlockCommand_SetMeta
 }
 
-type BlockFileAppendCommand struct {
+type BlockMessageCommand struct {
-	Command  string `json:"command"`
+	Command string `json:"command" tstype:"\"message\""`
+	Message string `json:"message"`
+}
+
+func (bmc *BlockMessageCommand) GetCommand() string {
+	return BlockCommand_Message
+}
+
+type BlockAppendFileCommand struct {
+	Command  string `json:"command" tstype:"\"blockfile:append\""`
 	FileName string `json:"filename"`
 	Data     []byte `json:"data"`
 }
 
-func (bfac *BlockFileAppendCommand) GetCommand() string {
+func (bwc *BlockAppendFileCommand) GetCommand() string {
-	return CommandBlockFileAppend
+	return BlockCommand_AppendBlockFile
 }
 
-type StreamFileCommand struct {
+type BlockAppendIJsonCommand struct {
-	Command  string `json:"command"`
+	Command  string        `json:"command" tstype:"\"blockfile:appendijson\""`
 	FileName string        `json:"filename"`
+	Data     ijson.Command `json:"data"`
 }
 
-func (c *StreamFileCommand) GetCommand() string {
+func (bwc *BlockAppendIJsonCommand) GetCommand() string {
-	return CommandStreamFile
+	return BlockCommand_AppendIJson
 }

pkg/wshutil/wshutil.go

@@ -25,7 +25,7 @@ var WaveOSCPrefixBytes = []byte(WaveOSCPrefix)
 // JSON = must escape all ASCII control characters ([\x00-\x1F\x7F])
 // we can tell the difference between JSON and base64-JSON by the first character: '{' or not
 
-func EncodeWaveOSCMessage(cmd Command) ([]byte, error) {
+func EncodeWaveOSCMessage(cmd BlockCommand) ([]byte, error) {
 	if cmd.GetCommand() == "" {
 		return nil, fmt.Errorf("Command field not set in struct")
 	}
@@ -74,7 +74,7 @@ func EncodeWaveOSCMessage(cmd Command) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-func decodeWaveOSCMessage(data []byte) (Command, error) {
+func decodeWaveOSCMessage(data []byte) (BlockCommand, error) {
 	var baseCmd baseCommand
 	err := json.Unmarshal(data, &baseCmd)
 	if err != nil {
@@ -85,12 +85,12 @@ func decodeWaveOSCMessage(data []byte) (Command, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error unmarshalling json: %w", err)
 	}
-	return rtnCmd.(Command), nil
+	return rtnCmd.(BlockCommand), nil
 }
 
 // data does not contain the escape sequence, just the innards
 // this function implements the switch between JSON and base64-JSON
-func DecodeWaveOSCMessage(data []byte) (Command, error) {
+func DecodeWaveOSCMessage(data []byte) (BlockCommand, error) {
 	if len(data) == 0 {
 		return nil, fmt.Errorf("empty data")
 	}

pkg/wstore/wstore_dbops.go

@@ -6,7 +6,9 @@ package wstore
 import (
 	"context"
 	"fmt"
+	"log"
 
+	"github.com/wavetermdev/thenextwave/pkg/filestore"
 	"github.com/wavetermdev/thenextwave/pkg/waveobj"
 	"github.com/wavetermdev/waveterm/wavesrv/pkg/dbutil"
 )
@@ -167,13 +169,21 @@ func DBSelectMap[T waveobj.WaveObj](ctx context.Context, ids []string) (map[stri
 }
 
 func DBDelete(ctx context.Context, otype string, id string) error {
-	return WithTx(ctx, func(tx *TxWrap) error {
+	err := WithTx(ctx, func(tx *TxWrap) error {
 		table := tableNameFromOType(otype)
 		query := fmt.Sprintf("DELETE FROM %s WHERE oid = ?", table)
 		tx.Exec(query, id)
 		ContextAddUpdate(ctx, WaveObjUpdate{UpdateType: UpdateType_Delete, OType: otype, OID: id})
 		return nil
 	})
+	if err != nil {
+		return err
+	}
+	err = filestore.WFS.DeleteZone(ctx, id)
+	if err != nil {
+		log.Printf("error deleting filestore zone (after deleting block): %v", err)
+	}
+	return nil
 }
 
 func DBUpdate(ctx context.Context, val waveobj.WaveObj) error {

pkg/wstore/wstore_types.go

@@ -130,6 +130,7 @@ type Window struct {
 	Version        int               `json:"version"`
 	WorkspaceId    string            `json:"workspaceid"`
 	ActiveTabId    string            `json:"activetabid"`
+	ActiveBlockId  string            `json:"activeblockid,omitempty"`
 	ActiveBlockMap map[string]string `json:"activeblockmap"` // map from tabid to blockid
 	Pos            Point             `json:"pos"`
 	WinSize        WinSize           `json:"winsize"`

yarn.lock

@@ -11015,6 +11015,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"react-frame-component@npm:^5.2.7":
+  version: 5.2.7
+  resolution: "react-frame-component@npm:5.2.7"
+  peerDependencies:
+    prop-types: ^15.5.9
+    react: ">= 16.3"
+    react-dom: ">= 16.3"
+  checksum: 10c0/e138602aa98557c021ae825f51468026c53b9939140c5961d5371b65ad07ff9a5adaf2cd4e4a8a77414a05ae0f95a842939c8e102aa576ef21ff096368b905a3
+  languageName: node
+  linkType: hard
+
 "react-is@npm:18.1.0":
   version: 18.1.0
   resolution: "react-is@npm:18.1.0"
@@ -12277,6 +12288,7 @@ __metadata:
     react-dnd: "npm:^16.0.1"
     react-dnd-html5-backend: "npm:^16.0.1"
     react-dom: "npm:^18.3.1"
+    react-frame-component: "npm:^5.2.7"
     react-markdown: "npm:^9.0.1"
     remark-gfm: "npm:^4.0.0"
     rxjs: "npm:^7.8.1"