VDom 9 (#1205)

Lots of quality of life improvements (and bug fixes):
* Class(), ClassIf(), ClassIfElse() methods
* <wave:style> that can use a file vdom:/// url
* UseStateWithFn() to allow for functional setters
* If, IfElse, and ForEach for vdom construction
* batched updates for large updates -- streaming -- to get around packet size issues
* more flexible file sending code, for []byte, io.Reader, fs.File, and a fileName (with optional MimeType)
* fix the vdom:// protocol handler to work with fetch
* updated wshcmd-html for new best practices

cmd/wsh/cmd/htmlstyle.css

@@ -0,0 +1,42 @@
+.root {
+    padding: 10px;
+}
+
+.background {
+    display: flex;
+    align-items: center;
+    width: 100%;
+}
+
+.background-inner {
+    max-width: 300px;
+}
+
+.bg-item {
+    cursor: pointer;
+    padding: 8px 12px;
+    border-radius: 4px;
+    display: flex;
+    flex-direction: row;
+    align-items: flex-start;
+    justify-content: flex-start;
+}
+
+.bg-item:hover {
+    background-color: var(--button-grey-hover-bg);
+}
+
+.bg-preview {
+    width: 20px;
+    height: 20px;
+    margin-right: 10px;
+    border-radius: 50%;
+    border: 1px solid #777;
+}
+
+.bg-label {
+    display: block;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+}

cmd/wsh/cmd/wshcmd-html.go

@@ -5,6 +5,7 @@ package cmd
 
 import (
 	"context"
+	_ "embed"
 	"log"
 
 	"github.com/spf13/cobra"
@@ -16,6 +17,9 @@ import (
 	"github.com/wavetermdev/waveterm/pkg/wshutil"
 )
 
+//go:embed htmlstyle.css
+var htmlStyleCSS []byte
+
 var htmlCmdNewBlock bool
 var HtmlVDomClient *vdomclient.Client = vdomclient.MakeClient(&vdom.VDomBackendOpts{CloseOnCtrlC: true})
 
@@ -50,66 +54,25 @@ type BgItem struct {
 // Components
 var Style = vdomclient.DefineComponent[struct{}](HtmlVDomClient, "Style",
 	func(ctx context.Context, _ struct{}) any {
-		return vdom.E("style", nil, `
+		return vdom.E("wave:style",
-			.root {
+			vdom.P("src", "vdom:///style.css"),
-				padding: 10px;
+		)
-			}
-
-			.background {
-				display: flex;
-				align-items: center;
-				width: 100%;
-			}
-
-			.background-inner {
-				max-width: 300px;
-			}
-
-			.bg-item {
-				cursor: pointer;
-				padding: 8px 12px;
-				border-radius: 4px;
-				display: flex;
-				flex-direction: row;
-				align-items: flex-start;
-				justify-content: flex-start;
-			}
-
-			.bg-item:hover {
-				background-color: var(--button-grey-hover-bg);
-			}
-
-			.bg-preview {
-				width: 20px;
-				height: 20px;
-				margin-right: 10px;
-				border-radius: 50%;
-				border: 1px solid #777;
-			}
-
-			.bg-label {
-				display: block;
-				white-space: nowrap;
-				overflow: hidden;
-				text-overflow: ellipsis;
-			}
-		`)
 	},
 )
 
 var BgItemTag = vdomclient.DefineComponent[BgItemProps](HtmlVDomClient, "BgItem",
 	func(ctx context.Context, props BgItemProps) any {
 		return vdom.E("div",
-			vdom.P("className", "bg-item"),
+			vdom.Class("bg-item"),
-			vdom.P("onClick", props.OnClick),
 			vdom.E("div",
-				vdom.P("className", "bg-preview"),
+				vdom.Class("bg-preview"),
 				vdom.PStyle("background", props.Bg),
 			),
 			vdom.E("div",
-				vdom.P("className", "bg-label"),
+				vdom.Class("bg-label"),
 				props.Label,
 			),
+			vdom.P("onClick", props.OnClick),
 		)
 	},
 )
@@ -133,20 +96,17 @@ var BgList = vdomclient.DefineComponent[BgListProps](HtmlVDomClient, "BgList",
 			}
 		}
 
-		items := make([]*vdom.VDomElem, 0, len(props.Items))
-		for _, item := range props.Items {
-			items = append(items, BgItemTag(BgItemProps{
-				Bg:      item.Bg,
-				Label:   item.Label,
-				OnClick: setBackground(item.Bg),
-			}))
-		}
-
 		return vdom.E("div",
-			vdom.P("className", "background"),
+			vdom.Class("background"),
 			vdom.E("div",
-				vdom.P("className", "background-inner"),
+				vdom.Class("background-inner"),
-				items,
+				vdom.ForEach(props.Items, func(item BgItem) any {
+					return BgItemTag(BgItemProps{
+						Bg:      item.Bg,
+						Label:   item.Label,
+						OnClick: setBackground(item.Bg),
+					})
+				}),
 			),
 		)
 	},
@@ -164,7 +124,7 @@ var App = vdomclient.DefineComponent[struct{}](HtmlVDomClient, "App",
 		}
 
 		return vdom.E("div",
-			vdom.P("className", "root"),
+			vdom.Class("root"),
 			Style(struct{}{}),
 			vdom.E("h1", nil, "Set Background"),
 			vdom.E("div", nil,
@@ -177,7 +137,11 @@ var App = vdomclient.DefineComponent[struct{}](HtmlVDomClient, "App",
 			),
 			vdom.E("div", nil,
 				vdom.E("img",
-					vdom.P("style", "width: 100%; height: 100%; max-width: 300px; max-height: 300px; object-fit: contain;"),
+					vdom.PStyle("width", "100%"),
+					vdom.PStyle("height", "100%"),
+					vdom.PStyle("maxWidth", "300px"),
+					vdom.PStyle("maxHeight", "300px"),
+					vdom.PStyle("objectFit", "contain"),
 					vdom.P("src", "vdom:///test.png"),
 				),
 			),
@@ -189,9 +153,7 @@ var App = vdomclient.DefineComponent[struct{}](HtmlVDomClient, "App",
 						setInputText(e.TargetValue)
 					}),
 				),
-				vdom.E("div", nil,
+				vdom.E("div", nil, "text ", inputText),
-					"text ", inputText,
-				),
 			),
 		)
 	},
@@ -205,19 +167,20 @@ func htmlRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Set up the root component
 	client.SetRootElem(App(struct{}{}))
+	client.RegisterFileHandler("/style.css", vdomclient.FileHandlerOption{
+		Data:     htmlStyleCSS,
+		MimeType: "text/css",
+	})
+	client.RegisterFileHandler("/test.png", vdomclient.FileHandlerOption{
+		FilePath: "~/Downloads/IMG_1939.png",
+	})
 
-	// Set up file handler
-	client.RegisterFileHandler("/test.png", "~/Downloads/IMG_1939.png")
-
-	// Create the VDOM context
 	err = client.CreateVDomContext(&vdom.VDomTarget{NewBlock: htmlCmdNewBlock})
 	if err != nil {
 		return err
 	}
 
-	// Handle shutdown
 	go func() {
 		<-client.DoneCh
 		wshutil.DoShutdown("vdom closed by FE", 0, true)

emain/emain-vdomhandler.ts

@@ -3,6 +3,18 @@ import { RpcApi } from "../frontend/app/store/wshclientapi";
 import { base64ToArray } from "../frontend/util/util";
 import { ElectronWshClient } from "./emain-wsh";
 
+export function registerVDomProtocol() {
+    protocol.registerSchemesAsPrivileged([
+        {
+            scheme: "vdom",
+            privileges: {
+                standard: true,
+                supportFetchAPI: true,
+            },
+        },
+    ]);
+}
+
 export function setupVdomUrlHandler() {
     protocol.handle("vdom", async (request) => {
         // Only handle GET requests for now

emain/emain.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import * as electron from "electron";
-import { setupVdomUrlHandler } from "emain/emain-vdomhandler";
+import { registerVDomProtocol, setupVdomUrlHandler } from "emain/emain-vdomhandler";
 import { FastAverageColor } from "fast-average-color";
 import fs from "fs";
 import * as child_process from "node:child_process";
@@ -700,6 +700,7 @@ async function appMain() {
         electronApp.quit();
         return;
     }
+    registerVDomProtocol();
     makeAppMenu();
     try {
         await runWaveSrv(handleWSEvent);

frontend/app/store/wshclientapi.ts

@@ -247,9 +247,9 @@ class RpcApiType {
         return client.wshRpcCall("vdomcreatecontext", data, opts);
     }
 
-    // command "vdomrender" [call]
+    // command "vdomrender" [responsestream]
-    VDomRenderCommand(client: WshClient, data: VDomFrontendUpdate, opts?: RpcOpts): Promise<VDomBackendUpdate> {
+	VDomRenderCommand(client: WshClient, data: VDomFrontendUpdate, opts?: RpcOpts): AsyncGenerator<VDomBackendUpdate, void, boolean> {
-        return client.wshRpcCall("vdomrender", data, opts);
+        return client.wshRpcStream("vdomrender", data, opts);
     }
 
     // command "vdomurlrequest" [responsestream]

frontend/app/view/vdom/vdom-model.tsx

@@ -9,6 +9,7 @@ import { RpcResponseHelper, WshClient } from "@/app/store/wshclient";
 import { RpcApi } from "@/app/store/wshclientapi";
 import { makeFeBlockRouteId } from "@/app/store/wshrouter";
 import { DefaultRouter, TabRpcClient } from "@/app/store/wshrpcutil";
+import { mergeBackendUpdates, restoreVDomElems } from "@/app/view/vdom/vdom-utils";
 import { adaptFromReactOrNativeKeyEvent, checkKeyPressed } from "@/util/keyutil";
 import debug from "debug";
 import * as jotai from "jotai";
@@ -330,8 +331,21 @@ export class VDomModel {
         try {
             const feUpdate = this.createFeUpdate();
             dlog("fe-update", feUpdate);
-            const beUpdate = await RpcApi.VDomRenderCommand(TabRpcClient, feUpdate, { route: backendRoute });
+            const beUpdateGen = await RpcApi.VDomRenderCommand(TabRpcClient, feUpdate, { route: backendRoute });
-            this.handleBackendUpdate(beUpdate);
+            let baseUpdate: VDomBackendUpdate = null;
+            for await (const beUpdate of beUpdateGen) {
+                if (baseUpdate === null) {
+                    baseUpdate = beUpdate;
+                } else {
+                    mergeBackendUpdates(baseUpdate, beUpdate);
+                }
+            }
+            if (baseUpdate !== null) {
+                restoreVDomElems(baseUpdate);
+                dlog("be-update", baseUpdate);
+                this.handleBackendUpdate(baseUpdate);
+            }
+            dlog("update cycle done");
         } finally {
             this.lastUpdateTs = Date.now();
             this.hasPendingRequest = false;

frontend/app/view/vdom/vdom-utils.tsx

@@ -137,48 +137,63 @@ export function validateAndWrapReactStyle(model: VDomModel, style: Record<string
     return sanitizedStyle;
 }
 
-type VDomTransferElem = {
+export function restoreVDomElems(backendUpdate: VDomBackendUpdate) {
-    root?: boolean;
+    if (!backendUpdate.transferelems || !backendUpdate.renderupdates) {
-    waveid?: string;
+        return;
-    tag: string;
+    }
-    props?: { [key: string]: any };
-    children?: string[]; // References to child WaveIds
-    text?: string;
-};
 
-export function UnmarshalTransferElems(transferElems: VDomTransferElem[]): VDomElem[] {
+    // Step 1: Map of waveid to VDomElem, skipping any without a waveid
-    const elemMap: { [id: string]: VDomElem } = {};
+    const elemMap = new Map<string, VDomElem>();
-    const roots: VDomElem[] = [];
+    backendUpdate.transferelems.forEach((transferElem) => {
-
-    // Initialize each VDomTransferElem in the map without children, as we'll link them after
-    transferElems.forEach((transferElem) => {
         if (!transferElem.waveid) {
-            return; // Skip elements without waveid
+            return;
         }
-        const elem: VDomElem = {
+        elemMap.set(transferElem.waveid, {
-            waveid: transferElem.tag !== "#text" ? transferElem.waveid : undefined,
+            waveid: transferElem.waveid,
             tag: transferElem.tag,
             props: transferElem.props,
+            children: [], // Will populate children later
             text: transferElem.text,
-            children: [], // Placeholder to be populated later
+        });
-        };
+    });
-        elemMap[transferElem.waveid] = elem;
 
-        // Collect root elements
+    // Step 2: Build VDomElem trees by linking children
-        if (transferElem.root) {
+    backendUpdate.transferelems.forEach((transferElem) => {
-            roots.push(elem);
+        const parent = elemMap.get(transferElem.waveid);
+        if (!parent || !transferElem.children || transferElem.children.length === 0) {
+            return;
+        }
+        parent.children = transferElem.children.map((childId) => elemMap.get(childId)).filter((child) => child != null); // Explicit null check
+    });
+
+    // Step 3: Update renderupdates with rebuilt VDomElem trees
+    backendUpdate.renderupdates.forEach((update) => {
+        if (update.vdomwaveid) {
+            update.vdom = elemMap.get(update.vdomwaveid);
         }
     });
-
+}
-    // Now populate children for each element
+
-    transferElems.forEach((transferElem) => {
+export function mergeBackendUpdates(baseUpdate: VDomBackendUpdate, nextUpdate: VDomBackendUpdate) {
-        if (!transferElem.waveid || !transferElem.children) return;
+    // Verify the updates are from the same block/sequence
-
+    if (baseUpdate.blockid !== nextUpdate.blockid || baseUpdate.ts !== nextUpdate.ts) {
-        const currentElem = elemMap[transferElem.waveid];
+        console.error("Attempted to merge updates from different blocks or timestamps");
-        currentElem.children = transferElem.children
+        return;
-            .map((childId) => elemMap[childId])
+    }
-            .filter((child) => child !== undefined); // Filter out any undefined children
+
-    });
+    // Merge TransferElems
-
+    if (nextUpdate.transferelems?.length > 0) {
-    return roots;
+        if (!baseUpdate.transferelems) {
+            baseUpdate.transferelems = [];
+        }
+        baseUpdate.transferelems.push(...nextUpdate.transferelems);
+    }
+
+    // Merge StateSync
+    if (nextUpdate.statesync?.length > 0) {
+        if (!baseUpdate.statesync) {
+            baseUpdate.statesync = [];
+        }
+        baseUpdate.statesync.push(...nextUpdate.statesync);
+    }
 }

frontend/app/view/vdom/vdom.tsx

@@ -23,6 +23,7 @@ const FragmentTag = "#fragment";
 const WaveTextTag = "wave:text";
 const WaveNullTag = "wave:null";
 const StyleTagName = "style";
+const WaveStyleTagName = "wave:style";
 
 const VDomObjType_Ref = "ref";
 const VDomObjType_Binding = "binding";
@@ -367,6 +368,36 @@ function StyleTag({ elem, model }: { elem: VDomElem; model: VDomModel }) {
     return <style>{sanitizedCss}</style>;
 }
 
+function WaveStyle({ src, model }: { src: string; model: VDomModel }) {
+    const [styleContent, setStyleContent] = React.useState<string | null>(null);
+    React.useEffect(() => {
+        async function fetchAndSanitizeCss() {
+            try {
+                const response = await fetch(src);
+                if (!response.ok) {
+                    console.error(`Failed to load CSS from ${src}`);
+                    return;
+                }
+                const cssText = await response.text();
+                const wrapperClassName = "vdom-" + model.blockId;
+                const sanitizedCss = validateAndWrapCss(model, cssText, wrapperClassName);
+                if (sanitizedCss) {
+                    setStyleContent(sanitizedCss);
+                } else {
+                    console.error("Failed to sanitize CSS");
+                }
+            } catch (error) {
+                console.error("Error fetching CSS:", error);
+            }
+        }
+        fetchAndSanitizeCss();
+    }, [src, model]);
+    if (!styleContent) {
+        return null;
+    }
+    return <style>{styleContent}</style>;
+}
+
 function VDomTag({ elem, model }: { elem: VDomElem; model: VDomModel }) {
     const props = useVDom(model, elem);
     if (elem.tag == WaveNullTag) {
@@ -382,6 +413,9 @@ function VDomTag({ elem, model }: { elem: VDomElem; model: VDomModel }) {
     if (elem.tag == StyleTagName) {
         return <StyleTag elem={elem} model={model} />;
     }
+    if (elem.tag == WaveStyleTagName) {
+        return <WaveStyle src={props.src} model={model} />;
+    }
     if (!AllowedSimpleTags[elem.tag] && !AllowedSvgTags[elem.tag]) {
         return <div>{"Invalid Tag <" + elem.tag + ">"}</div>;
     }

frontend/types/gotypes.d.ts

@@ -652,6 +652,7 @@ declare global {
         opts?: VDomBackendOpts;
         haswork?: boolean;
         renderupdates?: VDomRenderUpdate[];
+        transferelems?: VDomTransferElem[];
         statesync?: VDomStateSync[];
         refoperations?: VDomRefOperation[];
         messages?: VDomMessage[];
@@ -773,7 +774,8 @@ declare global {
     type VDomRenderUpdate = {
         updatetype: "root"|"append"|"replace"|"remove"|"insert";
         waveid?: string;
-        vdom: VDomElem;
+        vdomwaveid?: string;
+        vdom?: VDomElem;
         index?: number;
     };
 
@@ -789,6 +791,15 @@ declare global {
         magnified?: boolean;
     };
 
+    // vdom.VDomTransferElem
+    type VDomTransferElem = {
+        waveid?: string;
+        tag: string;
+        props?: {[key: string]: any};
+        children?: string[];
+        text?: string;
+    };
+
     // wshrpc.VDomUrlRequestData
     type VDomUrlRequestData = {
         method: string;

pkg/vdom/vdom.go

@@ -35,6 +35,11 @@ type styleAttrWrapper struct {
 	Val       any
 }
 
+type classAttrWrapper struct {
+	ClassName string
+	Cond      bool
+}
+
 type styleAttrMapWrapper struct {
 	StyleAttrMap map[string]any
 }
@@ -82,6 +87,47 @@ func mergeStyleAttr(props *map[string]any, styleAttr styleAttrWrapper) {
 	styleMap[styleAttr.StyleAttr] = styleAttr.Val
 }
 
+func mergeClassAttr(props *map[string]any, classAttr classAttrWrapper) {
+	if *props == nil {
+		*props = make(map[string]any)
+	}
+	if classAttr.Cond {
+		if (*props)["className"] == nil {
+			(*props)["className"] = classAttr.ClassName
+			return
+		}
+		classVal, ok := (*props)["className"].(string)
+		if !ok {
+			return
+		}
+		// check if class already exists (must split, contains won't work)
+		splitArr := strings.Split(classVal, " ")
+		for _, class := range splitArr {
+			if class == classAttr.ClassName {
+				return
+			}
+		}
+		(*props)["className"] = classVal + " " + classAttr.ClassName
+	} else {
+		classVal, ok := (*props)["className"].(string)
+		if !ok {
+			return
+		}
+		splitArr := strings.Split(classVal, " ")
+		for i, class := range splitArr {
+			if class == classAttr.ClassName {
+				splitArr = append(splitArr[:i], splitArr[i+1:]...)
+				break
+			}
+		}
+		if len(splitArr) == 0 {
+			delete(*props, "className")
+		} else {
+			(*props)["className"] = strings.Join(splitArr, " ")
+		}
+	}
+}
+
 func E(tag string, parts ...any) *VDomElem {
 	rtn := &VDomElem{Tag: tag}
 	for _, part := range parts {
@@ -103,12 +149,54 @@ func E(tag string, parts ...any) *VDomElem {
 			}
 			continue
 		}
+		if classAttr, ok := part.(classAttrWrapper); ok {
+			mergeClassAttr(&rtn.Props, classAttr)
+			continue
+		}
 		elems := partToElems(part)
 		rtn.Children = append(rtn.Children, elems...)
 	}
 	return rtn
 }
 
+func Class(name string) classAttrWrapper {
+	return classAttrWrapper{ClassName: name, Cond: true}
+}
+
+func ClassIf(cond bool, name string) classAttrWrapper {
+	return classAttrWrapper{ClassName: name, Cond: cond}
+}
+
+func ClassIfElse(cond bool, name string, elseName string) classAttrWrapper {
+	if cond {
+		return classAttrWrapper{ClassName: name, Cond: true}
+	}
+	return classAttrWrapper{ClassName: elseName, Cond: true}
+}
+
+func If(cond bool, part any) any {
+	if cond {
+		return part
+	}
+	return nil
+}
+
+func IfElse(cond bool, part any, elsePart any) any {
+	if cond {
+		return part
+	}
+	return elsePart
+}
+
+func ForEach[T any](items []T, fn func(T) any) []any {
+	var elems []any
+	for _, item := range items {
+		fnResult := fn(item)
+		elems = append(elems, fnResult)
+	}
+	return elems
+}
+
 func Props(props any) map[string]any {
 	m, err := utilfn.StructToMap(props)
 	if err != nil {
@@ -173,6 +261,31 @@ func UseState[T any](ctx context.Context, initialVal T) (T, func(T)) {
 	return rtnVal, setVal
 }
 
+func UseStateWithFn[T any](ctx context.Context, initialVal T) (T, func(T), func(func(T) T)) {
+	vc, hookVal := getHookFromCtx(ctx)
+	if !hookVal.Init {
+		hookVal.Init = true
+		hookVal.Val = initialVal
+	}
+	var rtnVal T
+	rtnVal, ok := hookVal.Val.(T)
+	if !ok {
+		panic("UseState hook value is not a state (possible out of order or conditional hooks)")
+	}
+
+	setVal := func(newVal T) {
+		hookVal.Val = newVal
+		vc.Root.AddRenderWork(vc.Comp.WaveId)
+	}
+
+	setFuncVal := func(updateFunc func(T) T) {
+		hookVal.Val = updateFunc(hookVal.Val.(T))
+		vc.Root.AddRenderWork(vc.Comp.WaveId)
+	}
+
+	return rtnVal, setVal, setFuncVal
+}
+
 func UseAtom[T any](ctx context.Context, atomName string) (T, func(T)) {
 	vc, hookVal := getHookFromCtx(ctx)
 	if !hookVal.Init {
@@ -212,6 +325,19 @@ func UseVDomRef(ctx context.Context) *VDomRef {
 	return refVal
 }
 
+func UseRef[T any](ctx context.Context, val T) *VDomSimpleRef[T] {
+	_, hookVal := getHookFromCtx(ctx)
+	if !hookVal.Init {
+		hookVal.Init = true
+		hookVal.Val = &VDomSimpleRef[T]{Current: val}
+	}
+	refVal, ok := hookVal.Val.(*VDomSimpleRef[T])
+	if !ok {
+		panic("UseRef hook value is not a ref (possible out of order or conditional hooks)")
+	}
+	return refVal
+}
+
 func UseId(ctx context.Context) string {
 	vc := getRenderContext(ctx)
 	if vc == nil {

pkg/vdom/vdom_root.go

@@ -12,6 +12,11 @@ import (
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
 )
 
+const (
+	BackendUpdate_InitialChunkSize = 50  // Size for initial chunks that contain both TransferElems and StateSync
+	BackendUpdate_ChunkSize        = 100 // Size for subsequent chunks
+)
+
 type vdomContextKeyType struct{}
 
 var vdomContextKey = vdomContextKeyType{}
@@ -470,14 +475,13 @@ func ConvertElemsToTransferElems(elems []VDomElem) []VDomTransferElem {
 	textCounter := 0 // Counter for generating unique IDs for #text nodes
 
 	// Helper function to recursively process each VDomElem in preorder
-	var processElem func(elem VDomElem, isRoot bool) string
+	var processElem func(elem VDomElem) string
-	processElem = func(elem VDomElem, isRoot bool) string {
+	processElem = func(elem VDomElem) string {
 		// Handle #text nodes by generating a unique placeholder ID
 		if elem.Tag == "#text" {
 			textId := fmt.Sprintf("text-%d", textCounter)
 			textCounter++
 			transferElems = append(transferElems, VDomTransferElem{
-				Root:     isRoot,
 				WaveId:   textId,
 				Tag:      elem.Tag,
 				Text:     elem.Text,
@@ -490,12 +494,11 @@ func ConvertElemsToTransferElems(elems []VDomElem) []VDomTransferElem {
 		// Convert children to WaveId references, handling potential #text nodes
 		childrenIds := make([]string, len(elem.Children))
 		for i, child := range elem.Children {
-			childrenIds[i] = processElem(child, false) // Children are not roots
+			childrenIds[i] = processElem(child) // Children are not roots
 		}
 
 		// Create the VDomTransferElem for the current element
 		transferElem := VDomTransferElem{
-			Root:     isRoot,
 			WaveId:   elem.WaveId,
 			Tag:      elem.Tag,
 			Props:    elem.Props,
@@ -509,8 +512,99 @@ func ConvertElemsToTransferElems(elems []VDomElem) []VDomTransferElem {
 
 	// Start processing each top-level element, marking them as roots
 	for _, elem := range elems {
-		processElem(elem, true)
+		processElem(elem)
 	}
 
 	return transferElems
 }
+
+func DedupTransferElems(elems []VDomTransferElem) []VDomTransferElem {
+	seen := make(map[string]int) // maps WaveId to its index in the result slice
+	var result []VDomTransferElem
+
+	for _, elem := range elems {
+		if idx, exists := seen[elem.WaveId]; exists {
+			// Overwrite the previous element with the latest one
+			result[idx] = elem
+		} else {
+			// Add new element and store its index
+			seen[elem.WaveId] = len(result)
+			result = append(result, elem)
+		}
+	}
+
+	return result
+}
+
+func (beUpdate *VDomBackendUpdate) CreateTransferElems() {
+	var vdomElems []VDomElem
+	for idx, reUpdate := range beUpdate.RenderUpdates {
+		if reUpdate.VDom == nil {
+			continue
+		}
+		vdomElems = append(vdomElems, *reUpdate.VDom)
+		beUpdate.RenderUpdates[idx].VDomWaveId = reUpdate.VDom.WaveId
+		beUpdate.RenderUpdates[idx].VDom = nil
+	}
+	transferElems := ConvertElemsToTransferElems(vdomElems)
+	transferElems = DedupTransferElems(transferElems)
+	beUpdate.TransferElems = transferElems
+}
+
+// SplitBackendUpdate splits a large VDomBackendUpdate into multiple smaller updates
+// The first update contains all the core fields, while subsequent updates only contain
+// array elements that need to be appended
+func SplitBackendUpdate(update *VDomBackendUpdate) []*VDomBackendUpdate {
+	// If the update is small enough, return it as is
+	if len(update.TransferElems) <= BackendUpdate_InitialChunkSize && len(update.StateSync) <= BackendUpdate_InitialChunkSize {
+		return []*VDomBackendUpdate{update}
+	}
+
+	var updates []*VDomBackendUpdate
+
+	// First update contains core fields and initial chunks
+	firstUpdate := &VDomBackendUpdate{
+		Type:          update.Type,
+		Ts:            update.Ts,
+		BlockId:       update.BlockId,
+		Opts:          update.Opts,
+		HasWork:       update.HasWork,
+		RenderUpdates: update.RenderUpdates,
+		RefOperations: update.RefOperations,
+		Messages:      update.Messages,
+	}
+
+	// Add initial chunks of arrays
+	if len(update.TransferElems) > 0 {
+		firstUpdate.TransferElems = update.TransferElems[:min(BackendUpdate_InitialChunkSize, len(update.TransferElems))]
+	}
+	if len(update.StateSync) > 0 {
+		firstUpdate.StateSync = update.StateSync[:min(BackendUpdate_InitialChunkSize, len(update.StateSync))]
+	}
+
+	updates = append(updates, firstUpdate)
+
+	// Create subsequent updates for remaining TransferElems
+	for i := BackendUpdate_InitialChunkSize; i < len(update.TransferElems); i += BackendUpdate_ChunkSize {
+		end := min(i+BackendUpdate_ChunkSize, len(update.TransferElems))
+		updates = append(updates, &VDomBackendUpdate{
+			Type:          update.Type,
+			Ts:            update.Ts,
+			BlockId:       update.BlockId,
+			TransferElems: update.TransferElems[i:end],
+		})
+	}
+
+	// Create subsequent updates for remaining StateSync
+	for i := BackendUpdate_InitialChunkSize; i < len(update.StateSync); i += BackendUpdate_ChunkSize {
+		end := min(i+BackendUpdate_ChunkSize, len(update.StateSync))
+		updates = append(updates, &VDomBackendUpdate{
+			Type:      update.Type,
+			Ts:        update.Ts,
+			BlockId:   update.BlockId,
+			StateSync: update.StateSync[i:end],
+		})
+	}
+
+	return updates
+}

pkg/vdom/vdom_types.go

@@ -33,7 +33,6 @@ type VDomElem struct {
 
 // the over the wire format for a vdom element
 type VDomTransferElem struct {
-	Root     bool           `json:"root,omitempty"`
 	WaveId   string         `json:"waveid,omitempty"` // required, except for #text nodes
 	Tag      string         `json:"tag"`
 	Props    map[string]any `json:"props,omitempty"`
@@ -86,6 +85,7 @@ type VDomBackendUpdate struct {
 	Opts          *VDomBackendOpts   `json:"opts,omitempty"`
 	HasWork       bool               `json:"haswork,omitempty"`
 	RenderUpdates []VDomRenderUpdate `json:"renderupdates,omitempty"`
+	TransferElems []VDomTransferElem `json:"transferelems,omitempty"`
 	StateSync     []VDomStateSync    `json:"statesync,omitempty"`
 	RefOperations []VDomRefOperation `json:"refoperations,omitempty"`
 	Messages      []VDomMessage      `json:"messages,omitempty"`
@@ -118,6 +118,10 @@ type VDomRef struct {
 	HasCurrent    bool             `json:"hascurrent,omitempty"`
 }
 
+type VDomSimpleRef[T any] struct {
+	Current T `json:"current"`
+}
+
 type DomRect struct {
 	Top    float64 `json:"top"`
 	Left   float64 `json:"left"`
@@ -176,10 +180,11 @@ type VDomBackendOpts struct {
 }
 
 type VDomRenderUpdate struct {
-	UpdateType string   `json:"updatetype" tstype:"\"root\"|\"append\"|\"replace\"|\"remove\"|\"insert\""`
+	UpdateType string    `json:"updatetype" tstype:"\"root\"|\"append\"|\"replace\"|\"remove\"|\"insert\""`
-	WaveId     string   `json:"waveid,omitempty"`
+	WaveId     string    `json:"waveid,omitempty"`
-	VDom       VDomElem `json:"vdom"`
+	VDomWaveId string    `json:"vdomwaveid,omitempty"`
-	Index      *int     `json:"index,omitempty"`
+	VDom       *VDomElem `json:"vdom,omitempty"` // these get removed for transfer (encoded to transferelems)
+	Index      *int      `json:"index,omitempty"`
 }
 
 type VDomRefOperation struct {

pkg/vdom/vdomclient/vdomclient.go

@@ -6,6 +6,8 @@ package vdomclient
 import (
 	"context"
 	"fmt"
+	"io"
+	"io/fs"
 	"log"
 	"net/http"
 	"os"
@@ -207,7 +209,7 @@ func (c *Client) fullRender() (*vdom.VDomBackendUpdate, error) {
 		HasWork: len(c.Root.EffectWorkQueue) > 0,
 		Opts:    &c.Opts,
 		RenderUpdates: []vdom.VDomRenderUpdate{
-			{UpdateType: "root", VDom: *renderedVDom},
+			{UpdateType: "root", VDom: renderedVDom},
 		},
 		StateSync: c.Root.GetStateSync(true),
 	}, nil
@@ -224,7 +226,7 @@ func (c *Client) incrementalRender() (*vdom.VDomBackendUpdate, error) {
 		Ts:      time.Now().UnixMilli(),
 		BlockId: c.RpcContext.BlockId,
 		RenderUpdates: []vdom.VDomRenderUpdate{
-			{UpdateType: "root", VDom: *renderedVDom},
+			{UpdateType: "root", VDom: renderedVDom},
 		},
 		StateSync: c.Root.GetStateSync(false),
 	}, nil
@@ -234,9 +236,111 @@ func (c *Client) RegisterUrlPathHandler(path string, handler http.Handler) {
 	c.UrlHandlerMux.Handle(path, handler)
 }
 
-func (c *Client) RegisterFileHandler(path string, fileName string) {
+type FileHandlerOption struct {
-	fileName = wavebase.ExpandHomeDirSafe(fileName)
+	FilePath string    // optional file path on disk
+	Data     []byte    // optional byte slice content
+	Reader   io.Reader // optional reader for content
+	File     fs.File   // optional embedded or opened file
+	MimeType string    // optional mime type
+}
+
+func determineMimeType(option FileHandlerOption) (string, []byte) {
+	// If MimeType is set, use it directly
+	if option.MimeType != "" {
+		return option.MimeType, nil
+	}
+
+	// Detect from Data if available, no need to buffer
+	if option.Data != nil {
+		return http.DetectContentType(option.Data), nil
+	}
+
+	// Detect from FilePath, no buffering necessary
+	if option.FilePath != "" {
+		filePath := wavebase.ExpandHomeDirSafe(option.FilePath)
+		file, err := os.Open(filePath)
+		if err != nil {
+			return "application/octet-stream", nil // Fallback on error
+		}
+		defer file.Close()
+
+		// Read first 512 bytes for MIME detection
+		buf := make([]byte, 512)
+		_, err = file.Read(buf)
+		if err != nil && err != io.EOF {
+			return "application/octet-stream", nil
+		}
+		return http.DetectContentType(buf), nil
+	}
+
+	// Buffer for File (fs.File), since it lacks Seek
+	if option.File != nil {
+		buf := make([]byte, 512)
+		n, err := option.File.Read(buf)
+		if err != nil && err != io.EOF {
+			return "application/octet-stream", nil
+		}
+		return http.DetectContentType(buf[:n]), buf[:n]
+	}
+
+	// Buffer for Reader (io.Reader), same as File
+	if option.Reader != nil {
+		buf := make([]byte, 512)
+		n, err := option.Reader.Read(buf)
+		if err != nil && err != io.EOF {
+			return "application/octet-stream", nil
+		}
+		return http.DetectContentType(buf[:n]), buf[:n]
+	}
+
+	// Default MIME type if none specified
+	return "application/octet-stream", nil
+}
+
+func (c *Client) RegisterFileHandler(path string, option FileHandlerOption) {
 	c.UrlHandlerMux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
-		http.ServeFile(w, r, fileName)
+		// Determine MIME type and get buffered data if needed
+		contentType, bufferedData := determineMimeType(option)
+		w.Header().Set("Content-Type", contentType)
+
+		if option.FilePath != "" {
+			// Serve file from path
+			filePath := wavebase.ExpandHomeDirSafe(option.FilePath)
+			http.ServeFile(w, r, filePath)
+		} else if option.Data != nil {
+			// Set content length and serve content from in-memory data
+			w.Header().Set("Content-Length", fmt.Sprintf("%d", len(option.Data)))
+			w.WriteHeader(http.StatusOK) // Ensure headers are sent before writing body
+			if _, err := w.Write(option.Data); err != nil {
+				http.Error(w, "Failed to serve content", http.StatusInternalServerError)
+			}
+		} else if option.File != nil {
+			// Write buffered data if available, then continue with remaining File content
+			if bufferedData != nil {
+				w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bufferedData)))
+				if _, err := w.Write(bufferedData); err != nil {
+					http.Error(w, "Failed to serve content", http.StatusInternalServerError)
+					return
+				}
+			}
+			// Serve remaining content from File
+			if _, err := io.Copy(w, option.File); err != nil {
+				http.Error(w, "Failed to serve content", http.StatusInternalServerError)
+			}
+		} else if option.Reader != nil {
+			// Write buffered data if available, then continue with remaining Reader content
+			if bufferedData != nil {
+				if _, err := w.Write(bufferedData); err != nil {
+					http.Error(w, "Failed to serve content", http.StatusInternalServerError)
+					return
+				}
+			}
+			// Serve remaining content from Reader
+			if _, err := io.Copy(w, option.Reader); err != nil {
+				http.Error(w, "Failed to serve content", http.StatusInternalServerError)
+			}
+		} else {
+			http.Error(w, "No content available", http.StatusNotFound)
+		}
 	})
 }

pkg/vdom/vdomclient/vdomserverimpl.go

@@ -21,15 +21,31 @@ type VDomServerImpl struct {
 
 func (*VDomServerImpl) WshServerImpl() {}
 
-func (impl *VDomServerImpl) VDomRenderCommand(ctx context.Context, feUpdate vdom.VDomFrontendUpdate) (*vdom.VDomBackendUpdate, error) {
+func (impl *VDomServerImpl) VDomRenderCommand(ctx context.Context, feUpdate vdom.VDomFrontendUpdate) chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate] {
+	respChan := make(chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate], 5)
+
+	defer func() {
+		if r := recover(); r != nil {
+			log.Printf("panic in VDomRenderCommand: %v\n", r)
+			respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{
+				Error: fmt.Errorf("internal error: %v", r),
+			}
+			close(respChan)
+		}
+	}()
+
 	if feUpdate.Dispose {
+		defer close(respChan)
 		log.Printf("got dispose from frontend\n")
 		impl.Client.doShutdown("got dispose from frontend")
-		return nil, nil
+		return respChan
 	}
+
 	if impl.Client.GetIsDone() {
-		return nil, nil
+		close(respChan)
+		return respChan
 	}
+
 	// set atoms
 	for _, ss := range feUpdate.StateSync {
 		impl.Client.Root.SetAtomVal(ss.Atom, ss.Value, false)
@@ -44,10 +60,37 @@ func (impl *VDomServerImpl) VDomRenderCommand(ctx context.Context, feUpdate vdom
 			impl.Client.Root.Event(event.WaveId, event.EventType, event)
 		}
 	}
+
+	var update *vdom.VDomBackendUpdate
+	var err error
+
 	if feUpdate.Resync || true {
-		return impl.Client.fullRender()
+		update, err = impl.Client.fullRender()
+	} else {
+		update, err = impl.Client.incrementalRender()
 	}
-	return impl.Client.incrementalRender()
+	update.CreateTransferElems()
+
+	if err != nil {
+		respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{
+			Error: err,
+		}
+		close(respChan)
+		return respChan
+	}
+
+	// Split the update into chunks and send them sequentially
+	updates := vdom.SplitBackendUpdate(update)
+	go func() {
+		defer close(respChan)
+		for _, splitUpdate := range updates {
+			respChan <- wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate]{
+				Response: splitUpdate,
+			}
+		}
+	}()
+
+	return respChan
 }
 
 func (impl *VDomServerImpl) VDomUrlRequestCommand(ctx context.Context, data wshrpc.VDomUrlRequestData) chan wshrpc.RespOrErrorUnion[wshrpc.VDomUrlRequestResponse] {
@@ -56,13 +99,14 @@ func (impl *VDomServerImpl) VDomUrlRequestCommand(ctx context.Context, data wshr
 	writer := NewStreamingResponseWriter(respChan)
 
 	go func() {
+		defer close(respChan) // Declared first, so it executes last
+		defer writer.Close()  // Ensures writer is closed before the channel is closed
+
 		defer func() {
 			if r := recover(); r != nil {
-				// On panic, send 500 status code
 				writer.WriteHeader(http.StatusInternalServerError)
 				writer.Write([]byte(fmt.Sprintf("internal server error: %v", r)))
 			}
-			close(respChan)
 		}()
 
 		// Create an HTTP request from the RPC request data

pkg/wshrpc/wshclient/wshclient.go

@@ -298,9 +298,8 @@ func VDomCreateContextCommand(w *wshutil.WshRpc, data vdom.VDomCreateContext, op
 }
 
 // command "vdomrender", wshserver.VDomRenderCommand
-func VDomRenderCommand(w *wshutil.WshRpc, data vdom.VDomFrontendUpdate, opts *wshrpc.RpcOpts) (*vdom.VDomBackendUpdate, error) {
+func VDomRenderCommand(w *wshutil.WshRpc, data vdom.VDomFrontendUpdate, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[*vdom.VDomBackendUpdate] {
-	resp, err := sendRpcRequestCallHelper[*vdom.VDomBackendUpdate](w, "vdomrender", data, opts)
+	return sendRpcRequestResponseStreamHelper[*vdom.VDomBackendUpdate](w, "vdomrender", data, opts)
-	return resp, err
 }
 
 // command "vdomurlrequest", wshserver.VDomUrlRequestCommand

pkg/wshrpc/wshrpctypes.go

@@ -157,7 +157,7 @@ type WshRpcInterface interface {
 	VDomAsyncInitiationCommand(ctx context.Context, data vdom.VDomAsyncInitiationRequest) error
 
 	// proc
-	VDomRenderCommand(ctx context.Context, data vdom.VDomFrontendUpdate) (*vdom.VDomBackendUpdate, error)
+	VDomRenderCommand(ctx context.Context, data vdom.VDomFrontendUpdate) chan RespOrErrorUnion[*vdom.VDomBackendUpdate]
 	VDomUrlRequestCommand(ctx context.Context, data VDomUrlRequestData) chan RespOrErrorUnion[VDomUrlRequestResponse]
 }