Allow separate directories for each config part, add dropdown for editing AI presets (#1074)

Adds new functionality on the backend that will merge any file from the
config directory that matches `<partName>.json` or `<partName>/*.json`
into the corresponding config part (presets, termthemes, etc.). This
lets us separate the AI presets into `presets/ai.json` so that we can
add a dropdown in the AI preset selector that will directly open the
file so a user can edit it more easily. Right now, this will create a
preview block in the layout, but in the future we can look into making
this block disconnected from the layout.

If you put AI presets in the regular presets.json file, it will still
work, since all the presets get merged. Same for any other config part.

cmd/server/main-server.go

@@ -192,11 +192,18 @@ func main() {
 		log.Printf("error ensuring wave db dir: %v\n", err)
 		return
 	}
-	err = wavebase.EnsureWaveConfigDir()
+	err = wconfig.EnsureWaveConfigDir()
 	if err != nil {
 		log.Printf("error ensuring wave config dir: %v\n", err)
 		return
 	}
+
+	// TODO: rather than ensure this dir exists, we should let the editor recursively create parent dirs on save
+	err = wconfig.EnsureWavePresetsDir()
+	if err != nil {
+		log.Printf("error ensuring wave presets dir: %v\n", err)
+		return
+	}
 	waveLock, err := wavebase.AcquireWaveLock()
 	if err != nil {
 		log.Printf("error acquiring wave lock (another instance of Wave is likely running): %v\n", err)

emain/platform.ts

@@ -23,23 +23,6 @@ const unamePlatform = process.platform;
 const unameArch: string = process.arch;
 keyutil.setKeyUtilPlatform(unamePlatform);
 
-ipcMain.on("get-is-dev", (event) => {
-    event.returnValue = isDev;
-});
-ipcMain.on("get-platform", (event, url) => {
-    event.returnValue = unamePlatform;
-});
-ipcMain.on("get-user-name", (event) => {
-    const userInfo = os.userInfo();
-    event.returnValue = userInfo.username;
-});
-ipcMain.on("get-host-name", (event) => {
-    event.returnValue = os.hostname();
-});
-ipcMain.on("get-webview-preload", (event) => {
-    event.returnValue = path.join(getElectronAppBasePath(), "preload", "preload-webview.cjs");
-});
-
 // must match golang
 function getWaveHomeDir() {
     const override = process.env[WaveHomeVarName];
@@ -72,6 +55,26 @@ function getWaveSrvCwd(): string {
     return getWaveHomeDir();
 }
 
+ipcMain.on("get-is-dev", (event) => {
+    event.returnValue = isDev;
+});
+ipcMain.on("get-platform", (event, url) => {
+    event.returnValue = unamePlatform;
+});
+ipcMain.on("get-user-name", (event) => {
+    const userInfo = os.userInfo();
+    event.returnValue = userInfo.username;
+});
+ipcMain.on("get-host-name", (event) => {
+    event.returnValue = os.hostname();
+});
+ipcMain.on("get-webview-preload", (event) => {
+    event.returnValue = path.join(getElectronAppBasePath(), "preload", "preload-webview.cjs");
+});
+ipcMain.on("get-config-dir", (event) => {
+    event.returnValue = path.join(getWaveHomeDir(), "config");
+});
+
 export {
     getElectronAppBasePath,
     getElectronAppUnpackedBasePath,

emain/preload.ts

@@ -10,6 +10,7 @@ contextBridge.exposeInMainWorld("api", {
     getCursorPoint: () => ipcRenderer.sendSync("get-cursor-point"),
     getUserName: () => ipcRenderer.sendSync("get-user-name"),
     getHostName: () => ipcRenderer.sendSync("get-host-name"),
+    getConfigDir: () => ipcRenderer.sendSync("get-config-dir"),
     getAboutModalDetails: () => ipcRenderer.sendSync("get-about-modal-details"),
     getDocsiteUrl: () => ipcRenderer.sendSync("get-docsite-url"),
     getWebviewPreload: () => ipcRenderer.sendSync("get-webview-preload"),

frontend/app/store/global.ts

@@ -63,7 +63,6 @@ function initGlobalAtoms(initOpts: GlobalInitOptions) {
         // do nothing
     }
 
-    const showAboutModalAtom = atom(false) as PrimitiveAtom<boolean>;
     try {
         getApi().onMenuItemAbout(() => {
             modalsModel.pushModal("AboutModal");

frontend/app/view/waveai/waveai.tsx

@@ -6,7 +6,7 @@ import { Markdown } from "@/app/element/markdown";
 import { TypingIndicator } from "@/app/element/typingindicator";
 import { RpcApi } from "@/app/store/wshclientapi";
 import { TabRpcClient } from "@/app/store/wshrpcutil";
-import { atoms, fetchWaveFile, globalStore, WOS } from "@/store/global";
+import { atoms, createBlock, fetchWaveFile, getApi, globalStore, WOS } from "@/store/global";
 import { BlockService, ObjectService } from "@/store/services";
 import { adaptFromReactOrNativeKeyEvent, checkKeyPressed } from "@/util/keyutil";
 import { fireAndForget, isBlank, makeIconClass } from "@/util/util";
@@ -182,26 +182,41 @@ export class WaveAiModel implements ViewModel {
                     });
                 }
             }
-
+            const dropdownItems = Object.entries(presets)
+                .sort((a, b) => (a[1]["display:order"] > b[1]["display:order"] ? 1 : -1))
+                .map(
+                    (preset) =>
+                        ({
+                            label: preset[1]["display:name"],
+                            onClick: () =>
+                                fireAndForget(async () => {
+                                    await ObjectService.UpdateObjectMeta(WOS.makeORef("block", this.blockId), {
+                                        ...preset[1],
+                                        "ai:preset": preset[0],
+                                    });
+                                }),
+                        }) as MenuItem
+                );
+            dropdownItems.push({
+                label: "Add AI preset...",
+                onClick: () => {
+                    fireAndForget(async () => {
+                        const path = `${getApi().getConfigDir()}/presets/ai.json`;
+                        const blockDef: BlockDef = {
+                            meta: {
+                                view: "preview",
+                                file: path,
+                            },
+                        };
+                        await createBlock(blockDef, true);
+                    });
+                },
+            });
             viewTextChildren.push({
                 elemtype: "menubutton",
                 text: presetName,
                 title: "Select AI Configuration",
-                items: Object.entries(presets)
+                items: dropdownItems,
-                    .sort((a, b) => (a[1]["display:order"] > b[1]["display:order"] ? 1 : -1))
-                    .map(
-                        (preset) =>
-                            ({
-                                label: preset[1]["display:name"],
-                                onClick: () =>
-                                    fireAndForget(async () => {
-                                        await ObjectService.UpdateObjectMeta(WOS.makeORef("block", this.blockId), {
-                                            ...preset[1],
-                                            "ai:preset": preset[0],
-                                        });
-                                    }),
-                            }) as MenuItem
-                    ),
             });
             return viewTextChildren;
         });

frontend/types/custom.d.ts

@@ -64,6 +64,7 @@ declare global {
         getEnv: (varName: string) => string;
         getUserName: () => string;
         getHostName: () => string;
+        getConfigDir: () => string;
         getWebviewPreload: () => string;
         getAboutModalDetails: () => AboutModalDetails;
         getDocsiteUrl: () => string;

pkg/wavebase/wavebase.go

@@ -119,10 +119,6 @@ func EnsureWaveDBDir() error {
 	return CacheEnsureDir(filepath.Join(GetWaveHomeDir(), WaveDBDir), "wavedb", 0700, "wave db directory")
 }
 
-func EnsureWaveConfigDir() error {
-	return CacheEnsureDir(filepath.Join(GetWaveHomeDir(), ConfigDir), "waveconfig", 0700, "wave config directory")
-}
-
 func CacheEnsureDir(dirName string, cacheKey string, perm os.FileMode, dirDesc string) error {
 	baseLock.Lock()
 	ok := ensureDirCache[cacheKey]

pkg/wconfig/defaultconfig/defaultconfig.go

@@ -5,5 +5,5 @@ package defaultconfig
 
 import "embed"
 
-//go:embed *.json
+//go:embed *.json all:*/*.json
 var ConfigFS embed.FS

pkg/wconfig/defaultconfig/presets.json

@@ -94,23 +94,5 @@
         "bg": "linear-gradient(120deg,hsla(350, 65%, 57%, 1),hsla(30,60%,60%, .75), hsla(208,69%,50%,.15), hsl(230,60%,40%)),radial-gradient(at top right,hsla(300,60%,70%,0.3),transparent),radial-gradient(at top left,hsla(330,100%,70%,.20),transparent),radial-gradient(at top right,hsla(190,100%,40%,.20),transparent),radial-gradient(at bottom left,hsla(323,54%,50%,.5),transparent),radial-gradient(at bottom left,hsla(144,54%,50%,.25),transparent)",
         "bg:blendmode": "overlay",
         "bg:text": "rgb(200, 200, 200)"
-    },
-    "ai@global": {
-        "display:name": "Global default",
-        "display:order": -1,
-        "ai:*": true
-    },
-    "ai@wave": {
-        "display:name": "Wave Proxy - gpt-4o-mini",
-        "display:order": 0,
-        "ai:*": true,
-        "ai:apitype": "",
-        "ai:baseurl": "",
-        "ai:apitoken": "",
-        "ai:name": "",
-        "ai:orgid": "",
-        "ai:model": "gpt-4o-mini",
-        "ai:maxtokens": 2048,
-        "ai:timeoutms": 60000
     }
 }

pkg/wconfig/defaultconfig/presets/ai.json

@@ -0,0 +1,20 @@
+{
+    "ai@global": {
+        "display:name": "Global default",
+        "display:order": -1,
+        "ai:*": true
+    },
+    "ai@wave": {
+        "display:name": "Wave Proxy - gpt-4o-mini",
+        "display:order": 0,
+        "ai:*": true,
+        "ai:apitype": "",
+        "ai:baseurl": "",
+        "ai:apitoken": "",
+        "ai:name": "",
+        "ai:orgid": "",
+        "ai:model": "gpt-4o-mini",
+        "ai:maxtokens": 2048,
+        "ai:timeoutms": 60000
+    }
+}

pkg/wconfig/filewatcher.go

@@ -40,8 +40,17 @@ func GetWatcher() *Watcher {
 		}
 		instance = &Watcher{watcher: watcher}
 		err = instance.watcher.Add(configDirAbsPath)
+		const failedStr = "failed to add path %s to watcher: %v"
 		if err != nil {
-			log.Printf("failed to add path %s to watcher: %v", configDirAbsPath, err)
+			log.Printf(failedStr, configDirAbsPath, err)
+		}
+
+		subdirs := GetConfigSubdirs()
+		for _, dir := range subdirs {
+			err = instance.watcher.Add(dir)
+			if err != nil {
+				log.Printf(failedStr, dir, err)
+			}
 		}
 	})
 	return instance

pkg/wconfig/settingsconfig.go

@@ -7,6 +7,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/fs"
+	"log"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -14,6 +16,7 @@ import (
 	"strings"
 
 	"github.com/wavetermdev/waveterm/pkg/util/utilfn"
+	"github.com/wavetermdev/waveterm/pkg/wavebase"
 	"github.com/wavetermdev/waveterm/pkg/waveobj"
 	"github.com/wavetermdev/waveterm/pkg/wconfig/defaultconfig"
 )
@@ -181,15 +184,19 @@ func readConfigHelper(fileName string, barr []byte, readErr error) (waveobj.Meta
 	return rtn, cerrs
 }
 
+var configDirFsys = os.DirFS(configDirAbsPath)
+
+func readConfigFileFS(fsys fs.FS, logPrefix string, fileName string) (waveobj.MetaMapType, []ConfigError) {
+	barr, readErr := fs.ReadFile(fsys, fileName)
+	return readConfigHelper(logPrefix+fileName, barr, readErr)
+}
+
 func ReadDefaultsConfigFile(fileName string) (waveobj.MetaMapType, []ConfigError) {
-	barr, readErr := defaultconfig.ConfigFS.ReadFile(fileName)
+	return readConfigFileFS(defaultconfig.ConfigFS, "defaults:", fileName)
-	return readConfigHelper("defaults:"+fileName, barr, readErr)
 }
 
 func ReadWaveHomeConfigFile(fileName string) (waveobj.MetaMapType, []ConfigError) {
-	fullFileName := filepath.Join(configDirAbsPath, fileName)
+	return readConfigFileFS(configDirFsys, "", fileName)
-	barr, err := os.ReadFile(fullFileName)
-	return readConfigHelper(fullFileName, barr, err)
 }
 
 func WriteWaveHomeConfigFile(fileName string, m waveobj.MetaMapType) error {
@@ -222,17 +229,69 @@ func mergeMetaMapSimple(m waveobj.MetaMapType, toMerge waveobj.MetaMapType) wave
 	return m
 }
 
-func ReadConfigPart(partName string, simpleMerge bool) (waveobj.MetaMapType, []ConfigError) {
+func mergeMetaMap(m waveobj.MetaMapType, toMerge waveobj.MetaMapType, simpleMerge bool) waveobj.MetaMapType {
-	defConfig, cerrs1 := ReadDefaultsConfigFile(partName)
-	userConfig, cerrs2 := ReadWaveHomeConfigFile(partName)
-	allErrs := append(cerrs1, cerrs2...)
 	if simpleMerge {
-		return mergeMetaMapSimple(defConfig, userConfig), allErrs
+		return mergeMetaMapSimple(m, toMerge)
 	} else {
-		return waveobj.MergeMeta(defConfig, userConfig, true), allErrs
+		return waveobj.MergeMeta(m, toMerge, true)
 	}
 }
 
+func selectDirEntsBySuffix(dirEnts []fs.DirEntry, fileNameSuffix string) []fs.DirEntry {
+	var rtn []fs.DirEntry
+	for _, ent := range dirEnts {
+		if ent.IsDir() {
+			continue
+		}
+		if !strings.HasSuffix(ent.Name(), fileNameSuffix) {
+			continue
+		}
+		rtn = append(rtn, ent)
+	}
+	return rtn
+}
+
+func SortFileNameDescend(files []fs.DirEntry) {
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].Name() > files[j].Name()
+	})
+}
+
+// Read and merge all files in the specified directory matching the supplied suffix
+func readConfigFilesForDir(fsys fs.FS, logPrefix string, dirName string, fileName string, simpleMerge bool) (waveobj.MetaMapType, []ConfigError) {
+	dirEnts, _ := fs.ReadDir(fsys, dirName)
+	suffixEnts := selectDirEntsBySuffix(dirEnts, fileName+".json")
+	SortFileNameDescend(suffixEnts)
+	var rtn waveobj.MetaMapType
+	var errs []ConfigError
+	for _, ent := range suffixEnts {
+		fileVal, cerrs := readConfigFileFS(fsys, logPrefix, filepath.Join(dirName, ent.Name()))
+		rtn = mergeMetaMap(rtn, fileVal, simpleMerge)
+		errs = append(errs, cerrs...)
+	}
+	return rtn, errs
+}
+
+// Read and merge all files in the specified config filesystem matching the patterns `<partName>.json` and `<partName>/*.json`
+func readConfigPartForFS(fsys fs.FS, logPrefix string, partName string, simpleMerge bool) (waveobj.MetaMapType, []ConfigError) {
+	config, errs := readConfigFilesForDir(fsys, logPrefix, partName, "", simpleMerge)
+	allErrs := errs
+	rtn := config
+	config, errs = readConfigFileFS(fsys, logPrefix, partName+".json")
+	allErrs = append(allErrs, errs...)
+	return mergeMetaMap(rtn, config, simpleMerge), allErrs
+}
+
+// Combine files from the defaults and home directory for the specified config part name
+func readConfigPart(partName string, simpleMerge bool) (waveobj.MetaMapType, []ConfigError) {
+	defaultConfigs, cerrs := readConfigPartForFS(defaultconfig.ConfigFS, "defaults:", partName, simpleMerge)
+	homeConfigs, cerrs1 := readConfigPartForFS(configDirFsys, "", partName, simpleMerge)
+
+	rtn := defaultConfigs
+	allErrs := append(cerrs, cerrs1...)
+	return mergeMetaMap(rtn, homeConfigs, simpleMerge), allErrs
+}
+
 func ReadFullConfig() FullConfigType {
 	var fullConfig FullConfigType
 	configRType := reflect.TypeOf(fullConfig)
@@ -247,13 +306,15 @@ func ReadFullConfig() FullConfigType {
 			continue
 		}
 		jsonTag := utilfn.GetJsonTag(field)
+		simpleMerge := field.Tag.Get("merge") == ""
+		var configPart waveobj.MetaMapType
+		var errs []ConfigError
 		if jsonTag == "-" || jsonTag == "" {
 			continue
+		} else {
+			configPart, errs = readConfigPart(jsonTag, simpleMerge)
 		}
-		simpleMerge := field.Tag.Get("merge") == ""
+		fullConfig.ConfigErrors = append(fullConfig.ConfigErrors, errs...)
-		fileName := jsonTag + ".json"
-		configPart, cerrs := ReadConfigPart(fileName, simpleMerge)
-		fullConfig.ConfigErrors = append(fullConfig.ConfigErrors, cerrs...)
 		if configPart != nil {
 			fieldPtr := configRVal.Field(fieldIdx).Addr().Interface()
 			utilfn.ReUnmarshal(fieldPtr, configPart)
@@ -262,6 +323,28 @@ func ReadFullConfig() FullConfigType {
 	return fullConfig
 }
 
+func GetConfigSubdirs() []string {
+	var fullConfig FullConfigType
+	configRType := reflect.TypeOf(fullConfig)
+	var retVal []string
+	for fieldIdx := 0; fieldIdx < configRType.NumField(); fieldIdx++ {
+		field := configRType.Field(fieldIdx)
+		if field.PkgPath != "" {
+			continue
+		}
+		configFile := field.Tag.Get("configfile")
+		if configFile == "-" {
+			continue
+		}
+		jsonTag := utilfn.GetJsonTag(field)
+		if jsonTag != "-" && jsonTag != "" && jsonTag != "settings" {
+			retVal = append(retVal, filepath.Join(configDirAbsPath, jsonTag))
+		}
+	}
+	log.Printf("subdirs: %v\n", retVal)
+	return retVal
+}
+
 func getConfigKeyType(configKey string) reflect.Type {
 	ctype := reflect.TypeOf(SettingsType{})
 	for i := 0; i < ctype.NumField(); i++ {
@@ -415,6 +498,14 @@ func SetBaseConfigValue(toMerge waveobj.MetaMapType) error {
 	return WriteWaveHomeConfigFile(SettingsFile, m)
 }
 
+func EnsureWaveConfigDir() error {
+	return wavebase.CacheEnsureDir(configDirAbsPath, "waveconfig", 0700, "wave config directory")
+}
+
+func EnsureWavePresetsDir() error {
+	return wavebase.CacheEnsureDir(filepath.Join(configDirAbsPath, "presets"), "wavepresets", 0700, "wave presets directory")
+}
+
 type WidgetConfigType struct {
 	DisplayOrder float64          `json:"display:order,omitempty"`
 	Icon         string           `json:"icon,omitempty"`