From 8651659c02c5c4c699c9f5a96d8c1cb6279cbdac Mon Sep 17 00:00:00 2001 From: Mike Sawka Date: Mon, 19 Aug 2024 11:02:40 -0700 Subject: [PATCH] get remote preview working (#246) almost all there -- just need to fix streamfile for web urls. --- frontend/app/store/services.ts | 12 +- frontend/app/store/wshserver.ts | 5 + .../app/view/preview/directorypreview.tsx | 17 +- frontend/app/view/preview/preview.tsx | 9 +- frontend/types/gotypes.d.ts | 2 +- pkg/service/fileservice/fileservice.go | 158 +++++++++--------- pkg/wshrpc/wshclient/wshclient.go | 6 + pkg/wshrpc/wshremote/wshremote.go | 43 +++-- pkg/wshrpc/wshrpctypes.go | 6 +- 9 files changed, 155 insertions(+), 103 deletions(-) diff --git a/frontend/app/store/services.ts b/frontend/app/store/services.ts index b8a122063..c3c63653b 100644 --- a/frontend/app/store/services.ts +++ b/frontend/app/store/services.ts @@ -56,7 +56,9 @@ class FileServiceType { AddWidget(arg1: WidgetsConfigType): Promise { return WOS.callBackendService("file", "AddWidget", Array.from(arguments)) } - DeleteFile(arg1: string): Promise { + + // delete file + DeleteFile(connection: string, path: string): Promise { return WOS.callBackendService("file", "DeleteFile", Array.from(arguments)) } GetSettingsConfig(): Promise { @@ -65,13 +67,17 @@ class FileServiceType { GetWaveFile(arg1: string, arg2: string): Promise { return WOS.callBackendService("file", "GetWaveFile", Array.from(arguments)) } - ReadFile(arg1: string): Promise { + + // read file + ReadFile(connection: string, path: string): Promise { return WOS.callBackendService("file", "ReadFile", Array.from(arguments)) } RemoveWidget(arg1: number): Promise { return WOS.callBackendService("file", "RemoveWidget", Array.from(arguments)) } - SaveFile(arg1: string, arg2: string): Promise { + + // save file + SaveFile(connection: string, path: string, data64: string): Promise { return WOS.callBackendService("file", "SaveFile", Array.from(arguments)) } diff --git a/frontend/app/store/wshserver.ts b/frontend/app/store/wshserver.ts index 8822e0256..fc2b15686 100644 --- a/frontend/app/store/wshserver.ts +++ b/frontend/app/store/wshserver.ts @@ -92,6 +92,11 @@ class WshServerType { return WOS.wshServerRpcHelper_call("message", data, opts); } + // command "remotefiledelete" [call] + RemoteFileDeleteCommand(data: string, opts?: RpcOpts): Promise { + return WOS.wshServerRpcHelper_call("remotefiledelete", data, opts); + } + // command "remotefileinfo" [call] RemoteFileInfoCommand(data: string, opts?: RpcOpts): Promise { return WOS.wshServerRpcHelper_call("remotefileinfo", data, opts); diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index bffe11cd7..d54c65099 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -29,6 +29,7 @@ import { OverlayScrollbars } from "overlayscrollbars"; import "./directorypreview.less"; interface DirectoryTableProps { + model: PreviewModel; data: FileInfo[]; search: string; focusIndex: number; @@ -124,6 +125,7 @@ function cleanMimetype(input: string): string { } function DirectoryTable({ + model, data, search, focusIndex, @@ -294,6 +296,7 @@ function DirectoryTable({ {table.getState().columnSizingInfo.isResizingColumn ? ( ) : ( ; table: Table; search: string; @@ -334,6 +339,7 @@ interface TableBodyProps { } function TableBody({ + model, data, table, search, @@ -353,6 +359,7 @@ function TableBody({ const rowRefs = useRef([]); const parentHeight = useHeight(parentRef); + const conn = jotai.useAtomValue(model.connection); useEffect(() => { if (dummyLineRef.current && data && parentRef.current) { @@ -454,13 +461,13 @@ function TableBody({ menu.push({ label: "Delete File", click: async () => { - await services.FileService.DeleteFile(path).catch((e) => console.log(e)); + await services.FileService.DeleteFile(conn, path).catch((e) => console.log(e)); setRefreshVersion((current) => current + 1); }, }); ContextMenuModel.showContextMenu(menu, e); }, - [setRefreshVersion] + [setRefreshVersion, conn] ); const displayRow = useCallback( @@ -541,6 +548,7 @@ function DirectoryPreview({ fileNameAtom, model }: DirectoryPreviewProps) { const showHiddenFiles = jotai.useAtomValue(model.showHiddenFiles); const [selectedPath, setSelectedPath] = useState(""); const [refreshVersion, setRefreshVersion] = jotai.useAtom(model.refreshVersion); + const conn = jotai.useAtomValue(model.connection); useEffect(() => { model.refreshCallback = () => { @@ -553,13 +561,13 @@ function DirectoryPreview({ fileNameAtom, model }: DirectoryPreviewProps) { useEffect(() => { const getContent = async () => { - const file = await services.FileService.ReadFile(fileName); + const file = await services.FileService.ReadFile(conn, fileName); const serializedContent = util.base64ToString(file?.data64); const content: FileInfo[] = JSON.parse(serializedContent); setUnfilteredData(content); }; getContent(); - }, [fileName, refreshVersion]); + }, [conn, fileName, refreshVersion]); useEffect(() => { const filtered = unfilteredData.filter((fileInfo) => { @@ -633,6 +641,7 @@ function DirectoryPreview({ fileNameAtom, model }: DirectoryPreviewProps) { /> >(async (get) => { @@ -253,8 +251,9 @@ export class PreviewModel implements ViewModel { async handleFileSave() { const fileName = globalStore.get(this.fileName); const newFileContent = globalStore.get(this.newFileContent); + const conn = globalStore.get(this.connection) ?? ""; try { - services.FileService.SaveFile(fileName, util.stringToBase64(newFileContent)); + services.FileService.SaveFile(conn, fileName, util.stringToBase64(newFileContent)); globalStore.set(this.newFileContent, null); } catch (error) { console.error("Error saving file:", error); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index bf410cd98..86b3c7916 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -124,7 +124,7 @@ declare global { // wshrpc.CommandRemoteStreamFileRtnData type CommandRemoteStreamFileRtnData = { - fileinfo?: FileInfo; + fileinfo?: FileInfo[]; data64?: string; }; diff --git a/pkg/service/fileservice/fileservice.go b/pkg/service/fileservice/fileservice.go index 0b222f750..4e8d05a8f 100644 --- a/pkg/service/fileservice/fileservice.go +++ b/pkg/service/fileservice/fileservice.go @@ -1,19 +1,16 @@ package fileservice import ( + "bytes" "context" "encoding/base64" "encoding/json" "fmt" - "log" - "os" - "path/filepath" + "io" "time" "github.com/wavetermdev/thenextwave/pkg/filestore" "github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta" - "github.com/wavetermdev/thenextwave/pkg/util/utilfn" - "github.com/wavetermdev/thenextwave/pkg/wavebase" "github.com/wavetermdev/thenextwave/pkg/wconfig" "github.com/wavetermdev/thenextwave/pkg/wshrpc" "github.com/wavetermdev/thenextwave/pkg/wshrpc/wshclient" @@ -31,17 +28,21 @@ type FullFile struct { Data64 string `json:"data64"` // base64 encoded } -func (fs *FileService) SaveFile(path string, data64 string) error { - cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path)) - data, err := base64.StdEncoding.DecodeString(data64) - if err != nil { - return fmt.Errorf("failed to decode base64 data: %w", err) +func (fs *FileService) SaveFile_Meta() tsgenmeta.MethodMeta { + return tsgenmeta.MethodMeta{ + Desc: "save file", + ArgNames: []string{"connection", "path", "data64"}, } - err = os.WriteFile(cleanedPath, data, 0644) - if err != nil { - return fmt.Errorf("failed to write file %q: %w", path, err) +} + +func (fs *FileService) SaveFile(connection string, path string, data64 string) error { + if connection == "" { + connection = wshrpc.LocalConnName } - return nil + connRoute := wshutil.MakeConnectionRouteId(connection) + client := wshserver.GetMainRpcClient() + writeData := wshrpc.CommandRemoteWriteFileData{Path: path, Data64: data64} + return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: connRoute}) } func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta { @@ -60,78 +61,70 @@ func (fs *FileService) StatFile(connection string, path string) (*wshrpc.FileInf return wshclient.RemoteFileInfoCommand(client, path, &wshrpc.RpcOpts{Route: connRoute}) } -func (fs *FileService) ReadFile(path string) (*FullFile, error) { - finfo, err := fs.StatFile("", path) - if err != nil { - return nil, fmt.Errorf("cannot stat file %q: %w", path, err) +func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { + return tsgenmeta.MethodMeta{ + Desc: "read file", + ArgNames: []string{"connection", "path"}, } - if finfo.NotFound { - return &FullFile{Info: finfo}, nil +} + +func (fs *FileService) ReadFile(connection string, path string) (*FullFile, error) { + if connection == "" { + connection = wshrpc.LocalConnName } - if finfo.Size > MaxFileSize { - return nil, fmt.Errorf("file %q is too large to read, use /wave/stream-file", path) - } - if finfo.IsDir { - innerFilesEntries, err := os.ReadDir(finfo.Path) - if err != nil { - return nil, fmt.Errorf("unable to parse directory %s", finfo.Path) + connRoute := wshutil.MakeConnectionRouteId(connection) + client := wshserver.GetMainRpcClient() + streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: connRoute}) + fullFile := &FullFile{} + firstPk := true + isDir := false + var fileBuf bytes.Buffer + var fileInfoArr []*wshrpc.FileInfo + for respUnion := range rtnCh { + if respUnion.Error != nil { + return nil, respUnion.Error } - if len(innerFilesEntries) > 1000 { - innerFilesEntries = innerFilesEntries[:1000] + resp := respUnion.Response + if firstPk { + firstPk = false + // first packet has the fileinfo + if len(resp.FileInfo) != 1 { + return nil, fmt.Errorf("stream file protocol error, first pk fileinfo len=%d", len(resp.FileInfo)) + } + fullFile.Info = resp.FileInfo[0] + if fullFile.Info.IsDir { + isDir = true + } + continue } - var innerFilesInfo []wshrpc.FileInfo - parent := filepath.Dir(finfo.Path) - parentFileInfo, err := fs.StatFile("", parent) - if err == nil && parent != finfo.Path { - log.Printf("adding parent") - parentFileInfo.Name = ".." - parentFileInfo.Size = -1 - innerFilesInfo = append(innerFilesInfo, *parentFileInfo) - } - for _, innerFileEntry := range innerFilesEntries { - innerFileInfoInt, err := innerFileEntry.Info() - if err != nil { - log.Printf("unable to get file info for (innerFileInfo) %s: %v", innerFileEntry.Name(), err) + if isDir { + if len(resp.FileInfo) == 0 { continue } - mimeType := utilfn.DetectMimeType(filepath.Join(finfo.Path, innerFileInfoInt.Name())) - var fileSize int64 - if mimeType == "directory" { - fileSize = -1 - } else { - fileSize = innerFileInfoInt.Size() + fileInfoArr = append(fileInfoArr, resp.FileInfo...) + } else { + if resp.Data64 == "" { + continue } - innerFileInfo := wshrpc.FileInfo{ - Path: filepath.Join(finfo.Path, innerFileInfoInt.Name()), - Name: innerFileInfoInt.Name(), - Size: fileSize, - Mode: innerFileInfoInt.Mode(), - ModeStr: innerFileInfoInt.Mode().String(), - ModTime: innerFileInfoInt.ModTime().UnixMilli(), - IsDir: innerFileInfoInt.IsDir(), - MimeType: mimeType, + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) + _, err := io.Copy(&fileBuf, decoder) + if err != nil { + return nil, fmt.Errorf("stream file, failed to decode base64 data %q: %w", resp.Data64, err) } - innerFilesInfo = append(innerFilesInfo, innerFileInfo) } - - filesSerialized, err := json.Marshal(innerFilesInfo) + } + if isDir { + fiBytes, err := json.Marshal(fileInfoArr) if err != nil { - return nil, fmt.Errorf("unable to serialize files %s", finfo.Path) + return nil, fmt.Errorf("unable to serialize files %s", path) } - return &FullFile{ - Info: finfo, - Data64: base64.StdEncoding.EncodeToString(filesSerialized), - }, nil + fullFile.Data64 = base64.StdEncoding.EncodeToString(fiBytes) + } else { + // we can avoid this re-encoding if we ensure the remote side always encodes chunks of 3 bytes so we don't get padding chars + fullFile.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) } - cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path)) - barr, err := os.ReadFile(cleanedPath) - if err != nil { - return nil, fmt.Errorf("cannot read file %q: %w", path, err) - } - return &FullFile{ - Info: finfo, - Data64: base64.StdEncoding.EncodeToString(barr), - }, nil + return fullFile, nil } func (fs *FileService) GetWaveFile(id string, path string) (any, error) { @@ -144,9 +137,20 @@ func (fs *FileService) GetWaveFile(id string, path string) (any, error) { return file, nil } -func (fs *FileService) DeleteFile(path string) error { - cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path)) - return os.Remove(cleanedPath) +func (fs *FileService) DeleteFile_Meta() tsgenmeta.MethodMeta { + return tsgenmeta.MethodMeta{ + Desc: "delete file", + ArgNames: []string{"connection", "path"}, + } +} + +func (fs *FileService) DeleteFile(connection string, path string) error { + if connection == "" { + connection = wshrpc.LocalConnName + } + connRoute := wshutil.MakeConnectionRouteId(connection) + client := wshserver.GetMainRpcClient() + return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: connRoute}) } func (fs *FileService) GetSettingsConfig() wconfig.SettingsConfigType { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index d01f02908..1d28245c3 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -113,6 +113,12 @@ func MessageCommand(w *wshutil.WshRpc, data wshrpc.CommandMessageData, opts *wsh return err } +// command "remotefiledelete", wshserver.RemoteFileDeleteCommand +func RemoteFileDeleteCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) error { + _, err := sendRpcRequestCallHelper[any](w, "remotefiledelete", data, opts) + return err +} + // command "remotefileinfo", wshserver.RemoteFileInfoCommand func RemoteFileInfoCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) (*wshrpc.FileInfo, error) { resp, err := sendRpcRequestCallHelper[*wshrpc.FileInfo](w, "remotefileinfo", data, opts) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 241722929..1e4dc68d1 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -19,6 +19,9 @@ import ( ) const MaxFileSize = 50 * 1024 * 1024 // 10M +const MaxDirSize = 1024 +const FileChunkSize = 16 * 1024 +const DirChunkSize = 128 type ServerImpl struct { LogWriter io.Writer @@ -64,14 +67,14 @@ func parseByteRange(rangeStr string) (ByteRangeType, error) { return ByteRangeType{Start: start, End: end}, nil } -func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo *wshrpc.FileInfo, data []byte)) error { +func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error { innerFilesEntries, err := os.ReadDir(path) if err != nil { return fmt.Errorf("cannot open dir %q: %w", path, err) } if byteRange.All { - if len(innerFilesEntries) > 1000 { - innerFilesEntries = innerFilesEntries[:1000] + if len(innerFilesEntries) > MaxDirSize { + innerFilesEntries = innerFilesEntries[:MaxDirSize] } } else { if byteRange.Start >= int64(len(innerFilesEntries)) { @@ -83,12 +86,13 @@ func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, by } innerFilesEntries = innerFilesEntries[byteRange.Start:realEnd] } + var fileInfoArr []*wshrpc.FileInfo parent := filepath.Dir(path) parentFileInfo, err := impl.RemoteFileInfoCommand(ctx, parent) if err == nil && parent != path { parentFileInfo.Name = ".." parentFileInfo.Size = -1 - dataCallback(parentFileInfo, nil) + fileInfoArr = append(fileInfoArr, parentFileInfo) } for _, innerFileEntry := range innerFilesEntries { if ctx.Err() != nil { @@ -115,12 +119,20 @@ func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, by IsDir: innerFileInfoInt.IsDir(), MimeType: mimeType, } - dataCallback(&innerFileInfo, nil) + fileInfoArr = append(fileInfoArr, &innerFileInfo) + if len(fileInfoArr) >= DirChunkSize { + dataCallback(fileInfoArr, nil) + fileInfoArr = nil + } + } + if len(fileInfoArr) > 0 { + dataCallback(fileInfoArr, nil) } return nil } -func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo *wshrpc.FileInfo, data []byte)) error { +// TODO make sure the read is in chunks of 3 bytes (so 4 bytes of base64) in order to make decoding more efficient +func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error { fd, err := os.Open(path) if err != nil { return fmt.Errorf("cannot open file %q: %w", path, err) @@ -134,7 +146,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string } filePos = byteRange.Start } - buf := make([]byte, 4096) + buf := make([]byte, FileChunkSize) for { if ctx.Err() != nil { return ctx.Err() @@ -160,7 +172,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string return nil } -func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo *wshrpc.FileInfo, data []byte)) error { +func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error { byteRange, err := parseByteRange(data.ByteRange) if err != nil { return err @@ -171,7 +183,7 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp if err != nil { return fmt.Errorf("cannot stat file %q: %w", path, err) } - dataCallback(finfo, nil) + dataCallback([]*wshrpc.FileInfo{finfo}, nil) if finfo.NotFound { return nil } @@ -188,11 +200,11 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc.CommandRemoteStreamFileData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData] { ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData], 16) defer close(ch) - err := impl.remoteStreamFileInternal(ctx, data, func(fileInfo *wshrpc.FileInfo, data []byte) { + err := impl.remoteStreamFileInternal(ctx, data, func(fileInfo []*wshrpc.FileInfo, data []byte) { resp := wshrpc.CommandRemoteStreamFileRtnData{} resp.FileInfo = fileInfo if len(data) > 0 { - resp.Data64 = base64.RawStdEncoding.EncodeToString(data) + resp.Data64 = base64.StdEncoding.EncodeToString(data) } ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData]{Response: resp} }) @@ -242,3 +254,12 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.Comma } return nil } + +func (*ServerImpl) RemoteFileDeleteCommand(ctx context.Context, path string) error { + cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path)) + err := os.Remove(cleanedPath) + if err != nil { + return fmt.Errorf("cannot delete file %q: %w", path, err) + } + return nil +} diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index e590b4ad2..d647d0868 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -52,6 +52,7 @@ const ( Command_RemoteStreamFile = "remotestreamfile" Command_RemoteFileInfo = "remotefileinfo" Command_RemoteWriteFile = "remotewritefile" + Command_RemoteFileDelete = "remotefiledelete" Command_Event = "event" ) @@ -90,6 +91,7 @@ type WshRpcInterface interface { // remotes RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[CommandRemoteStreamFileRtnData] RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) + RemoteFileDeleteCommand(ctx context.Context, path string) error RemoteWriteFileCommand(ctx context.Context, data CommandRemoteWriteFileData) error } @@ -281,8 +283,8 @@ type CommandRemoteStreamFileData struct { } type CommandRemoteStreamFileRtnData struct { - FileInfo *FileInfo `json:"fileinfo,omitempty"` - Data64 string `json:"data64,omitempty"` + FileInfo []*FileInfo `json:"fileinfo,omitempty"` + Data64 string `json:"data64,omitempty"` } type CommandRemoteWriteFileData struct {