Upstream/waveterm
1
0
Fork 0
mirror of https://github.com/wavetermdev/waveterm.git synced 2025-01-02 18:39:05 +01:00
Code Issues Packages Projects Releases Wiki Activity

get remote preview working (#246)

Browse Source 
almost all there -- just need to fix streamfile for web urls.
This commit is contained in:
Mike Sawka 2024-08-19 11:02:40 -07:00 committed by GitHub
parent 85874f92ca
commit 8651659c02
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 155 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
frontend/app/store/services.ts
View File

@ -56,7 +56,9 @@ class FileServiceType {
AddWidget(arg1: WidgetsConfigType): Promise<void> {
return WOS.callBackendService("file", "AddWidget", Array.from(arguments))
}
DeleteFile(arg1: string): Promise<void> {
// delete file
DeleteFile(connection: string, path: string): Promise<void> {
return WOS.callBackendService("file", "DeleteFile", Array.from(arguments))
}
GetSettingsConfig(): Promise<SettingsConfigType> {
@ -65,13 +67,17 @@ class FileServiceType {
GetWaveFile(arg1: string, arg2: string): Promise<any> {
return WOS.callBackendService("file", "GetWaveFile", Array.from(arguments))
}
ReadFile(arg1: string): Promise<FullFile> {
// read file
ReadFile(connection: string, path: string): Promise<FullFile> {
return WOS.callBackendService("file", "ReadFile", Array.from(arguments))
}
RemoveWidget(arg1: number): Promise<void> {
return WOS.callBackendService("file", "RemoveWidget", Array.from(arguments))
}
SaveFile(arg1: string, arg2: string): Promise<void> {
// save file
SaveFile(connection: string, path: string, data64: string): Promise<void> {
return WOS.callBackendService("file", "SaveFile", Array.from(arguments))
}

5
frontend/app/store/wshserver.ts
View File

@ -92,6 +92,11 @@ class WshServerType {
return WOS.wshServerRpcHelper_call("message", data, opts);
}
// command "remotefiledelete" [call]
RemoteFileDeleteCommand(data: string, opts?: RpcOpts): Promise<void> {
return WOS.wshServerRpcHelper_call("remotefiledelete", data, opts);
}
// command "remotefileinfo" [call]
RemoteFileInfoCommand(data: string, opts?: RpcOpts): Promise<FileInfo> {
return WOS.wshServerRpcHelper_call("remotefileinfo", data, opts);

17
frontend/app/view/preview/directorypreview.tsx
View File

@ -29,6 +29,7 @@ import { OverlayScrollbars } from "overlayscrollbars";
import "./directorypreview.less";
interface DirectoryTableProps {
model: PreviewModel;
data: FileInfo[];
search: string;
focusIndex: number;
@ -124,6 +125,7 @@ function cleanMimetype(input: string): string {
}
function DirectoryTable({
model,
data,
search,
focusIndex,
@ -294,6 +296,7 @@ function DirectoryTable({
</div>
{table.getState().columnSizingInfo.isResizingColumn ? (
<MemoizedTableBody
model={model}
data={data}
table={table}
search={search}
@ -306,6 +309,7 @@ function DirectoryTable({
/>
) : (
<TableBody
model={model}
data={data}
table={table}
search={search}
@ -322,6 +326,7 @@ function DirectoryTable({
}
interface TableBodyProps {
model: PreviewModel;
data: Array<FileInfo>;
table: Table<FileInfo>;
search: string;
@ -334,6 +339,7 @@ interface TableBodyProps {
}
function TableBody({
model,
data,
table,
search,
@ -353,6 +359,7 @@ function TableBody({
const rowRefs = useRef<HTMLDivElement[]>([]);
const parentHeight = useHeight(parentRef);
const conn = jotai.useAtomValue(model.connection);
useEffect(() => {
if (dummyLineRef.current && data && parentRef.current) {
@ -454,13 +461,13 @@ function TableBody({
menu.push({
label: "Delete File",
click: async () => {
await services.FileService.DeleteFile(path).catch((e) => console.log(e));
await services.FileService.DeleteFile(conn, path).catch((e) => console.log(e));
setRefreshVersion((current) => current + 1);
},
});
ContextMenuModel.showContextMenu(menu, e);
},
[setRefreshVersion]
[setRefreshVersion, conn]
);
const displayRow = useCallback(
@ -541,6 +548,7 @@ function DirectoryPreview({ fileNameAtom, model }: DirectoryPreviewProps) {
const showHiddenFiles = jotai.useAtomValue(model.showHiddenFiles);
const [selectedPath, setSelectedPath] = useState("");
const [refreshVersion, setRefreshVersion] = jotai.useAtom(model.refreshVersion);
const conn = jotai.useAtomValue(model.connection);
useEffect(() => {
model.refreshCallback = () => {
@ -553,13 +561,13 @@ function DirectoryPreview({ fileNameAtom, model }: DirectoryPreviewProps) {
useEffect(() => {
const getContent = async () => {
const file = await services.FileService.ReadFile(fileName);
const file = await services.FileService.ReadFile(conn, fileName);
const serializedContent = util.base64ToString(file?.data64);
const content: FileInfo[] = JSON.parse(serializedContent);
setUnfilteredData(content);
};
getContent();
}, [fileName, refreshVersion]);
}, [conn, fileName, refreshVersion]);
useEffect(() => {
const filtered = unfilteredData.filter((fileInfo) => {
@ -633,6 +641,7 @@ function DirectoryPreview({ fileNameAtom, model }: DirectoryPreviewProps) {
/>
</div>
<DirectoryTable
model={model}
data={filteredData}
search={searchText}
focusIndex={focusIndex}

9
frontend/app/view/preview/preview.tsx
View File

@ -205,8 +205,6 @@ export class PreviewModel implements ViewModel {
return null;
}
const conn = get(this.connection) ?? "";
// const statFile = await FileService.StatFile(fileName);
console.log("PreviewModel calling StatFile", conn, fileName);
const statFile = await services.FileService.StatFile(conn, fileName);
return statFile;
});
@ -215,8 +213,8 @@ export class PreviewModel implements ViewModel {
if (fileName == null) {
return null;
}
// const file = await FileService.ReadFile(fileName);
const file = await services.FileService.ReadFile(fileName);
const conn = get(this.connection) ?? "";
const file = await services.FileService.ReadFile(conn, fileName);
return file;
});
this.fileMimeType = jotai.atom<Promise<string>>(async (get) => {
@ -253,8 +251,9 @@ export class PreviewModel implements ViewModel {
async handleFileSave() {
const fileName = globalStore.get(this.fileName);
const newFileContent = globalStore.get(this.newFileContent);
const conn = globalStore.get(this.connection) ?? "";
try {
services.FileService.SaveFile(fileName, util.stringToBase64(newFileContent));
services.FileService.SaveFile(conn, fileName, util.stringToBase64(newFileContent));
globalStore.set(this.newFileContent, null);
} catch (error) {
console.error("Error saving file:", error);

2
frontend/types/gotypes.d.ts vendored
View File

@ -124,7 +124,7 @@ declare global {
// wshrpc.CommandRemoteStreamFileRtnData
type CommandRemoteStreamFileRtnData = {
fileinfo?: FileInfo;
fileinfo?: FileInfo[];
data64?: string;
};

154
pkg/service/fileservice/fileservice.go
View File

@ -1,19 +1,16 @@
package fileservice
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"io"
"time"
"github.com/wavetermdev/thenextwave/pkg/filestore"
"github.com/wavetermdev/thenextwave/pkg/tsgen/tsgenmeta"
"github.com/wavetermdev/thenextwave/pkg/util/utilfn"
"github.com/wavetermdev/thenextwave/pkg/wavebase"
"github.com/wavetermdev/thenextwave/pkg/wconfig"
"github.com/wavetermdev/thenextwave/pkg/wshrpc"
"github.com/wavetermdev/thenextwave/pkg/wshrpc/wshclient"
@ -31,17 +28,21 @@ type FullFile struct {
Data64 string `json:"data64"` // base64 encoded
}
func (fs *FileService) SaveFile(path string, data64 string) error {
cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path))
data, err := base64.StdEncoding.DecodeString(data64)
if err != nil {
return fmt.Errorf("failed to decode base64 data: %w", err)
func (fs *FileService) SaveFile_Meta() tsgenmeta.MethodMeta {
return tsgenmeta.MethodMeta{
Desc: "save file",
ArgNames: []string{"connection", "path", "data64"},
}
err = os.WriteFile(cleanedPath, data, 0644)
if err != nil {
return fmt.Errorf("failed to write file %q: %w", path, err)
}
func (fs *FileService) SaveFile(connection string, path string, data64 string) error {
if connection == "" {
connection = wshrpc.LocalConnName
}
return nil
connRoute := wshutil.MakeConnectionRouteId(connection)
client := wshserver.GetMainRpcClient()
writeData := wshrpc.CommandRemoteWriteFileData{Path: path, Data64: data64}
return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: connRoute})
}
func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta {
@ -60,78 +61,70 @@ func (fs *FileService) StatFile(connection string, path string) (*wshrpc.FileInf
return wshclient.RemoteFileInfoCommand(client, path, &wshrpc.RpcOpts{Route: connRoute})
}
func (fs *FileService) ReadFile(path string) (*FullFile, error) {
finfo, err := fs.StatFile("", path)
if err != nil {
return nil, fmt.Errorf("cannot stat file %q: %w", path, err)
func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta {
return tsgenmeta.MethodMeta{
Desc: "read file",
ArgNames: []string{"connection", "path"},
}
if finfo.NotFound {
return &FullFile{Info: finfo}, nil
}
func (fs *FileService) ReadFile(connection string, path string) (*FullFile, error) {
if connection == "" {
connection = wshrpc.LocalConnName
}
if finfo.Size > MaxFileSize {
return nil, fmt.Errorf("file %q is too large to read, use /wave/stream-file", path)
connRoute := wshutil.MakeConnectionRouteId(connection)
client := wshserver.GetMainRpcClient()
streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path}
rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: connRoute})
fullFile := &FullFile{}
firstPk := true
isDir := false
var fileBuf bytes.Buffer
var fileInfoArr []*wshrpc.FileInfo
for respUnion := range rtnCh {
if respUnion.Error != nil {
return nil, respUnion.Error
}
if finfo.IsDir {
innerFilesEntries, err := os.ReadDir(finfo.Path)
if err != nil {
return nil, fmt.Errorf("unable to parse directory %s", finfo.Path)
resp := respUnion.Response
if firstPk {
firstPk = false
// first packet has the fileinfo
if len(resp.FileInfo) != 1 {
return nil, fmt.Errorf("stream file protocol error, first pk fileinfo len=%d", len(resp.FileInfo))
}
if len(innerFilesEntries) > 1000 {
innerFilesEntries = innerFilesEntries[:1000]
fullFile.Info = resp.FileInfo[0]
if fullFile.Info.IsDir {
isDir = true
}
var innerFilesInfo []wshrpc.FileInfo
parent := filepath.Dir(finfo.Path)
parentFileInfo, err := fs.StatFile("", parent)
if err == nil && parent != finfo.Path {
log.Printf("adding parent")
parentFileInfo.Name = ".."
parentFileInfo.Size = -1
innerFilesInfo = append(innerFilesInfo, *parentFileInfo)
}
for _, innerFileEntry := range innerFilesEntries {
innerFileInfoInt, err := innerFileEntry.Info()
if err != nil {
log.Printf("unable to get file info for (innerFileInfo) %s: %v", innerFileEntry.Name(), err)
continue
}
mimeType := utilfn.DetectMimeType(filepath.Join(finfo.Path, innerFileInfoInt.Name()))
var fileSize int64
if mimeType == "directory" {
fileSize = -1
if isDir {
if len(resp.FileInfo) == 0 {
continue
}
fileInfoArr = append(fileInfoArr, resp.FileInfo...)
} else {
fileSize = innerFileInfoInt.Size()
if resp.Data64 == "" {
continue
}
innerFileInfo := wshrpc.FileInfo{
Path: filepath.Join(finfo.Path, innerFileInfoInt.Name()),
Name: innerFileInfoInt.Name(),
Size: fileSize,
Mode: innerFileInfoInt.Mode(),
ModeStr: innerFileInfoInt.Mode().String(),
ModTime: innerFileInfoInt.ModTime().UnixMilli(),
IsDir: innerFileInfoInt.IsDir(),
MimeType: mimeType,
}
innerFilesInfo = append(innerFilesInfo, innerFileInfo)
}
filesSerialized, err := json.Marshal(innerFilesInfo)
decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64)))
_, err := io.Copy(&fileBuf, decoder)
if err != nil {
return nil, fmt.Errorf("unable to serialize files %s", finfo.Path)
return nil, fmt.Errorf("stream file, failed to decode base64 data %q: %w", resp.Data64, err)
}
return &FullFile{
Info: finfo,
Data64: base64.StdEncoding.EncodeToString(filesSerialized),
}, nil
}
cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path))
barr, err := os.ReadFile(cleanedPath)
}
if isDir {
fiBytes, err := json.Marshal(fileInfoArr)
if err != nil {
return nil, fmt.Errorf("cannot read file %q: %w", path, err)
return nil, fmt.Errorf("unable to serialize files %s", path)
}
return &FullFile{
Info: finfo,
Data64: base64.StdEncoding.EncodeToString(barr),
}, nil
fullFile.Data64 = base64.StdEncoding.EncodeToString(fiBytes)
} else {
// we can avoid this re-encoding if we ensure the remote side always encodes chunks of 3 bytes so we don't get padding chars
fullFile.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes())
}
return fullFile, nil
}
func (fs *FileService) GetWaveFile(id string, path string) (any, error) {
@ -144,9 +137,20 @@ func (fs *FileService) GetWaveFile(id string, path string) (any, error) {
return file, nil
}
func (fs *FileService) DeleteFile(path string) error {
cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path))
return os.Remove(cleanedPath)
func (fs *FileService) DeleteFile_Meta() tsgenmeta.MethodMeta {
return tsgenmeta.MethodMeta{
Desc: "delete file",
ArgNames: []string{"connection", "path"},
}
}
func (fs *FileService) DeleteFile(connection string, path string) error {
if connection == "" {
connection = wshrpc.LocalConnName
}
connRoute := wshutil.MakeConnectionRouteId(connection)
client := wshserver.GetMainRpcClient()
return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: connRoute})
}
func (fs *FileService) GetSettingsConfig() wconfig.SettingsConfigType {

6
pkg/wshrpc/wshclient/wshclient.go
View File

@ -113,6 +113,12 @@ func MessageCommand(w *wshutil.WshRpc, data wshrpc.CommandMessageData, opts *wsh
return err
}
// command "remotefiledelete", wshserver.RemoteFileDeleteCommand
func RemoteFileDeleteCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) error {
_, err := sendRpcRequestCallHelper[any](w, "remotefiledelete", data, opts)
return err
}
// command "remotefileinfo", wshserver.RemoteFileInfoCommand
func RemoteFileInfoCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) (*wshrpc.FileInfo, error) {
resp, err := sendRpcRequestCallHelper[*wshrpc.FileInfo](w, "remotefileinfo", data, opts)

43
pkg/wshrpc/wshremote/wshremote.go
View File

@ -19,6 +19,9 @@ import (
)
const MaxFileSize = 50 * 1024 * 1024 // 10M
const MaxDirSize = 1024
const FileChunkSize = 16 * 1024
const DirChunkSize = 128
type ServerImpl struct {
LogWriter io.Writer
@ -64,14 +67,14 @@ func parseByteRange(rangeStr string) (ByteRangeType, error) {
return ByteRangeType{Start: start, End: end}, nil
}
func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo *wshrpc.FileInfo, data []byte)) error {
func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error {
innerFilesEntries, err := os.ReadDir(path)
if err != nil {
return fmt.Errorf("cannot open dir %q: %w", path, err)
}
if byteRange.All {
if len(innerFilesEntries) > 1000 {
innerFilesEntries = innerFilesEntries[:1000]
if len(innerFilesEntries) > MaxDirSize {
innerFilesEntries = innerFilesEntries[:MaxDirSize]
}
} else {
if byteRange.Start >= int64(len(innerFilesEntries)) {
@ -83,12 +86,13 @@ func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, by
}
innerFilesEntries = innerFilesEntries[byteRange.Start:realEnd]
}
var fileInfoArr []*wshrpc.FileInfo
parent := filepath.Dir(path)
parentFileInfo, err := impl.RemoteFileInfoCommand(ctx, parent)
if err == nil && parent != path {
parentFileInfo.Name = ".."
parentFileInfo.Size = -1
dataCallback(parentFileInfo, nil)
fileInfoArr = append(fileInfoArr, parentFileInfo)
}
for _, innerFileEntry := range innerFilesEntries {
if ctx.Err() != nil {
@ -115,12 +119,20 @@ func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, by
IsDir: innerFileInfoInt.IsDir(),
MimeType: mimeType,
}
dataCallback(&innerFileInfo, nil)
fileInfoArr = append(fileInfoArr, &innerFileInfo)
if len(fileInfoArr) >= DirChunkSize {
dataCallback(fileInfoArr, nil)
fileInfoArr = nil
}
}
if len(fileInfoArr) > 0 {
dataCallback(fileInfoArr, nil)
}
return nil
}
func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo *wshrpc.FileInfo, data []byte)) error {
// TODO make sure the read is in chunks of 3 bytes (so 4 bytes of base64) in order to make decoding more efficient
func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error {
fd, err := os.Open(path)
if err != nil {
return fmt.Errorf("cannot open file %q: %w", path, err)
@ -134,7 +146,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string
}
filePos = byteRange.Start
}
buf := make([]byte, 4096)
buf := make([]byte, FileChunkSize)
for {
if ctx.Err() != nil {
return ctx.Err()
@ -160,7 +172,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string
return nil
}
func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo *wshrpc.FileInfo, data []byte)) error {
func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error {
byteRange, err := parseByteRange(data.ByteRange)
if err != nil {
return err
@ -171,7 +183,7 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp
if err != nil {
return fmt.Errorf("cannot stat file %q: %w", path, err)
}
dataCallback(finfo, nil)
dataCallback([]*wshrpc.FileInfo{finfo}, nil)
if finfo.NotFound {
return nil
}
@ -188,11 +200,11 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp
func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc.CommandRemoteStreamFileData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData] {
ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData], 16)
defer close(ch)
err := impl.remoteStreamFileInternal(ctx, data, func(fileInfo *wshrpc.FileInfo, data []byte) {
err := impl.remoteStreamFileInternal(ctx, data, func(fileInfo []*wshrpc.FileInfo, data []byte) {
resp := wshrpc.CommandRemoteStreamFileRtnData{}
resp.FileInfo = fileInfo
if len(data) > 0 {
resp.Data64 = base64.RawStdEncoding.EncodeToString(data)
resp.Data64 = base64.StdEncoding.EncodeToString(data)
}
ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData]{Response: resp}
})
@ -242,3 +254,12 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.Comma
}
return nil
}
func (*ServerImpl) RemoteFileDeleteCommand(ctx context.Context, path string) error {
cleanedPath := filepath.Clean(wavebase.ExpandHomeDir(path))
err := os.Remove(cleanedPath)
if err != nil {
return fmt.Errorf("cannot delete file %q: %w", path, err)
}
return nil
}

4
pkg/wshrpc/wshrpctypes.go
View File

@ -52,6 +52,7 @@ const (
Command_RemoteStreamFile = "remotestreamfile"
Command_RemoteFileInfo = "remotefileinfo"
Command_RemoteWriteFile = "remotewritefile"
Command_RemoteFileDelete = "remotefiledelete"
Command_Event = "event"
)
@ -90,6 +91,7 @@ type WshRpcInterface interface {
// remotes
RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[CommandRemoteStreamFileRtnData]
RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error)
RemoteFileDeleteCommand(ctx context.Context, path string) error
RemoteWriteFileCommand(ctx context.Context, data CommandRemoteWriteFileData) error
}
@ -281,7 +283,7 @@ type CommandRemoteStreamFileData struct {
}
type CommandRemoteStreamFileRtnData struct {
FileInfo *FileInfo `json:"fileinfo,omitempty"`
FileInfo []*FileInfo `json:"fileinfo,omitempty"`
Data64 string `json:"data64,omitempty"`
}