From 4c219ee68bdd59e3b2e7cb93f9a39c89c9173ca5 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 3 Jan 2025 12:54:16 -0500 Subject: [PATCH 001/126] initial commit --- go.mod | 18 +++++++++++ go.sum | 36 +++++++++++++++++++++ pkg/remote/fileshare/fileshare.go | 14 ++++++++ pkg/remote/fileshare/s3bucket/s3bucket.go | 39 +++++++++++++++++++++++ 4 files changed, 107 insertions(+) create mode 100644 pkg/remote/fileshare/fileshare.go create mode 100644 pkg/remote/fileshare/s3bucket/s3bucket.go diff --git a/go.mod b/go.mod index d40722952..09003c272 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,24 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/longrunning v0.5.7 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.7 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.28.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.48 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect + github.com/aws/smithy-go v1.22.1 // indirect github.com/ebitengine/purego v0.8.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect diff --git a/go.sum b/go.sum index 9379a5a57..29e1d1779 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,42 @@ github.com/0xrawsec/golang-utils v1.3.2 h1:ww4jrtHRSnX9xrGzJYbalx5nXoZewy4zPxiY+ github.com/0xrawsec/golang-utils v1.3.2/go.mod h1:m7AzHXgdSAkFCD9tWWsApxNVxMlyy7anpPVOyT/yM7E= github.com/alexflint/go-filemutex v1.3.0 h1:LgE+nTUWnQCyRKbpoceKZsPQbs84LivvgwUymZXdOcM= github.com/alexflint/go-filemutex v1.3.0/go.mod h1:U0+VA/i30mGBlLCrFPGtTe9y6wGQfNAWPBTekHQ+c8A= +github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= +github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE= +github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= +github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= +github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go new file mode 100644 index 000000000..8b2b91adf --- /dev/null +++ b/pkg/remote/fileshare/fileshare.go @@ -0,0 +1,14 @@ +package fileshare + +type FileShare interface { + // GetFile returns the file at the given path + GetFile(path string) ([]byte, error) + // PutFile writes the given data to the file at the given path + PutFile(path string, data []byte) error + // DeleteFile deletes the file at the given path + DeleteFile(path string) error + // ListFiles returns a list of files in the given directory + ListFiles(path string) ([]string, error) + // GetFileShareName returns the name of the fileshare + GetFileShareName() string +} diff --git a/pkg/remote/fileshare/s3bucket/s3bucket.go b/pkg/remote/fileshare/s3bucket/s3bucket.go new file mode 100644 index 000000000..d7132da3e --- /dev/null +++ b/pkg/remote/fileshare/s3bucket/s3bucket.go @@ -0,0 +1,39 @@ +package s3bucket + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" +) + +type S3Bucket struct { + client *s3.Client +} + +var _ fileshare.FileShare = S3Bucket{} + +func NewS3Bucket(config aws.Config) *S3Bucket { + return &S3Bucket{ + client: s3.NewFromConfig(config), + } +} + +func (s S3Bucket) GetFile(path string) ([]byte, error) { + return nil, nil +} + +func (s S3Bucket) PutFile(path string, data []byte) error { + return nil +} + +func (s S3Bucket) DeleteFile(path string) error { + return nil +} + +func (s S3Bucket) ListFiles(path string) ([]string, error) { + return nil, nil +} + +func (s S3Bucket) GetFileShareName() string { + return "S3Bucket" +} From 1003187201558ac15b8a0d4aa969e5daa6a2415e Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 3 Jan 2025 13:32:09 -0500 Subject: [PATCH 002/126] save --- pkg/remote/fileshare/fileshare.go | 4 ++++ pkg/remote/fileshare/s3bucket/s3bucket.go | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 8b2b91adf..ed99dcbfa 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -3,8 +3,12 @@ package fileshare type FileShare interface { // GetFile returns the file at the given path GetFile(path string) ([]byte, error) + // StatFile returns the file info at the given path + StatFile(path string) (any, error) // PutFile writes the given data to the file at the given path PutFile(path string, data []byte) error + // MoveFile moves the file from srcPath to destPath + MoveFile(srcPath, destPath string) error // DeleteFile deletes the file at the given path DeleteFile(path string) error // ListFiles returns a list of files in the given directory diff --git a/pkg/remote/fileshare/s3bucket/s3bucket.go b/pkg/remote/fileshare/s3bucket/s3bucket.go index d7132da3e..cbea6607d 100644 --- a/pkg/remote/fileshare/s3bucket/s3bucket.go +++ b/pkg/remote/fileshare/s3bucket/s3bucket.go @@ -22,10 +22,18 @@ func (s S3Bucket) GetFile(path string) ([]byte, error) { return nil, nil } +func (s S3Bucket) StatFile(path string) (any, error) { + return nil, nil +} + func (s S3Bucket) PutFile(path string, data []byte) error { return nil } +func (s S3Bucket) MoveFile(srcPath, destPath string) error { + return nil +} + func (s S3Bucket) DeleteFile(path string) error { return nil } From 39aae45df648e7b21b465c2bf7c86a76cfefda97 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 6 Jan 2025 15:12:29 -0800 Subject: [PATCH 003/126] save --- pkg/remote/connresolver.go | 0 pkg/remote/fileshare/fileshare.go | 37 +++++-- pkg/remote/fileshare/s3bucket/s3bucket.go | 47 --------- pkg/remote/fileshare/s3fs/s3fs.go | 56 ++++++++++ pkg/remote/fileshare/wshfs/wshfs.go | 120 ++++++++++++++++++++++ 5 files changed, 202 insertions(+), 58 deletions(-) create mode 100644 pkg/remote/connresolver.go delete mode 100644 pkg/remote/fileshare/s3bucket/s3bucket.go create mode 100644 pkg/remote/fileshare/s3fs/s3fs.go create mode 100644 pkg/remote/fileshare/wshfs/wshfs.go diff --git a/pkg/remote/connresolver.go b/pkg/remote/connresolver.go new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index ed99dcbfa..01b7d262c 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -1,18 +1,33 @@ package fileshare +import "github.com/wavetermdev/waveterm/pkg/wshrpc" + +type FullFile struct { + Info *wshrpc.FileInfo `json:"info"` + Data64 string `json:"data64"` // base64 encoded +} + type FileShare interface { - // GetFile returns the file at the given path - GetFile(path string) ([]byte, error) - // StatFile returns the file info at the given path - StatFile(path string) (any, error) + // Stat returns the file info at the given path + Stat(path string) (*wshrpc.FileInfo, error) + // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo + Read(path string) (*FullFile, error) // PutFile writes the given data to the file at the given path - PutFile(path string, data []byte) error - // MoveFile moves the file from srcPath to destPath - MoveFile(srcPath, destPath string) error - // DeleteFile deletes the file at the given path - DeleteFile(path string) error - // ListFiles returns a list of files in the given directory - ListFiles(path string) ([]string, error) + PutFile(path string, data64 string) error + // Mkdir creates a directory at the given path + Mkdir(path string) error + // Move moves the file from srcPath to destPath + Move(srcPath, destPath string, recursive bool) error + // Copy copies the file from srcPath to destPath + Copy(srcPath, destPath string, recursive bool) error + // Delete deletes the entry at the given path + Delete(path string) error + // ListEntries returns a list of entries in the given directory + ListEntries(path string) ([]wshrpc.FileInfo, error) // GetFileShareName returns the name of the fileshare GetFileShareName() string } + +func CreateFileShare(connection string) FileShare { + return nil +} diff --git a/pkg/remote/fileshare/s3bucket/s3bucket.go b/pkg/remote/fileshare/s3bucket/s3bucket.go deleted file mode 100644 index cbea6607d..000000000 --- a/pkg/remote/fileshare/s3bucket/s3bucket.go +++ /dev/null @@ -1,47 +0,0 @@ -package s3bucket - -import ( - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare" -) - -type S3Bucket struct { - client *s3.Client -} - -var _ fileshare.FileShare = S3Bucket{} - -func NewS3Bucket(config aws.Config) *S3Bucket { - return &S3Bucket{ - client: s3.NewFromConfig(config), - } -} - -func (s S3Bucket) GetFile(path string) ([]byte, error) { - return nil, nil -} - -func (s S3Bucket) StatFile(path string) (any, error) { - return nil, nil -} - -func (s S3Bucket) PutFile(path string, data []byte) error { - return nil -} - -func (s S3Bucket) MoveFile(srcPath, destPath string) error { - return nil -} - -func (s S3Bucket) DeleteFile(path string) error { - return nil -} - -func (s S3Bucket) ListFiles(path string) ([]string, error) { - return nil, nil -} - -func (s S3Bucket) GetFileShareName() string { - return "S3Bucket" -} diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go new file mode 100644 index 000000000..099438f3e --- /dev/null +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -0,0 +1,56 @@ +package s3fs + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) + +type S3Client struct { + client *s3.Client +} + +var _ fileshare.FileShare = S3Client{} + +func NewClient(config aws.Config) *S3Client { + return &S3Client{ + client: s3.NewFromConfig(config), + } +} + +func (c S3Client) Read(path string) (*fileshare.FullFile, error) { + return nil, nil +} + +func (c S3Client) Stat(path string) (*wshrpc.FileInfo, error) { + return nil, nil +} + +func (c S3Client) PutFile(path string, data64 string) error { + return nil +} + +func (c S3Client) Mkdir(path string) error { + return nil +} + +func (c S3Client) Move(srcPath, destPath string, recursive bool) error { + return nil +} + +func (c S3Client) Copy(srcPath, destPath string, recursive bool) error { + return nil +} + +func (c S3Client) Delete(path string) error { + return nil +} + +func (c S3Client) ListEntries(path string) ([]wshrpc.FileInfo, error) { + return nil, nil +} + +func (c S3Client) GetFileShareName() string { + return "S3Client" +} diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go new file mode 100644 index 000000000..29e9fa366 --- /dev/null +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -0,0 +1,120 @@ +package wshfs + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" + "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" + "github.com/wavetermdev/waveterm/pkg/wshrpc/wshserver" + "github.com/wavetermdev/waveterm/pkg/wshutil" +) + +type WshClient struct { + connRoute string +} + +var _ fileshare.FileShare = WshClient{} + +func NewClient(connection string) *WshClient { + return &WshClient{ + connRoute: wshutil.MakeConnectionRouteId(connection), + } +} + +func (c WshClient) Read(path string) (*fileshare.FullFile, error) { + client := wshserver.GetMainRpcClient() + streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: c.connRoute}) + fullFile := &fileshare.FullFile{} + firstPk := true + isDir := false + var fileBuf bytes.Buffer + var fileInfoArr []*wshrpc.FileInfo + for respUnion := range rtnCh { + if respUnion.Error != nil { + return nil, respUnion.Error + } + resp := respUnion.Response + if firstPk { + firstPk = false + // first packet has the fileinfo + if len(resp.FileInfo) != 1 { + return nil, fmt.Errorf("stream file protocol error, first pk fileinfo len=%d", len(resp.FileInfo)) + } + fullFile.Info = resp.FileInfo[0] + if fullFile.Info.IsDir { + isDir = true + } + continue + } + if isDir { + if len(resp.FileInfo) == 0 { + continue + } + fileInfoArr = append(fileInfoArr, resp.FileInfo...) + } else { + if resp.Data64 == "" { + continue + } + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) + _, err := io.Copy(&fileBuf, decoder) + if err != nil { + return nil, fmt.Errorf("stream file, failed to decode base64 data %q: %w", resp.Data64, err) + } + } + } + if isDir { + fiBytes, err := json.Marshal(fileInfoArr) + if err != nil { + return nil, fmt.Errorf("unable to serialize files %s", path) + } + fullFile.Data64 = base64.StdEncoding.EncodeToString(fiBytes) + } else { + // we can avoid this re-encoding if we ensure the remote side always encodes chunks of 3 bytes so we don't get padding chars + fullFile.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) + } + return fullFile, nil +} + +func (c WshClient) Stat(path string) (*wshrpc.FileInfo, error) { + client := wshserver.GetMainRpcClient() + return wshclient.RemoteFileInfoCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) +} + +func (c WshClient) PutFile(path string, data64 string) error { + client := wshserver.GetMainRpcClient() + writeData := wshrpc.CommandRemoteWriteFileData{Path: path, Data64: data64} + return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: c.connRoute}) +} + +func (c WshClient) Mkdir(path string) error { + client := wshserver.GetMainRpcClient() + return wshclient.RemoteMkdirCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) +} + +func (c WshClient) Move(srcPath, destPath string, recursive bool) error { + client := wshserver.GetMainRpcClient() + return wshclient.RemoteFileRenameCommand(client, [2]string{srcPath, destPath}, &wshrpc.RpcOpts{Route: c.connRoute}) +} + +func (c WshClient) Copy(srcPath, destPath string, recursive bool) error { + return nil +} + +func (c WshClient) Delete(path string) error { + client := wshserver.GetMainRpcClient() + return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) +} + +func (c WshClient) ListEntries(path string) ([]wshrpc.FileInfo, error) { + return nil, nil +} + +func (c WshClient) GetFileShareName() string { + return "S3Client" +} From 4d20ebbd9d375d7fa6000bdd1a7c3043692fc0dd Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 6 Jan 2025 16:34:07 -0800 Subject: [PATCH 004/126] use fsclient in fileservice --- frontend/types/gotypes.d.ts | 2 +- go.mod | 11 +-- go.sum | 14 --- pkg/remote/connresolver.go | 0 pkg/remote/fileshare/fileshare.go | 6 +- pkg/remote/fileshare/{s3fs => }/s3fs.go | 9 +- pkg/remote/fileshare/{wshfs => }/wshfs.go | 11 ++- pkg/service/fileservice/fileservice.go | 101 ++++------------------ 8 files changed, 31 insertions(+), 123 deletions(-) delete mode 100644 pkg/remote/connresolver.go rename pkg/remote/fileshare/{s3fs => }/s3fs.go (79%) rename pkg/remote/fileshare/{wshfs => }/wshfs.go (92%) diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 682275134..546c7043b 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -388,7 +388,7 @@ declare global { configerrors: ConfigError[]; }; - // fileservice.FullFile + // fileshare.FullFile type FullFile = { info: FileInfo; data64: string; diff --git a/go.mod b/go.mod index 70781a0c3..19a2836b6 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.23.4 require ( github.com/alexflint/go-filemutex v1.3.0 + github.com/aws/aws-sdk-go-v2 v1.32.7 + github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 github.com/creack/pty v1.1.21 github.com/fsnotify/fsnotify v1.8.0 github.com/golang-jwt/jwt/v5 v5.2.1 @@ -39,23 +41,14 @@ require ( cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/longrunning v0.5.7 // indirect - github.com/aws/aws-sdk-go-v2 v1.32.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/config v1.28.7 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.48 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/ebitengine/purego v0.8.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect diff --git a/go.sum b/go.sum index 7889c8cc6..42f8c25a3 100644 --- a/go.sum +++ b/go.sum @@ -20,18 +20,10 @@ github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= -github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE= -github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M= -github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= @@ -44,12 +36,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEH github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/pkg/remote/connresolver.go b/pkg/remote/connresolver.go deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 01b7d262c..f776f32f5 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -7,7 +7,7 @@ type FullFile struct { Data64 string `json:"data64"` // base64 encoded } -type FileShare interface { +type FileShareClient interface { // Stat returns the file info at the given path Stat(path string) (*wshrpc.FileInfo, error) // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo @@ -28,6 +28,6 @@ type FileShare interface { GetFileShareName() string } -func CreateFileShare(connection string) FileShare { - return nil +func CreateFileShareClient(connection string) FileShareClient { + return NewWshClient(connection) } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs.go similarity index 79% rename from pkg/remote/fileshare/s3fs/s3fs.go rename to pkg/remote/fileshare/s3fs.go index 099438f3e..9998b637a 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs.go @@ -1,9 +1,8 @@ -package s3fs +package fileshare import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -11,15 +10,15 @@ type S3Client struct { client *s3.Client } -var _ fileshare.FileShare = S3Client{} +var _ FileShareClient = S3Client{} -func NewClient(config aws.Config) *S3Client { +func NewS3Client(config aws.Config) *S3Client { return &S3Client{ client: s3.NewFromConfig(config), } } -func (c S3Client) Read(path string) (*fileshare.FullFile, error) { +func (c S3Client) Read(path string) (*FullFile, error) { return nil, nil } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs.go similarity index 92% rename from pkg/remote/fileshare/wshfs/wshfs.go rename to pkg/remote/fileshare/wshfs.go index 29e9fa366..7af3776e5 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs.go @@ -1,4 +1,4 @@ -package wshfs +package fileshare import ( "bytes" @@ -7,7 +7,6 @@ import ( "fmt" "io" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshserver" @@ -18,19 +17,19 @@ type WshClient struct { connRoute string } -var _ fileshare.FileShare = WshClient{} +var _ FileShareClient = WshClient{} -func NewClient(connection string) *WshClient { +func NewWshClient(connection string) *WshClient { return &WshClient{ connRoute: wshutil.MakeConnectionRouteId(connection), } } -func (c WshClient) Read(path string) (*fileshare.FullFile, error) { +func (c WshClient) Read(path string) (*FullFile, error) { client := wshserver.GetMainRpcClient() streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: c.connRoute}) - fullFile := &fileshare.FullFile{} + fullFile := &FullFile{} firstPk := true isDir := false var fileBuf bytes.Buffer diff --git a/pkg/service/fileservice/fileservice.go b/pkg/service/fileservice/fileservice.go index 68490a2f4..751714edd 100644 --- a/pkg/service/fileservice/fileservice.go +++ b/pkg/service/fileservice/fileservice.go @@ -1,21 +1,15 @@ package fileservice import ( - "bytes" "context" - "encoding/base64" - "encoding/json" "fmt" - "io" "time" "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta" "github.com/wavetermdev/waveterm/pkg/wconfig" "github.com/wavetermdev/waveterm/pkg/wshrpc" - "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" - "github.com/wavetermdev/waveterm/pkg/wshrpc/wshserver" - "github.com/wavetermdev/waveterm/pkg/wshutil" ) const MaxFileSize = 10 * 1024 * 1024 // 10M @@ -23,11 +17,6 @@ const DefaultTimeout = 2 * time.Second type FileService struct{} -type FullFile struct { - Info *wshrpc.FileInfo `json:"info"` - Data64 string `json:"data64"` // base64 encoded -} - func (fs *FileService) SaveFile_Meta() tsgenmeta.MethodMeta { return tsgenmeta.MethodMeta{ Desc: "save file", @@ -39,10 +28,8 @@ func (fs *FileService) SaveFile(connection string, path string, data64 string) e if connection == "" { connection = wshrpc.LocalConnName } - connRoute := wshutil.MakeConnectionRouteId(connection) - client := wshserver.GetMainRpcClient() - writeData := wshrpc.CommandRemoteWriteFileData{Path: path, Data64: data64} - return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: connRoute}) + fsclient := fileshare.CreateFileShareClient(connection) + return fsclient.PutFile(path, data64) } func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta { @@ -56,36 +43,32 @@ func (fs *FileService) StatFile(connection string, path string) (*wshrpc.FileInf if connection == "" { connection = wshrpc.LocalConnName } - connRoute := wshutil.MakeConnectionRouteId(connection) - client := wshserver.GetMainRpcClient() - return wshclient.RemoteFileInfoCommand(client, path, &wshrpc.RpcOpts{Route: connRoute}) + fsclient := fileshare.CreateFileShareClient(connection) + return fsclient.Stat(path) } func (fs *FileService) Mkdir(connection string, path string) error { if connection == "" { connection = wshrpc.LocalConnName } - connRoute := wshutil.MakeConnectionRouteId(connection) - client := wshserver.GetMainRpcClient() - return wshclient.RemoteMkdirCommand(client, path, &wshrpc.RpcOpts{Route: connRoute}) + fsclient := fileshare.CreateFileShareClient(connection) + return fsclient.Mkdir(path) } func (fs *FileService) TouchFile(connection string, path string) error { if connection == "" { connection = wshrpc.LocalConnName } - connRoute := wshutil.MakeConnectionRouteId(connection) - client := wshserver.GetMainRpcClient() - return wshclient.RemoteFileTouchCommand(client, path, &wshrpc.RpcOpts{Route: connRoute}) + fsclient := fileshare.CreateFileShareClient(connection) + return fsclient.PutFile(path, "") } func (fs *FileService) Rename(connection string, path string, newPath string) error { if connection == "" { connection = wshrpc.LocalConnName } - connRoute := wshutil.MakeConnectionRouteId(connection) - client := wshserver.GetMainRpcClient() - return wshclient.RemoteFileRenameCommand(client, [2]string{path, newPath}, &wshrpc.RpcOpts{Route: connRoute}) + fsclient := fileshare.CreateFileShareClient(connection) + return fsclient.Move(path, newPath, false) } func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { @@ -95,63 +78,12 @@ func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { } } -func (fs *FileService) ReadFile(connection string, path string) (*FullFile, error) { +func (fs *FileService) ReadFile(connection string, path string) (*fileshare.FullFile, error) { if connection == "" { connection = wshrpc.LocalConnName } - connRoute := wshutil.MakeConnectionRouteId(connection) - client := wshserver.GetMainRpcClient() - streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: connRoute}) - fullFile := &FullFile{} - firstPk := true - isDir := false - var fileBuf bytes.Buffer - var fileInfoArr []*wshrpc.FileInfo - for respUnion := range rtnCh { - if respUnion.Error != nil { - return nil, respUnion.Error - } - resp := respUnion.Response - if firstPk { - firstPk = false - // first packet has the fileinfo - if len(resp.FileInfo) != 1 { - return nil, fmt.Errorf("stream file protocol error, first pk fileinfo len=%d", len(resp.FileInfo)) - } - fullFile.Info = resp.FileInfo[0] - if fullFile.Info.IsDir { - isDir = true - } - continue - } - if isDir { - if len(resp.FileInfo) == 0 { - continue - } - fileInfoArr = append(fileInfoArr, resp.FileInfo...) - } else { - if resp.Data64 == "" { - continue - } - decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(resp.Data64))) - _, err := io.Copy(&fileBuf, decoder) - if err != nil { - return nil, fmt.Errorf("stream file, failed to decode base64 data %q: %w", resp.Data64, err) - } - } - } - if isDir { - fiBytes, err := json.Marshal(fileInfoArr) - if err != nil { - return nil, fmt.Errorf("unable to serialize files %s", path) - } - fullFile.Data64 = base64.StdEncoding.EncodeToString(fiBytes) - } else { - // we can avoid this re-encoding if we ensure the remote side always encodes chunks of 3 bytes so we don't get padding chars - fullFile.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) - } - return fullFile, nil + fsclient := fileshare.CreateFileShareClient(connection) + return fsclient.Read(path) } func (fs *FileService) GetWaveFile(id string, path string) (any, error) { @@ -175,9 +107,8 @@ func (fs *FileService) DeleteFile(connection string, path string) error { if connection == "" { connection = wshrpc.LocalConnName } - connRoute := wshutil.MakeConnectionRouteId(connection) - client := wshserver.GetMainRpcClient() - return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: connRoute}) + fsclient := fileshare.CreateFileShareClient(connection) + return fsclient.Delete(path) } func (fs *FileService) GetFullConfig() wconfig.FullConfigType { From 7b9fc525eac886e6b550b1993d2562f164d7f95c Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 6 Jan 2025 16:35:21 -0800 Subject: [PATCH 005/126] remove listentries --- pkg/remote/fileshare/fileshare.go | 2 -- pkg/remote/fileshare/s3fs.go | 4 ---- pkg/remote/fileshare/wshfs.go | 4 ---- 3 files changed, 10 deletions(-) diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index f776f32f5..07b5c2f2e 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -22,8 +22,6 @@ type FileShareClient interface { Copy(srcPath, destPath string, recursive bool) error // Delete deletes the entry at the given path Delete(path string) error - // ListEntries returns a list of entries in the given directory - ListEntries(path string) ([]wshrpc.FileInfo, error) // GetFileShareName returns the name of the fileshare GetFileShareName() string } diff --git a/pkg/remote/fileshare/s3fs.go b/pkg/remote/fileshare/s3fs.go index 9998b637a..a7f29e2c3 100644 --- a/pkg/remote/fileshare/s3fs.go +++ b/pkg/remote/fileshare/s3fs.go @@ -46,10 +46,6 @@ func (c S3Client) Delete(path string) error { return nil } -func (c S3Client) ListEntries(path string) ([]wshrpc.FileInfo, error) { - return nil, nil -} - func (c S3Client) GetFileShareName() string { return "S3Client" } diff --git a/pkg/remote/fileshare/wshfs.go b/pkg/remote/fileshare/wshfs.go index 7af3776e5..439279c5f 100644 --- a/pkg/remote/fileshare/wshfs.go +++ b/pkg/remote/fileshare/wshfs.go @@ -110,10 +110,6 @@ func (c WshClient) Delete(path string) error { return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) ListEntries(path string) ([]wshrpc.FileInfo, error) { - return nil, nil -} - func (c WshClient) GetFileShareName() string { return "S3Client" } From ac8a08be85024fb163d3fb1457d89354aea7103f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 6 Jan 2025 16:38:42 -0800 Subject: [PATCH 006/126] fix name --- pkg/remote/fileshare/wshfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/remote/fileshare/wshfs.go b/pkg/remote/fileshare/wshfs.go index 439279c5f..e4a39c2bb 100644 --- a/pkg/remote/fileshare/wshfs.go +++ b/pkg/remote/fileshare/wshfs.go @@ -111,5 +111,5 @@ func (c WshClient) Delete(path string) error { } func (c WshClient) GetFileShareName() string { - return "S3Client" + return "WshClient" } From 6a34b0b8843fc1060442758276a3045521520dd7 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 7 Jan 2025 13:55:17 -0800 Subject: [PATCH 007/126] save --- frontend/types/gotypes.d.ts | 4 ++++ go.mod | 9 +++++++- go.sum | 16 ++++++++++++++ pkg/remote/awsconn/awsconn.go | 39 +++++++++++++++++++++++++++++++++++ pkg/wshrpc/wshrpctypes.go | 5 +++++ 5 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 pkg/remote/awsconn/awsconn.go diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 546c7043b..5d40969e6 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -311,6 +311,10 @@ declare global { "ssh:proxyjump"?: string[]; "ssh:userknownhostsfile"?: string[]; "ssh:globalknownhostsfile"?: string[]; + "aws:access_key_id"?: string; + "aws:secret_access_key"?: string; + "aws:region"?: string; + "aws:session_token"?: string; }; // wshrpc.ConnRequest diff --git a/go.mod b/go.mod index 19a2836b6..fe909e804 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.23.4 require ( github.com/alexflint/go-filemutex v1.3.0 github.com/aws/aws-sdk-go-v2 v1.32.7 - github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 github.com/creack/pty v1.1.21 github.com/fsnotify/fsnotify v1.8.0 github.com/golang-jwt/jwt/v5 v5.2.1 @@ -42,13 +42,20 @@ require ( cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/longrunning v0.5.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect + github.com/aws/aws-sdk-go-v2/config v1.28.7 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.48 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect github.com/aws/smithy-go v1.22.1 // indirect github.com/ebitengine/purego v0.8.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect diff --git a/go.sum b/go.sum index 42f8c25a3..ad236dda7 100644 --- a/go.sum +++ b/go.sum @@ -20,10 +20,18 @@ github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= +github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE= +github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= @@ -36,6 +44,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEH github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE= github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 h1:SAfh4pNx5LuTafKKWR02Y+hL3A+3TX8cTKG1OIAJaBk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go new file mode 100644 index 000000000..3bdc1e601 --- /dev/null +++ b/pkg/remote/awsconn/awsconn.go @@ -0,0 +1,39 @@ +// Description: This package is used to create a connection to AWS services. +package awsconn + +import ( + "context" + "fmt" + "regexp" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/wavetermdev/waveterm/pkg/wconfig" + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) + +var connectionRe = regexp.MustCompile(`^aws:\/\/(.*)$`) + +func GetConfigForConnection(ctx context.Context, connection string) (*aws.Config, error) { + connMatch := connectionRe.FindStringSubmatch(connection) + if connMatch == nil { + return nil, fmt.Errorf("invalid connection string: %s)", connection) + } + connection = connMatch[1] + connfile, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ConnectionsFile) + if len(cerrs) > 0 { + return nil, fmt.Errorf("error reading config file: %v", cerrs[0]) + } + optfns := []func(*config.LoadOptions) error{} + if connfile[connection] != nil { + connectionconfig := connfile.GetMap(connection) + if connectionconfig["aws:access_"] != "" { + optfns = append(optfns, config.LoadSharedConfigProfile()() + } + if connectionconfig[wshrpc.ConnKeywords.AwsRegion] != "" { + optfns = append(optfns, config.WithRegion(connectionconfig[wshrpc.ConnKeywords.AwsRegion])) + } + } + config, err := config.LoadDefaultConfig(ctx) + +} diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index d28adf5c8..59ea91d64 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -485,6 +485,11 @@ type ConnKeywords struct { SshProxyJump []string `json:"ssh:proxyjump,omitempty"` SshUserKnownHostsFile []string `json:"ssh:userknownhostsfile,omitempty"` SshGlobalKnownHostsFile []string `json:"ssh:globalknownhostsfile,omitempty"` + + AwsAccessKeyId *string `json:"aws:access_key_id,omitempty"` + AwsSecretAccessKey *string `json:"aws:secret_access_key,omitempty"` + AwsRegion *string `json:"aws:region,omitempty"` + AwsSessionToken *string `json:"aws:session_token,omitempty"` } type ConnRequest struct { From acebeaa128be41c34739ac96b520e25cad9fa65d Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 7 Jan 2025 16:40:03 -0800 Subject: [PATCH 008/126] add parseprofiles --- go.mod | 1 + go.sum | 2 + pkg/remote/awsconn/awsconn.go | 71 +++++++++++++++++++++++++++++++--- pkg/util/iterfn/iterfn.go | 29 ++++++++++++++ pkg/util/iterfn/iterfn_test.go | 51 ++++++++++++++++++++++++ pkg/wshrpc/wshrpctypes.go | 5 +-- 6 files changed, 149 insertions(+), 10 deletions(-) create mode 100644 pkg/util/iterfn/iterfn.go create mode 100644 pkg/util/iterfn/iterfn_test.go diff --git a/go.mod b/go.mod index fe909e804..418d98ebf 100644 --- a/go.mod +++ b/go.mod @@ -91,6 +91,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.2 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect ) replace github.com/kevinburke/ssh_config => github.com/wavetermdev/ssh_config v0.0.0-20241219203747-6409e4292f34 diff --git a/go.sum b/go.sum index ad236dda7..72c11fce9 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,8 @@ google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFN google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index 3bdc1e601..dc06ae07d 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -4,16 +4,22 @@ package awsconn import ( "context" "fmt" + "log" + "os" "regexp" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/wavetermdev/waveterm/pkg/util/iterfn" "github.com/wavetermdev/waveterm/pkg/wconfig" - "github.com/wavetermdev/waveterm/pkg/wshrpc" + "gopkg.in/ini.v1" ) var connectionRe = regexp.MustCompile(`^aws:\/\/(.*)$`) +var tempfiles map[string]string = make(map[string]string) + func GetConfigForConnection(ctx context.Context, connection string) (*aws.Config, error) { connMatch := connectionRe.FindStringSubmatch(connection) if connMatch == nil { @@ -27,13 +33,66 @@ func GetConfigForConnection(ctx context.Context, connection string) (*aws.Config optfns := []func(*config.LoadOptions) error{} if connfile[connection] != nil { connectionconfig := connfile.GetMap(connection) - if connectionconfig["aws:access_"] != "" { - optfns = append(optfns, config.LoadSharedConfigProfile()() + if connectionconfig["aws:config"] != "" { + var tempfile string + if tempfiles[connection] != "" { + tempfile = tempfiles[connection] + } else { + awsConfig := connectionconfig.GetString("aws:config", "") + tempfile, err := os.CreateTemp("", fmt.Sprintf("waveterm-awsconfig-%s", connection)) + if err != nil { + return nil, fmt.Errorf("error creating temp file: %v", err) + } + tempfile.WriteString(awsConfig) + } + optfns = append(optfns, config.WithSharedCredentialsFiles([]string{tempfile})) } - if connectionconfig[wshrpc.ConnKeywords.AwsRegion] != "" { - optfns = append(optfns, config.WithRegion(connectionconfig[wshrpc.ConnKeywords.AwsRegion])) + } + optfns = append(optfns, config.WithSharedConfigProfile(connection)) + cfg, err := config.LoadDefaultConfig(ctx, optfns...) + if err != nil { + return nil, fmt.Errorf("error loading config: %v", err) + } + return &cfg, nil +} + +func ParseProfiles() []string { + connfile, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ConnectionsFile) + profiles := map[string]any{} + if len(cerrs) > 0 { + log.Printf("error reading wave connections file: %v", cerrs[0]) + } else { + for k, _ := range connfile { + connMatch := connectionRe.FindStringSubmatch(k) + if connMatch != nil { + profiles[connMatch[1]] = struct{}{} + } } } - config, err := config.LoadDefaultConfig(ctx) + fname := config.DefaultSharedConfigFilename() // Get aws.config default shared configuration file name + f, err := ini.Load(fname) // Load ini file + if err != nil { + log.Printf("error reading aws config file: %v", err) + return iterfn.MapKeysToSorted(profiles) + } + for _, v := range f.Sections() { + if len(v.Keys()) != 0 { // Get only the sections having Keys + parts := strings.Split(v.Name(), " ") + if len(parts) == 2 && parts[0] == "profile" { // skip default + profiles[parts[1]] = struct{}{} + } + } + } + + fname = config.DefaultSharedCredentialsFilename() + f, err = ini.Load(fname) + if err != nil { + log.Printf("error reading aws credentials file: %v", err) + return iterfn.MapKeysToSorted(profiles) + } + for _, v := range f.Sections() { + profiles[v.Name()] = struct{}{} + } + return iterfn.MapKeysToSorted(profiles) } diff --git a/pkg/util/iterfn/iterfn.go b/pkg/util/iterfn/iterfn.go new file mode 100644 index 000000000..2cd5cb653 --- /dev/null +++ b/pkg/util/iterfn/iterfn.go @@ -0,0 +1,29 @@ +package iterfn + +import ( + "cmp" + "iter" + "maps" + "slices" +) + +func CollectSeqToSorted[T cmp.Ordered](seq iter.Seq[T]) []T { + rtn := []T{} + for v := range seq { + rtn = append(rtn, v) + } + slices.Sort(rtn) + return rtn +} + +func CollectSeq[T any](seq iter.Seq[T]) []T { + rtn := []T{} + for v := range seq { + rtn = append(rtn, v) + } + return rtn +} + +func MapKeysToSorted[K cmp.Ordered, V any](m map[K]V) []K { + return CollectSeqToSorted(maps.Keys(m)) +} diff --git a/pkg/util/iterfn/iterfn_test.go b/pkg/util/iterfn/iterfn_test.go new file mode 100644 index 000000000..6df11da5e --- /dev/null +++ b/pkg/util/iterfn/iterfn_test.go @@ -0,0 +1,51 @@ +package iterfn_test + +import ( + "maps" + "slices" + "testing" + + "github.com/wavetermdev/waveterm/pkg/util/iterfn" +) + +func TestCollectSeqToSorted(t *testing.T) { + t.Parallel() + + // Test code here + m := map[int]struct{}{1: {}, 3: {}, 2: {}} + got := iterfn.CollectSeqToSorted(maps.Keys(m)) + want := []int{1, 2, 3} + if !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestCollectSeq(t *testing.T) { + t.Parallel() + + // Test code here + m := map[int]struct{}{1: {}, 3: {}, 2: {}} + got := iterfn.CollectSeq(maps.Keys(m)) + i := 0 + for _, v := range got { + if _, ok := m[v]; !ok { + t.Errorf("collected value %v not in original map", v) + } + i++ + } + if i != len(m) { + t.Errorf("collected array length %v, want %v", i, len(m)) + } +} + +func TestMapKeysToSorted(t *testing.T) { + t.Parallel() + + // Test code here + m := map[int]struct{}{1: {}, 3: {}, 2: {}} + got := iterfn.MapKeysToSorted(m) + want := []int{1, 2, 3} + if !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 59ea91d64..161830d9b 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -486,10 +486,7 @@ type ConnKeywords struct { SshUserKnownHostsFile []string `json:"ssh:userknownhostsfile,omitempty"` SshGlobalKnownHostsFile []string `json:"ssh:globalknownhostsfile,omitempty"` - AwsAccessKeyId *string `json:"aws:access_key_id,omitempty"` - AwsSecretAccessKey *string `json:"aws:secret_access_key,omitempty"` - AwsRegion *string `json:"aws:region,omitempty"` - AwsSessionToken *string `json:"aws:session_token,omitempty"` + AwsConfig *string `json:"aws:config,omitempty"` } type ConnRequest struct { From 58be97db0403d3911a913a90f83d04d64b68254f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 7 Jan 2025 17:11:25 -0800 Subject: [PATCH 009/126] add connparse --- frontend/app/store/services.ts | 6 ++--- frontend/types/gotypes.d.ts | 5 +--- go.mod | 4 +-- go.sum | 2 -- pkg/remote/connparse.go | 18 +++++++++++++ pkg/remote/fileshare/fileshare.go | 20 ++++++++++++-- pkg/remote/fileshare/s3fs.go | 4 +-- pkg/service/fileservice/fileservice.go | 36 +++++++++++++------------- 8 files changed, 62 insertions(+), 33 deletions(-) create mode 100644 pkg/remote/connparse.go diff --git a/frontend/app/store/services.ts b/frontend/app/store/services.ts index e86bba7c4..81d4e1925 100644 --- a/frontend/app/store/services.ts +++ b/frontend/app/store/services.ts @@ -59,7 +59,7 @@ class FileServiceType { GetWaveFile(arg1: string, arg2: string): Promise { return WOS.callBackendService("file", "GetWaveFile", Array.from(arguments)) } - Mkdir(arg1: string, arg2: string): Promise { + Mkdir(arg2: string, arg3: string): Promise { return WOS.callBackendService("file", "Mkdir", Array.from(arguments)) } @@ -67,7 +67,7 @@ class FileServiceType { ReadFile(connection: string, path: string): Promise { return WOS.callBackendService("file", "ReadFile", Array.from(arguments)) } - Rename(arg1: string, arg2: string, arg3: string): Promise { + Rename(arg2: string, arg3: string, arg4: string): Promise { return WOS.callBackendService("file", "Rename", Array.from(arguments)) } @@ -80,7 +80,7 @@ class FileServiceType { StatFile(connection: string, path: string): Promise { return WOS.callBackendService("file", "StatFile", Array.from(arguments)) } - TouchFile(arg1: string, arg2: string): Promise { + TouchFile(arg2: string, arg3: string): Promise { return WOS.callBackendService("file", "TouchFile", Array.from(arguments)) } } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 0d17089ab..03620a0a2 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -311,10 +311,7 @@ declare global { "ssh:proxyjump"?: string[]; "ssh:userknownhostsfile"?: string[]; "ssh:globalknownhostsfile"?: string[]; - "aws:access_key_id"?: string; - "aws:secret_access_key"?: string; - "aws:region"?: string; - "aws:session_token"?: string; + "aws:config"?: string; }; // wshrpc.ConnRequest diff --git a/go.mod b/go.mod index 418d98ebf..ef89a6e9b 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.23.4 require ( github.com/alexflint/go-filemutex v1.3.0 github.com/aws/aws-sdk-go-v2 v1.32.7 + github.com/aws/aws-sdk-go-v2/config v1.28.7 github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 github.com/creack/pty v1.1.21 github.com/fsnotify/fsnotify v1.8.0 @@ -32,6 +33,7 @@ require ( golang.org/x/sys v0.28.0 golang.org/x/term v0.27.0 google.golang.org/api v0.214.0 + gopkg.in/ini.v1 v1.67.0 ) require ( @@ -42,7 +44,6 @@ require ( cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/longrunning v0.5.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/config v1.28.7 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.48 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect @@ -91,7 +92,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20241209162323-e6fa225c2576 // indirect google.golang.org/grpc v1.67.1 // indirect google.golang.org/protobuf v1.35.2 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect ) replace github.com/kevinburke/ssh_config => github.com/wavetermdev/ssh_config v0.0.0-20241219203747-6409e4292f34 diff --git a/go.sum b/go.sum index 72c11fce9..0c21b27d4 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,6 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQz github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= -github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1 h1:aOVVZJgWbaH+EJYPvEgkNhCEbXXvH7+oML36oaPK3zE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.71.1/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 h1:SAfh4pNx5LuTafKKWR02Y+hL3A+3TX8cTKG1OIAJaBk= github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g= diff --git a/pkg/remote/connparse.go b/pkg/remote/connparse.go new file mode 100644 index 000000000..ced2733be --- /dev/null +++ b/pkg/remote/connparse.go @@ -0,0 +1,18 @@ +package remote + +import "regexp" + +const ( + ConnectionTypeWsh = "wsh" + ConnectionTypeAws = "aws" +) + +var connectionRe = regexp.MustCompile(`^(\w+):\/\/(.*)$`) + +func ParseConnectionType(connection string) string { + connMatch := connectionRe.FindStringSubmatch(connection) + if connMatch == nil { + return ConnectionTypeWsh + } + return connMatch[1] +} diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 07b5c2f2e..d5cc10633 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -1,6 +1,13 @@ package fileshare -import "github.com/wavetermdev/waveterm/pkg/wshrpc" +import ( + "context" + "log" + + "github.com/wavetermdev/waveterm/pkg/remote" + "github.com/wavetermdev/waveterm/pkg/remote/awsconn" + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) type FullFile struct { Info *wshrpc.FileInfo `json:"info"` @@ -26,6 +33,15 @@ type FileShareClient interface { GetFileShareName() string } -func CreateFileShareClient(connection string) FileShareClient { +func CreateFileShareClient(ctx context.Context, connection string) FileShareClient { + connType := remote.ParseConnectionType(connection) + if connType == remote.ConnectionTypeAws { + config, err := awsconn.GetConfigForConnection(ctx, connection) + if err != nil { + log.Printf("error getting aws config: %v", err) + return nil + } + return NewS3Client(config) + } return NewWshClient(connection) } diff --git a/pkg/remote/fileshare/s3fs.go b/pkg/remote/fileshare/s3fs.go index a7f29e2c3..332d08b82 100644 --- a/pkg/remote/fileshare/s3fs.go +++ b/pkg/remote/fileshare/s3fs.go @@ -12,9 +12,9 @@ type S3Client struct { var _ FileShareClient = S3Client{} -func NewS3Client(config aws.Config) *S3Client { +func NewS3Client(config *aws.Config) *S3Client { return &S3Client{ - client: s3.NewFromConfig(config), + client: s3.NewFromConfig(*config), } } diff --git a/pkg/service/fileservice/fileservice.go b/pkg/service/fileservice/fileservice.go index 751714edd..0f734b58b 100644 --- a/pkg/service/fileservice/fileservice.go +++ b/pkg/service/fileservice/fileservice.go @@ -20,69 +20,69 @@ type FileService struct{} func (fs *FileService) SaveFile_Meta() tsgenmeta.MethodMeta { return tsgenmeta.MethodMeta{ Desc: "save file", - ArgNames: []string{"connection", "path", "data64"}, + ArgNames: []string{"ctx", "connection", "path", "data64"}, } } -func (fs *FileService) SaveFile(connection string, path string, data64 string) error { +func (fs *FileService) SaveFile(ctx context.Context, connection string, path string, data64 string) error { if connection == "" { connection = wshrpc.LocalConnName } - fsclient := fileshare.CreateFileShareClient(connection) + fsclient := fileshare.CreateFileShareClient(ctx, connection) return fsclient.PutFile(path, data64) } func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta { return tsgenmeta.MethodMeta{ Desc: "get file info", - ArgNames: []string{"connection", "path"}, + ArgNames: []string{"ctx", "connection", "path"}, } } -func (fs *FileService) StatFile(connection string, path string) (*wshrpc.FileInfo, error) { +func (fs *FileService) StatFile(ctx context.Context, connection string, path string) (*wshrpc.FileInfo, error) { if connection == "" { connection = wshrpc.LocalConnName } - fsclient := fileshare.CreateFileShareClient(connection) + fsclient := fileshare.CreateFileShareClient(ctx, connection) return fsclient.Stat(path) } -func (fs *FileService) Mkdir(connection string, path string) error { +func (fs *FileService) Mkdir(ctx context.Context, connection string, path string) error { if connection == "" { connection = wshrpc.LocalConnName } - fsclient := fileshare.CreateFileShareClient(connection) + fsclient := fileshare.CreateFileShareClient(ctx, connection) return fsclient.Mkdir(path) } -func (fs *FileService) TouchFile(connection string, path string) error { +func (fs *FileService) TouchFile(ctx context.Context, connection string, path string) error { if connection == "" { connection = wshrpc.LocalConnName } - fsclient := fileshare.CreateFileShareClient(connection) + fsclient := fileshare.CreateFileShareClient(ctx, connection) return fsclient.PutFile(path, "") } -func (fs *FileService) Rename(connection string, path string, newPath string) error { +func (fs *FileService) Rename(ctx context.Context, connection string, path string, newPath string) error { if connection == "" { connection = wshrpc.LocalConnName } - fsclient := fileshare.CreateFileShareClient(connection) + fsclient := fileshare.CreateFileShareClient(ctx, connection) return fsclient.Move(path, newPath, false) } func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { return tsgenmeta.MethodMeta{ Desc: "read file", - ArgNames: []string{"connection", "path"}, + ArgNames: []string{"ctx", "connection", "path"}, } } -func (fs *FileService) ReadFile(connection string, path string) (*fileshare.FullFile, error) { +func (fs *FileService) ReadFile(ctx context.Context, connection string, path string) (*fileshare.FullFile, error) { if connection == "" { connection = wshrpc.LocalConnName } - fsclient := fileshare.CreateFileShareClient(connection) + fsclient := fileshare.CreateFileShareClient(ctx, connection) return fsclient.Read(path) } @@ -99,15 +99,15 @@ func (fs *FileService) GetWaveFile(id string, path string) (any, error) { func (fs *FileService) DeleteFile_Meta() tsgenmeta.MethodMeta { return tsgenmeta.MethodMeta{ Desc: "delete file", - ArgNames: []string{"connection", "path"}, + ArgNames: []string{"ctx", "connection", "path"}, } } -func (fs *FileService) DeleteFile(connection string, path string) error { +func (fs *FileService) DeleteFile(ctx context.Context, connection string, path string) error { if connection == "" { connection = wshrpc.LocalConnName } - fsclient := fileshare.CreateFileShareClient(connection) + fsclient := fileshare.CreateFileShareClient(ctx, connection) return fsclient.Delete(path) } From 5596aa7911b7b1d188001b2a7d47f3b10cb92355 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 7 Jan 2025 17:38:49 -0800 Subject: [PATCH 010/126] switch import order --- pkg/remote/connparse.go | 3 ++ pkg/remote/fileshare/fileshare.go | 34 ++++------------------- pkg/remote/fileshare/fstype/fstype.go | 27 ++++++++++++++++++ pkg/remote/fileshare/{ => s3fs}/s3fs.go | 10 +++++-- pkg/remote/fileshare/{ => wshfs}/wshfs.go | 12 +++++--- pkg/service/fileservice/fileservice.go | 3 +- 6 files changed, 53 insertions(+), 36 deletions(-) create mode 100644 pkg/remote/fileshare/fstype/fstype.go rename pkg/remote/fileshare/{ => s3fs}/s3fs.go (76%) rename pkg/remote/fileshare/{ => wshfs}/wshfs.go (91%) diff --git a/pkg/remote/connparse.go b/pkg/remote/connparse.go index ced2733be..adc3f62bd 100644 --- a/pkg/remote/connparse.go +++ b/pkg/remote/connparse.go @@ -1,3 +1,6 @@ +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + package remote import "regexp" diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index d5cc10633..3baf6edca 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -6,34 +6,12 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/awsconn" - "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/s3fs" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" ) -type FullFile struct { - Info *wshrpc.FileInfo `json:"info"` - Data64 string `json:"data64"` // base64 encoded -} - -type FileShareClient interface { - // Stat returns the file info at the given path - Stat(path string) (*wshrpc.FileInfo, error) - // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo - Read(path string) (*FullFile, error) - // PutFile writes the given data to the file at the given path - PutFile(path string, data64 string) error - // Mkdir creates a directory at the given path - Mkdir(path string) error - // Move moves the file from srcPath to destPath - Move(srcPath, destPath string, recursive bool) error - // Copy copies the file from srcPath to destPath - Copy(srcPath, destPath string, recursive bool) error - // Delete deletes the entry at the given path - Delete(path string) error - // GetFileShareName returns the name of the fileshare - GetFileShareName() string -} - -func CreateFileShareClient(ctx context.Context, connection string) FileShareClient { +func CreateFileShareClient(ctx context.Context, connection string) fstype.FileShareClient { connType := remote.ParseConnectionType(connection) if connType == remote.ConnectionTypeAws { config, err := awsconn.GetConfigForConnection(ctx, connection) @@ -41,7 +19,7 @@ func CreateFileShareClient(ctx context.Context, connection string) FileShareClie log.Printf("error getting aws config: %v", err) return nil } - return NewS3Client(config) + return s3fs.NewS3Client(config) } - return NewWshClient(connection) + return wshfs.NewWshClient(connection) } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go new file mode 100644 index 000000000..201ec2759 --- /dev/null +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -0,0 +1,27 @@ +package fstype + +import "github.com/wavetermdev/waveterm/pkg/wshrpc" + +type FullFile struct { + Info *wshrpc.FileInfo `json:"info"` + Data64 string `json:"data64"` // base64 encoded +} + +type FileShareClient interface { + // Stat returns the file info at the given path + Stat(path string) (*wshrpc.FileInfo, error) + // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo + Read(path string) (*FullFile, error) + // PutFile writes the given data to the file at the given path + PutFile(path string, data64 string) error + // Mkdir creates a directory at the given path + Mkdir(path string) error + // Move moves the file from srcPath to destPath + Move(srcPath, destPath string, recursive bool) error + // Copy copies the file from srcPath to destPath + Copy(srcPath, destPath string, recursive bool) error + // Delete deletes the entry at the given path + Delete(path string) error + // GetFileShareName returns the name of the fileshare + GetFileShareName() string +} diff --git a/pkg/remote/fileshare/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go similarity index 76% rename from pkg/remote/fileshare/s3fs.go rename to pkg/remote/fileshare/s3fs/s3fs.go index 332d08b82..3688b316f 100644 --- a/pkg/remote/fileshare/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -1,8 +1,12 @@ -package fileshare +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package s3fs import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -10,7 +14,7 @@ type S3Client struct { client *s3.Client } -var _ FileShareClient = S3Client{} +var _ fstype.FileShareClient = S3Client{} func NewS3Client(config *aws.Config) *S3Client { return &S3Client{ @@ -18,7 +22,7 @@ func NewS3Client(config *aws.Config) *S3Client { } } -func (c S3Client) Read(path string) (*FullFile, error) { +func (c S3Client) Read(path string) (*fstype.FullFile, error) { return nil, nil } diff --git a/pkg/remote/fileshare/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go similarity index 91% rename from pkg/remote/fileshare/wshfs.go rename to pkg/remote/fileshare/wshfs/wshfs.go index e4a39c2bb..e3253981d 100644 --- a/pkg/remote/fileshare/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -1,4 +1,7 @@ -package fileshare +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package wshfs import ( "bytes" @@ -7,6 +10,7 @@ import ( "fmt" "io" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshserver" @@ -17,7 +21,7 @@ type WshClient struct { connRoute string } -var _ FileShareClient = WshClient{} +var _ fstype.FileShareClient = WshClient{} func NewWshClient(connection string) *WshClient { return &WshClient{ @@ -25,11 +29,11 @@ func NewWshClient(connection string) *WshClient { } } -func (c WshClient) Read(path string) (*FullFile, error) { +func (c WshClient) Read(path string) (*fstype.FullFile, error) { client := wshserver.GetMainRpcClient() streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: c.connRoute}) - fullFile := &FullFile{} + fullFile := &fstype.FullFile{} firstPk := true isDir := false var fileBuf bytes.Buffer diff --git a/pkg/service/fileservice/fileservice.go b/pkg/service/fileservice/fileservice.go index 0f734b58b..bcaba0bb6 100644 --- a/pkg/service/fileservice/fileservice.go +++ b/pkg/service/fileservice/fileservice.go @@ -7,6 +7,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/fileshare" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta" "github.com/wavetermdev/waveterm/pkg/wconfig" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -78,7 +79,7 @@ func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { } } -func (fs *FileService) ReadFile(ctx context.Context, connection string, path string) (*fileshare.FullFile, error) { +func (fs *FileService) ReadFile(ctx context.Context, connection string, path string) (*fstype.FullFile, error) { if connection == "" { connection = wshrpc.LocalConnName } From a6581d4157af0fe38c54edaf261e1827a4ee4b51 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 7 Jan 2025 19:01:01 -0800 Subject: [PATCH 011/126] add boilerplate wavefs impl --- pkg/remote/connparse.go | 5 +-- pkg/remote/fileshare/fileshare.go | 6 +++- pkg/remote/fileshare/fstype/fstype.go | 4 +-- pkg/remote/fileshare/s3fs/s3fs.go | 5 +-- pkg/remote/fileshare/wavefs/wavefs.go | 50 +++++++++++++++++++++++++++ pkg/remote/fileshare/wshfs/wshfs.go | 5 +-- 6 files changed, 66 insertions(+), 9 deletions(-) create mode 100644 pkg/remote/fileshare/wavefs/wavefs.go diff --git a/pkg/remote/connparse.go b/pkg/remote/connparse.go index adc3f62bd..672945e92 100644 --- a/pkg/remote/connparse.go +++ b/pkg/remote/connparse.go @@ -6,8 +6,9 @@ package remote import "regexp" const ( - ConnectionTypeWsh = "wsh" - ConnectionTypeAws = "aws" + ConnectionTypeWsh = "wsh" + ConnectionTypeAws = "aws" + ConnectionTypeWave = "wave" ) var connectionRe = regexp.MustCompile(`^(\w+):\/\/(.*)$`) diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 3baf6edca..acd440aed 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -8,6 +8,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/s3fs" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" ) @@ -20,6 +21,9 @@ func CreateFileShareClient(ctx context.Context, connection string) fstype.FileSh return nil } return s3fs.NewS3Client(config) + } else if connType == remote.ConnectionTypeWave { + return wavefs.NewWaveClient() + } else { + return wshfs.NewWshClient(connection) } - return wshfs.NewWshClient(connection) } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 201ec2759..7c1d6b6b2 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -22,6 +22,6 @@ type FileShareClient interface { Copy(srcPath, destPath string, recursive bool) error // Delete deletes the entry at the given path Delete(path string) error - // GetFileShareName returns the name of the fileshare - GetFileShareName() string + // GetConnectionType returns the type of connection for the fileshare + GetConnectionType() string } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 3688b316f..d07687f8f 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -6,6 +6,7 @@ package s3fs import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -50,6 +51,6 @@ func (c S3Client) Delete(path string) error { return nil } -func (c S3Client) GetFileShareName() string { - return "S3Client" +func (c S3Client) GetConnectionType() string { + return remote.ConnectionTypeWsh } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go new file mode 100644 index 000000000..62943c3e4 --- /dev/null +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -0,0 +1,50 @@ +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package wavefs + +import ( + "github.com/wavetermdev/waveterm/pkg/remote" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) + +type WaveClient struct{} + +var _ fstype.FileShareClient = WaveClient{} + +func NewWaveClient() *WaveClient { + return &WaveClient{} +} + +func (c WaveClient) Read(path string) (*fstype.FullFile, error) { + return nil, nil +} + +func (c WaveClient) Stat(path string) (*wshrpc.FileInfo, error) { + return nil, nil +} + +func (c WaveClient) PutFile(path string, data64 string) error { + return nil +} + +func (c WaveClient) Mkdir(path string) error { + return nil +} + +func (c WaveClient) Move(srcPath, destPath string, recursive bool) error { + return nil +} + +func (c WaveClient) Copy(srcPath, destPath string, recursive bool) error { + return nil +} + +func (c WaveClient) Delete(path string) error { + return nil +} + +func (c WaveClient) GetConnectionType() string { + return remote.ConnectionTypeWave +} diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index e3253981d..c65c2b934 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -10,6 +10,7 @@ import ( "fmt" "io" + "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" @@ -114,6 +115,6 @@ func (c WshClient) Delete(path string) error { return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) GetFileShareName() string { - return "WshClient" +func (c WshClient) GetConnectionType() string { + return remote.ConnectionTypeWsh } From dfc0ea31c7d1b30dacbf3f165a6dcf83abcbff38 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 10 Jan 2025 17:58:13 -0800 Subject: [PATCH 012/126] save --- cmd/wsh/cmd/wshcmd-file-util.go | 2 +- pkg/blockcontroller/blockcontroller.go | 2 +- pkg/filestore/blockstore.go | 34 ++++----- pkg/filestore/blockstore_test.go | 29 ++++---- pkg/remote/fileshare/fstype/fstype.go | 20 +++-- pkg/remote/fileshare/s3fs/s3fs.go | 16 ++-- pkg/remote/fileshare/wavefs/wavefs.go | 78 +++++++++++++++++-- pkg/remote/fileshare/wshfs/wshfs.go | 17 +++-- pkg/service/blockservice/blockservice.go | 6 +- pkg/service/fileservice/fileservice.go | 14 ++-- pkg/wcore/block.go | 2 +- pkg/wshrpc/wshrpctypes.go | 95 ++++++++++++------------ pkg/wshrpc/wshserver/wshserver.go | 61 +++++++-------- 13 files changed, 213 insertions(+), 163 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 098a9eb28..ca8051338 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -100,7 +100,7 @@ func streamReadFromWaveFile(fileData wshrpc.CommandFileData, size int64, writer } // Set up the ReadAt request - fileData.At = &wshrpc.CommandFileDataAt{ + fileData.At = &wshrpc.FileDataAt{ Offset: offset, Size: int64(length), } diff --git a/pkg/blockcontroller/blockcontroller.go b/pkg/blockcontroller/blockcontroller.go index df1ff1be7..d9214ca87 100644 --- a/pkg/blockcontroller/blockcontroller.go +++ b/pkg/blockcontroller/blockcontroller.go @@ -285,7 +285,7 @@ func (bc *BlockController) setupAndStartShellProcess(rc *RunShellOpts, blockMeta // create a circular blockfile for the output ctx, cancelFn := context.WithTimeout(context.Background(), 2*time.Second) defer cancelFn() - fsErr := filestore.WFS.MakeFile(ctx, bc.BlockId, BlockFile_Term, nil, filestore.FileOptsType{MaxSize: DefaultTermMaxFileSize, Circular: true}) + fsErr := filestore.WFS.MakeFile(ctx, bc.BlockId, BlockFile_Term, nil, wshrpc.FileOptsType{MaxSize: DefaultTermMaxFileSize, Circular: true}) if fsErr != nil && fsErr != fs.ErrExist { return nil, fmt.Errorf("error creating blockfile: %w", fsErr) } diff --git a/pkg/filestore/blockstore.go b/pkg/filestore/blockstore.go index ec161e85f..d67dea93a 100644 --- a/pkg/filestore/blockstore.go +++ b/pkg/filestore/blockstore.go @@ -18,6 +18,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/ijson" "github.com/wavetermdev/waveterm/pkg/panichandler" + "github.com/wavetermdev/waveterm/pkg/wshrpc" ) const ( @@ -49,26 +50,17 @@ var WFS *FileStore = &FileStore{ Cache: make(map[cacheKey]*CacheEntry), } -type FileOptsType struct { - MaxSize int64 `json:"maxsize,omitempty"` - Circular bool `json:"circular,omitempty"` - IJson bool `json:"ijson,omitempty"` - IJsonBudget int `json:"ijsonbudget,omitempty"` -} - -type FileMeta = map[string]any - type WaveFile struct { // these fields are static (not updated) - ZoneId string `json:"zoneid"` - Name string `json:"name"` - Opts FileOptsType `json:"opts"` - CreatedTs int64 `json:"createdts"` + ZoneId string `json:"zoneid"` + Name string `json:"name"` + Opts wshrpc.FileOptsType `json:"opts"` + CreatedTs int64 `json:"createdts"` // these fields are mutable - Size int64 `json:"size"` - ModTs int64 `json:"modts"` - Meta FileMeta `json:"meta"` // only top-level keys can be updated (lower levels are immutable) + Size int64 `json:"size"` + ModTs int64 `json:"modts"` + Meta wshrpc.FileMeta `json:"meta"` // only top-level keys can be updated (lower levels are immutable) } // for regular files this is just Size @@ -90,8 +82,8 @@ func (f WaveFile) DataStartIdx() int64 { } // this works because lower levels are immutable -func copyMeta(meta FileMeta) FileMeta { - newMeta := make(FileMeta) +func copyMeta(meta wshrpc.FileMeta) wshrpc.FileMeta { + newMeta := make(wshrpc.FileMeta) for k, v := range meta { newMeta[k] = v } @@ -119,7 +111,7 @@ type FileData struct { func (FileData) UseDBMap() {} // synchronous (does not interact with the cache) -func (s *FileStore) MakeFile(ctx context.Context, zoneId string, name string, meta FileMeta, opts FileOptsType) error { +func (s *FileStore) MakeFile(ctx context.Context, zoneId string, name string, meta wshrpc.FileMeta, opts wshrpc.FileOptsType) error { if opts.MaxSize < 0 { return fmt.Errorf("max size must be non-negative") } @@ -210,7 +202,7 @@ func (s *FileStore) ListFiles(ctx context.Context, zoneId string) ([]*WaveFile, return files, nil } -func (s *FileStore) WriteMeta(ctx context.Context, zoneId string, name string, meta FileMeta, merge bool) error { +func (s *FileStore) WriteMeta(ctx context.Context, zoneId string, name string, meta wshrpc.FileMeta, merge bool) error { return withLock(s, zoneId, name, func(entry *CacheEntry) error { err := entry.loadFileIntoCache(ctx) if err != nil { @@ -289,7 +281,7 @@ func (s *FileStore) AppendData(ctx context.Context, zoneId string, name string, func metaIncrement(file *WaveFile, key string, amount int) int { if file.Meta == nil { - file.Meta = make(FileMeta) + file.Meta = make(wshrpc.FileMeta) } val, ok := file.Meta[key].(int) if !ok { diff --git a/pkg/filestore/blockstore_test.go b/pkg/filestore/blockstore_test.go index 8cf45d40f..42fc7a343 100644 --- a/pkg/filestore/blockstore_test.go +++ b/pkg/filestore/blockstore_test.go @@ -18,6 +18,7 @@ import ( "github.com/google/uuid" "github.com/wavetermdev/waveterm/pkg/ijson" + "github.com/wavetermdev/waveterm/pkg/wshrpc" ) func initDb(t *testing.T) { @@ -82,7 +83,7 @@ func TestCreate(t *testing.T) { ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() zoneId := uuid.NewString() - err := WFS.MakeFile(ctx, zoneId, "testfile", nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, "testfile", nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -156,7 +157,7 @@ func TestDelete(t *testing.T) { ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() zoneId := uuid.NewString() - err := WFS.MakeFile(ctx, zoneId, "testfile", nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, "testfile", nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -170,11 +171,11 @@ func TestDelete(t *testing.T) { } // create two files in same zone, use DeleteZone to delete - err = WFS.MakeFile(ctx, zoneId, "testfile1", nil, FileOptsType{}) + err = WFS.MakeFile(ctx, zoneId, "testfile1", nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } - err = WFS.MakeFile(ctx, zoneId, "testfile2", nil, FileOptsType{}) + err = WFS.MakeFile(ctx, zoneId, "testfile2", nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -219,7 +220,7 @@ func TestSetMeta(t *testing.T) { ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() zoneId := uuid.NewString() - err := WFS.MakeFile(ctx, zoneId, "testfile", nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, "testfile", nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -323,7 +324,7 @@ func TestAppend(t *testing.T) { defer cancelFn() zoneId := uuid.NewString() fileName := "t2" - err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -351,7 +352,7 @@ func TestWriteFile(t *testing.T) { defer cancelFn() zoneId := uuid.NewString() fileName := "t3" - err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -372,7 +373,7 @@ func TestWriteFile(t *testing.T) { checkFileData(t, ctx, zoneId, fileName, "hello") // circular file - err = WFS.MakeFile(ctx, zoneId, "c1", nil, FileOptsType{Circular: true, MaxSize: 50}) + err = WFS.MakeFile(ctx, zoneId, "c1", nil, wshrpc.FileOptsType{Circular: true, MaxSize: 50}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -394,7 +395,7 @@ func TestCircularWrites(t *testing.T) { ctx, cancelFn := context.WithTimeout(context.Background(), 5*time.Second) defer cancelFn() zoneId := uuid.NewString() - err := WFS.MakeFile(ctx, zoneId, "c1", nil, FileOptsType{Circular: true, MaxSize: 50}) + err := WFS.MakeFile(ctx, zoneId, "c1", nil, wshrpc.FileOptsType{Circular: true, MaxSize: 50}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -483,7 +484,7 @@ func TestMultiPart(t *testing.T) { zoneId := uuid.NewString() fileName := "m2" data := makeText(80) - err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -537,7 +538,7 @@ func TestComputePartMap(t *testing.T) { testIntMapsEq(t, "map5", m, map[int]int{8: 80, 9: 100, 10: 100, 11: 60}) // now test circular - file = &WaveFile{Opts: FileOptsType{Circular: true, MaxSize: 1000}} + file = &WaveFile{Opts: wshrpc.FileOptsType{Circular: true, MaxSize: 1000}} m = file.computePartMap(10, 250) testIntMapsEq(t, "map6", m, map[int]int{0: 90, 1: 100, 2: 60}) m = file.computePartMap(990, 40) @@ -558,7 +559,7 @@ func TestSimpleDBFlush(t *testing.T) { defer cancelFn() zoneId := uuid.NewString() fileName := "t1" - err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -590,7 +591,7 @@ func TestConcurrentAppend(t *testing.T) { defer cancelFn() zoneId := uuid.NewString() fileName := "t1" - err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{}) + err := WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOptsType{}) if err != nil { t.Fatalf("error creating file: %v", err) } @@ -678,7 +679,7 @@ func TestIJson(t *testing.T) { defer cancelFn() zoneId := uuid.NewString() fileName := "ij1" - err := WFS.MakeFile(ctx, zoneId, fileName, nil, FileOptsType{IJson: true}) + err := WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOptsType{IJson: true}) if err != nil { t.Fatalf("error creating file: %v", err) } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 7c1d6b6b2..d3d95d690 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -1,6 +1,10 @@ package fstype -import "github.com/wavetermdev/waveterm/pkg/wshrpc" +import ( + "context" + + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) type FullFile struct { Info *wshrpc.FileInfo `json:"info"` @@ -9,19 +13,19 @@ type FullFile struct { type FileShareClient interface { // Stat returns the file info at the given path - Stat(path string) (*wshrpc.FileInfo, error) + Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo - Read(path string) (*FullFile, error) + Read(ctx context.Context, path string) (*FullFile, error) // PutFile writes the given data to the file at the given path - PutFile(path string, data64 string) error + PutFile(ctx context.Context, data wshrpc.FileData) error // Mkdir creates a directory at the given path - Mkdir(path string) error + Mkdir(ctx context.Context, path string) error // Move moves the file from srcPath to destPath - Move(srcPath, destPath string, recursive bool) error + Move(ctx context.Context, srcPath, destPath string, recursive bool) error // Copy copies the file from srcPath to destPath - Copy(srcPath, destPath string, recursive bool) error + Copy(ctx context.Context, srcPath, destPath string, recursive bool) error // Delete deletes the entry at the given path - Delete(path string) error + Delete(ctx context.Context, path string) error // GetConnectionType returns the type of connection for the fileshare GetConnectionType() string } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index d07687f8f..fb8b9bdd9 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -4,6 +4,8 @@ package s3fs import ( + "context" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/wavetermdev/waveterm/pkg/remote" @@ -23,31 +25,31 @@ func NewS3Client(config *aws.Config) *S3Client { } } -func (c S3Client) Read(path string) (*fstype.FullFile, error) { +func (c S3Client) Read(ctx context.Context, path string) (*fstype.FullFile, error) { return nil, nil } -func (c S3Client) Stat(path string) (*wshrpc.FileInfo, error) { +func (c S3Client) Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { return nil, nil } -func (c S3Client) PutFile(path string, data64 string) error { +func (c S3Client) PutFile(ctx context.Context, data wshrpc.FileData) error { return nil } -func (c S3Client) Mkdir(path string) error { +func (c S3Client) Mkdir(ctx context.Context, path string) error { return nil } -func (c S3Client) Move(srcPath, destPath string, recursive bool) error { +func (c S3Client) Move(ctx context.Context, srcPath, destPath string, recursive bool) error { return nil } -func (c S3Client) Copy(srcPath, destPath string, recursive bool) error { +func (c S3Client) Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { return nil } -func (c S3Client) Delete(path string) error { +func (c S3Client) Delete(ctx context.Context, path string) error { return nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 62943c3e4..0f6989f83 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -4,8 +4,17 @@ package wavefs import ( + "context" + "encoding/base64" + "fmt" + "io/fs" + "strings" + + "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/waveobj" + "github.com/wavetermdev/waveterm/pkg/wps" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -17,34 +26,89 @@ func NewWaveClient() *WaveClient { return &WaveClient{} } -func (c WaveClient) Read(path string) (*fstype.FullFile, error) { +func (c WaveClient) Read(ctx context.Context, path string) (*fstype.FullFile, error) { return nil, nil } -func (c WaveClient) Stat(path string) (*wshrpc.FileInfo, error) { +func (c WaveClient) Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { return nil, nil } -func (c WaveClient) PutFile(path string, data64 string) error { +func (c WaveClient) PutFile(ctx context.Context, data wshrpc.FileData) error { + dataBuf, err := base64.StdEncoding.DecodeString(data.Data64) + if err != nil { + return fmt.Errorf("error decoding data64: %w", err) + } + zoneId, fileName, err := parseFilePath(data.Path) + if err != nil { + return fmt.Errorf("error parsing path: %w", err) + } + if data.At != nil { + err = filestore.WFS.WriteAt(ctx, zoneId, fileName, data.At.Offset, dataBuf) + if err == fs.ErrNotExist { + return fmt.Errorf("NOTFOUND: %w", err) + } + if err != nil { + return fmt.Errorf("error writing to blockfile: %w", err) + } + } else { + err = filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf) + if err == fs.ErrNotExist { + return fmt.Errorf("NOTFOUND: %w", err) + } + if err != nil { + return fmt.Errorf("error writing to blockfile: %w", err) + } + } + wps.Broker.Publish(wps.WaveEvent{ + Event: wps.Event_BlockFile, + Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, zoneId).String()}, + Data: &wps.WSFileEventData{ + ZoneId: zoneId, + FileName: fileName, + FileOp: wps.FileOp_Invalidate, + }, + }) return nil } -func (c WaveClient) Mkdir(path string) error { +func (c WaveClient) Mkdir(ctx context.Context, path string) error { return nil } -func (c WaveClient) Move(srcPath, destPath string, recursive bool) error { +func (c WaveClient) Move(ctx context.Context, srcPath, destPath string, recursive bool) error { return nil } -func (c WaveClient) Copy(srcPath, destPath string, recursive bool) error { +func (c WaveClient) Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { return nil } -func (c WaveClient) Delete(path string) error { +func (c WaveClient) Delete(ctx context.Context, path string) error { + err := filestore.WFS.DeleteFile(ctx, wshrpc.RpcContext.BlockId, path) + if err != nil { + return fmt.Errorf("error deleting blockfile: %w", err) + } + wps.Broker.Publish(wps.WaveEvent{ + Event: wps.Event_BlockFile, + Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, data.ZoneId).String()}, + Data: &wps.WSFileEventData{ + ZoneId: data.ZoneId, + FileName: data.FileName, + FileOp: wps.FileOp_Delete, + }, + }) return nil } func (c WaveClient) GetConnectionType() string { return remote.ConnectionTypeWave } + +func parseFilePath(path string) (zoneId, fileName string, err error) { + parts := strings.SplitN(path, "/", 2) + if len(parts) < 2 { + return "", "", fmt.Errorf("invalid path: %s", path) + } + return parts[0], parts[1], nil +} diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index c65c2b934..47d789d87 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -5,6 +5,7 @@ package wshfs import ( "bytes" + "context" "encoding/base64" "encoding/json" "fmt" @@ -30,7 +31,7 @@ func NewWshClient(connection string) *WshClient { } } -func (c WshClient) Read(path string) (*fstype.FullFile, error) { +func (c WshClient) Read(ctx context.Context, path string) (*fstype.FullFile, error) { client := wshserver.GetMainRpcClient() streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: c.connRoute}) @@ -85,32 +86,32 @@ func (c WshClient) Read(path string) (*fstype.FullFile, error) { return fullFile, nil } -func (c WshClient) Stat(path string) (*wshrpc.FileInfo, error) { +func (c WshClient) Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { client := wshserver.GetMainRpcClient() return wshclient.RemoteFileInfoCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) PutFile(path string, data64 string) error { +func (c WshClient) PutFile(ctx context.Context, data wshrpc.FileData) error { client := wshserver.GetMainRpcClient() - writeData := wshrpc.CommandRemoteWriteFileData{Path: path, Data64: data64} + writeData := wshrpc.CommandRemoteWriteFileData{Path: data.Path, Data64: data.Data64} return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) Mkdir(path string) error { +func (c WshClient) Mkdir(ctx context.Context, path string) error { client := wshserver.GetMainRpcClient() return wshclient.RemoteMkdirCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) Move(srcPath, destPath string, recursive bool) error { +func (c WshClient) Move(ctx context.Context, srcPath, destPath string, recursive bool) error { client := wshserver.GetMainRpcClient() return wshclient.RemoteFileRenameCommand(client, [2]string{srcPath, destPath}, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) Copy(srcPath, destPath string, recursive bool) error { +func (c WshClient) Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { return nil } -func (c WshClient) Delete(path string) error { +func (c WshClient) Delete(ctx context.Context, path string) error { client := wshserver.GetMainRpcClient() return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } diff --git a/pkg/service/blockservice/blockservice.go b/pkg/service/blockservice/blockservice.go index a4d6ac19c..78098541d 100644 --- a/pkg/service/blockservice/blockservice.go +++ b/pkg/service/blockservice/blockservice.go @@ -54,12 +54,12 @@ func (bs *BlockService) SaveTerminalState(ctx context.Context, blockId string, s return fmt.Errorf("invalid state type: %q", stateType) } // ignore MakeFile error (already exists is ok) - filestore.WFS.MakeFile(ctx, blockId, "cache:term:"+stateType, nil, filestore.FileOptsType{}) + filestore.WFS.MakeFile(ctx, blockId, "cache:term:"+stateType, nil, wshrpc.FileOptsType{}) err = filestore.WFS.WriteFile(ctx, blockId, "cache:term:"+stateType, []byte(state)) if err != nil { return fmt.Errorf("cannot save terminal state: %w", err) } - fileMeta := filestore.FileMeta{ + fileMeta := wshrpc.FileMeta{ "ptyoffset": ptyOffset, "termsize": termSize, } @@ -84,7 +84,7 @@ func (bs *BlockService) SaveWaveAiData(ctx context.Context, blockId string, hist return fmt.Errorf("unable to serialize ai history: %v", err) } // ignore MakeFile error (already exists is ok) - filestore.WFS.MakeFile(ctx, blockId, "aidata", nil, filestore.FileOptsType{}) + filestore.WFS.MakeFile(ctx, blockId, "aidata", nil, wshrpc.FileOptsType{}) err = filestore.WFS.WriteFile(ctx, blockId, "aidata", historyBytes) if err != nil { return fmt.Errorf("cannot save terminal state: %w", err) diff --git a/pkg/service/fileservice/fileservice.go b/pkg/service/fileservice/fileservice.go index bcaba0bb6..d447d072b 100644 --- a/pkg/service/fileservice/fileservice.go +++ b/pkg/service/fileservice/fileservice.go @@ -30,7 +30,7 @@ func (fs *FileService) SaveFile(ctx context.Context, connection string, path str connection = wshrpc.LocalConnName } fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.PutFile(path, data64) + return fsclient.PutFile(ctx, wshrpc.FileData{Path: path, Data64: data64}) } func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta { @@ -45,7 +45,7 @@ func (fs *FileService) StatFile(ctx context.Context, connection string, path str connection = wshrpc.LocalConnName } fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Stat(path) + return fsclient.Stat(ctx, path) } func (fs *FileService) Mkdir(ctx context.Context, connection string, path string) error { @@ -53,7 +53,7 @@ func (fs *FileService) Mkdir(ctx context.Context, connection string, path string connection = wshrpc.LocalConnName } fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Mkdir(path) + return fsclient.Mkdir(ctx, path) } func (fs *FileService) TouchFile(ctx context.Context, connection string, path string) error { @@ -61,7 +61,7 @@ func (fs *FileService) TouchFile(ctx context.Context, connection string, path st connection = wshrpc.LocalConnName } fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.PutFile(path, "") + return fsclient.PutFile(ctx, wshrpc.FileData{Path: path, Data64: ""}) } func (fs *FileService) Rename(ctx context.Context, connection string, path string, newPath string) error { @@ -69,7 +69,7 @@ func (fs *FileService) Rename(ctx context.Context, connection string, path strin connection = wshrpc.LocalConnName } fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Move(path, newPath, false) + return fsclient.Move(ctx, path, newPath, false) } func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { @@ -84,7 +84,7 @@ func (fs *FileService) ReadFile(ctx context.Context, connection string, path str connection = wshrpc.LocalConnName } fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Read(path) + return fsclient.Read(ctx, path) } func (fs *FileService) GetWaveFile(id string, path string) (any, error) { @@ -109,7 +109,7 @@ func (fs *FileService) DeleteFile(ctx context.Context, connection string, path s connection = wshrpc.LocalConnName } fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Delete(path) + return fsclient.Delete(ctx, path) } func (fs *FileService) GetFullConfig() wconfig.FullConfigType { diff --git a/pkg/wcore/block.go b/pkg/wcore/block.go index 64972f002..160ea71c9 100644 --- a/pkg/wcore/block.go +++ b/pkg/wcore/block.go @@ -83,7 +83,7 @@ func CreateBlock(ctx context.Context, tabId string, blockDef *waveobj.BlockDef, // upload the files if present if len(blockDef.Files) > 0 { for fileName, fileDef := range blockDef.Files { - err := filestore.WFS.MakeFile(ctx, newBlockOID, fileName, fileDef.Meta, filestore.FileOptsType{}) + err := filestore.WFS.MakeFile(ctx, newBlockOID, fileName, fileDef.Meta, wshrpc.FileOptsType{}) if err != nil { return nil, fmt.Errorf("error making blockfile %q: %w", fileName, err) } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index df2361fcf..ea7a1dd8a 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -12,7 +12,6 @@ import ( "os" "reflect" - "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/ijson" "github.com/wavetermdev/waveterm/pkg/vdom" "github.com/wavetermdev/waveterm/pkg/waveobj" @@ -125,14 +124,14 @@ type WshRpcInterface interface { DeleteBlockCommand(ctx context.Context, data CommandDeleteBlockData) error DeleteSubBlockCommand(ctx context.Context, data CommandDeleteBlockData) error WaitForRouteCommand(ctx context.Context, data CommandWaitForRouteData) (bool, error) - FileCreateCommand(ctx context.Context, data CommandFileCreateData) error - FileDeleteCommand(ctx context.Context, data CommandFileData) error - FileAppendCommand(ctx context.Context, data CommandFileData) error + FileCreateCommand(ctx context.Context, data FileData) error + FileDeleteCommand(ctx context.Context, data FileData) error + FileAppendCommand(ctx context.Context, data FileData) error FileAppendIJsonCommand(ctx context.Context, data CommandAppendIJsonData) error - FileWriteCommand(ctx context.Context, data CommandFileData) error - FileReadCommand(ctx context.Context, data CommandFileData) (string, error) - FileInfoCommand(ctx context.Context, data CommandFileData) (*WaveFileInfo, error) - FileListCommand(ctx context.Context, data CommandFileListData) ([]*WaveFileInfo, error) + FileWriteCommand(ctx context.Context, data FileData) error + FileReadCommand(ctx context.Context, data FileData) (string, error) + FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) + FileListCommand(ctx context.Context, data CommandFileListData) ([]*FileInfo, error) EventPublishCommand(ctx context.Context, data wps.WaveEvent) error EventSubCommand(ctx context.Context, data wps.SubscriptionRequest) error EventUnsubCommand(ctx context.Context, data string) error @@ -324,42 +323,29 @@ type CommandBlockInputData struct { TermSize *waveobj.TermSize `json:"termsize,omitempty"` } -type CommandFileDataAt struct { +type FileDataAt struct { Offset int64 `json:"offset"` Size int64 `json:"size,omitempty"` } -type CommandFileData struct { - ZoneId string `json:"zoneid" wshcontext:"BlockId"` - FileName string `json:"filename"` - Data64 string `json:"data64,omitempty"` - At *CommandFileDataAt `json:"at,omitempty"` // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) -} - -type WaveFileInfo struct { - ZoneId string `json:"zoneid"` - Name string `json:"name"` - Opts filestore.FileOptsType `json:"opts,omitempty"` - Size int64 `json:"size,omitempty"` - CreatedTs int64 `json:"createdts,omitempty"` - ModTs int64 `json:"modts,omitempty"` - Meta map[string]any `json:"meta,omitempty"` - IsDir bool `json:"isdir,omitempty"` +type FileData struct { + Path string `json:"path"` + Data64 string `json:"data64,omitempty"` + Opts *FileOptsType `json:"opts,omitempty"` + At *FileDataAt `json:"at,omitempty"` // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) } type CommandFileListData struct { - ZoneId string `json:"zoneid"` - Prefix string `json:"prefix,omitempty"` + Path string `json:"path"` All bool `json:"all,omitempty"` Offset int `json:"offset,omitempty"` Limit int `json:"limit,omitempty"` } -type CommandFileCreateData struct { - ZoneId string `json:"zoneid"` - FileName string `json:"filename"` - Meta map[string]any `json:"meta,omitempty"` - Opts *filestore.FileOptsType `json:"opts,omitempty"` +type FileCreateData struct { + Path string `json:"path"` + Meta map[string]any `json:"meta,omitempty"` + Opts *FileOptsType `json:"opts,omitempty"` } type CommandAppendIJsonData struct { @@ -435,18 +421,29 @@ type CpuDataType struct { } type FileInfo struct { - Path string `json:"path"` // cleaned path (may have "~") - Dir string `json:"dir"` // returns the directory part of the path (if this is a a directory, it will be equal to Path). "~" will be expanded, and separators will be normalized to "/" - Name string `json:"name"` - NotFound bool `json:"notfound,omitempty"` - Size int64 `json:"size"` - Mode os.FileMode `json:"mode"` - ModeStr string `json:"modestr"` - ModTime int64 `json:"modtime"` - IsDir bool `json:"isdir,omitempty"` - MimeType string `json:"mimetype,omitempty"` - ReadOnly bool `json:"readonly,omitempty"` // this is not set for fileinfo's returned from directory listings -} + Path string `json:"path"` // cleaned path (may have "~") + Dir string `json:"dir"` // returns the directory part of the path (if this is a a directory, it will be equal to Path). "~" will be expanded, and separators will be normalized to "/" + Name string `json:"name"` + NotFound bool `json:"notfound,omitempty"` + Opts FileOptsType `json:"opts,omitempty"` + Size int64 `json:"size"` + Meta FileMeta `json:"meta,omitempty"` + Mode os.FileMode `json:"mode"` + ModeStr string `json:"modestr"` + ModTime int64 `json:"modtime"` + IsDir bool `json:"isdir,omitempty"` + MimeType string `json:"mimetype,omitempty"` + ReadOnly bool `json:"readonly,omitempty"` // this is not set for fileinfo's returned from directory listings +} + +type FileOptsType struct { + MaxSize int64 `json:"maxsize,omitempty"` + Circular bool `json:"circular,omitempty"` + IJson bool `json:"ijson,omitempty"` + IJsonBudget int `json:"ijsonbudget,omitempty"` +} + +type FileMeta = map[string]any type CommandRemoteStreamFileData struct { Path string `json:"path"` @@ -562,11 +559,11 @@ type CommandWebSelectorData struct { } type BlockInfoData struct { - BlockId string `json:"blockid"` - TabId string `json:"tabid"` - WorkspaceId string `json:"workspaceid"` - Block *waveobj.Block `json:"block"` - Files []*filestore.WaveFile `json:"files"` + BlockId string `json:"blockid"` + TabId string `json:"tabid"` + WorkspaceId string `json:"workspaceid"` + Block *waveobj.Block `json:"block"` + Files []*FileInfo `json:"files"` } type WaveNotificationOptions struct { diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 3e0bc2554..928bf4130 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -24,6 +24,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/conncontroller" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/util/envutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" @@ -91,7 +92,7 @@ func MakePlotData(ctx context.Context, blockId string) error { if viewName != "cpuplot" && viewName != "sysinfo" { return fmt.Errorf("invalid view type: %s", viewName) } - return filestore.WFS.MakeFile(ctx, blockId, "cpuplotdata", nil, filestore.FileOptsType{}) + return filestore.WFS.MakeFile(ctx, blockId, "cpuplotdata", nil, wshrpc.FileOptsType{}) } func SavePlotData(ctx context.Context, blockId string, history string) error { @@ -275,46 +276,34 @@ func (ws *WshServer) ControllerAppendOutputCommand(ctx context.Context, data wsh return nil } -func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.CommandFileCreateData) error { - var fileOpts filestore.FileOptsType +func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.FileCreateData) error { + var fileOpts wshrpc.FileOptsType if data.Opts != nil { fileOpts = *data.Opts } - err := filestore.WFS.MakeFile(ctx, data.ZoneId, data.FileName, data.Meta, fileOpts) + client := fileshare.CreateFileShareClient(ctx, remote.ConnectionTypeWave) + err := client.PutFile(ctx, wshrpc.FileData{ + Path: data.Path, + Data64: "", + Opts: &fileOpts, + }) if err != nil { - return fmt.Errorf("error creating blockfile: %w", err) + return fmt.Errorf("error creating file: %w", err) } - wps.Broker.Publish(wps.WaveEvent{ - Event: wps.Event_BlockFile, - Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, data.ZoneId).String()}, - Data: &wps.WSFileEventData{ - ZoneId: data.ZoneId, - FileName: data.FileName, - FileOp: wps.FileOp_Create, - }, - }) return nil } -func (ws *WshServer) FileDeleteCommand(ctx context.Context, data wshrpc.CommandFileData) error { - err := filestore.WFS.DeleteFile(ctx, data.ZoneId, data.FileName) +func (ws *WshServer) FileDeleteCommand(ctx context.Context, data wshrpc.FileData) error { + client := fileshare.CreateFileShareClient(ctx, remote.ConnectionTypeWave) + err := client.Delete(ctx, data.Path) if err != nil { - return fmt.Errorf("error deleting blockfile: %w", err) + return fmt.Errorf("error deleting file: %w", err) } - wps.Broker.Publish(wps.WaveEvent{ - Event: wps.Event_BlockFile, - Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, data.ZoneId).String()}, - Data: &wps.WSFileEventData{ - ZoneId: data.ZoneId, - FileName: data.FileName, - FileOp: wps.FileOp_Delete, - }, - }) return nil } -func waveFileToWaveFileInfo(wf *filestore.WaveFile) *wshrpc.WaveFileInfo { - return &wshrpc.WaveFileInfo{ +func waveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { + return &wshrpc.FileInfo{ ZoneId: wf.ZoneId, Name: wf.Name, Opts: wf.Opts, @@ -325,7 +314,7 @@ func waveFileToWaveFileInfo(wf *filestore.WaveFile) *wshrpc.WaveFileInfo { } } -func (ws *WshServer) FileInfoCommand(ctx context.Context, data wshrpc.CommandFileData) (*wshrpc.WaveFileInfo, error) { +func (ws *WshServer) FileInfoCommand(ctx context.Context, data wshrpc.CommandFileData) (*wshrpc.FileInfo, error) { fileInfo, err := filestore.WFS.Stat(ctx, data.ZoneId, data.FileName) if err != nil { if err == fs.ErrNotExist { @@ -333,20 +322,20 @@ func (ws *WshServer) FileInfoCommand(ctx context.Context, data wshrpc.CommandFil } return nil, fmt.Errorf("error getting file info: %w", err) } - return waveFileToWaveFileInfo(fileInfo), nil + return waveFileToFileInfo(fileInfo), nil } -func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.CommandFileListData) ([]*wshrpc.WaveFileInfo, error) { +func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.CommandFileListData) ([]*wshrpc.FileInfo, error) { fileListOrig, err := filestore.WFS.ListFiles(ctx, data.ZoneId) if err != nil { return nil, fmt.Errorf("error listing blockfiles: %w", err) } - var fileList []*wshrpc.WaveFileInfo + var fileList []*wshrpc.FileInfo for _, wf := range fileListOrig { - fileList = append(fileList, waveFileToWaveFileInfo(wf)) + fileList = append(fileList, waveFileToFileInfo(wf)) } if data.Prefix != "" { - var filteredList []*wshrpc.WaveFileInfo + var filteredList []*wshrpc.FileInfo for _, file := range fileList { if strings.HasPrefix(file.Name, data.Prefix) { filteredList = append(filteredList, file) @@ -355,7 +344,7 @@ func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.CommandFil fileList = filteredList } if !data.All { - var filteredList []*wshrpc.WaveFileInfo + var filteredList []*wshrpc.FileInfo dirMap := make(map[string]int64) // the value is max modtime for _, file := range fileList { // if there is an extra "/" after the prefix, don't include it @@ -373,7 +362,7 @@ func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.CommandFil filteredList = append(filteredList, file) } for dir := range dirMap { - filteredList = append(filteredList, &wshrpc.WaveFileInfo{ + filteredList = append(filteredList, &wshrpc.FileInfo{ ZoneId: data.ZoneId, Name: data.Prefix + dir + "/", Size: 0, From ccdcf4ef65f66d66c70b78473e0e533bda507fbf Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 14:08:33 -0800 Subject: [PATCH 013/126] connparse impl --- pkg/remote/connparse.go | 22 --- pkg/remote/connparse/connparse.go | 118 +++++++++++++++ pkg/remote/connparse/connparse_test.go | 189 +++++++++++++++++++++++++ pkg/remote/fileshare/fileshare.go | 90 ++++++++++-- pkg/remote/fileshare/fstype/fstype.go | 24 ++-- pkg/remote/fileshare/s3fs/s3fs.go | 18 +-- pkg/remote/fileshare/wavefs/wavefs.go | 23 +-- pkg/remote/fileshare/wshfs/wshfs.go | 50 +++---- pkg/service/fileservice/fileservice.go | 42 +----- 9 files changed, 455 insertions(+), 121 deletions(-) delete mode 100644 pkg/remote/connparse.go create mode 100644 pkg/remote/connparse/connparse.go create mode 100644 pkg/remote/connparse/connparse_test.go diff --git a/pkg/remote/connparse.go b/pkg/remote/connparse.go deleted file mode 100644 index 672945e92..000000000 --- a/pkg/remote/connparse.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2025, Command Line Inc. -// SPDX-License-Identifier: Apache-2.0 - -package remote - -import "regexp" - -const ( - ConnectionTypeWsh = "wsh" - ConnectionTypeAws = "aws" - ConnectionTypeWave = "wave" -) - -var connectionRe = regexp.MustCompile(`^(\w+):\/\/(.*)$`) - -func ParseConnectionType(connection string) string { - connMatch := connectionRe.FindStringSubmatch(connection) - if connMatch == nil { - return ConnectionTypeWsh - } - return connMatch[1] -} diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go new file mode 100644 index 000000000..32fcbe2b3 --- /dev/null +++ b/pkg/remote/connparse/connparse.go @@ -0,0 +1,118 @@ +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package connparse + +import ( + "log" + "net/url" + "os" + "strings" +) + +const ( + ConnectionTypeWsh = "wsh" + ConnectionTypeS3 = "s3" + ConnectionTypeWave = "wavefile" +) + +type Connection struct { + Scheme string + Host string + Path string + Params *url.Values +} + +func (c *Connection) GetParam(key string) string { + return c.Params.Get(key) +} + +func (c *Connection) SetParam(key, value string) { + c.Params.Set(key, value) +} + +func (c *Connection) GetSchemeParts() []string { + return strings.Split(c.Scheme, ":") +} + +func (c *Connection) GetType() string { + lastInd := strings.LastIndex(c.Scheme, ":") + if lastInd == -1 { + return c.Scheme + } + return c.Scheme[lastInd+1:] +} + +func (c *Connection) GetPathWithHost() string { + if c.Host == "" { + return "" + } + if strings.HasPrefix(c.Path, "/") { + return c.Host + c.Path + } + return c.Host + "/" + c.Path +} + +func (c *Connection) GetFullPath() string { + return c.Scheme + "://" + c.GetPathWithHost() + "?" + c.Params.Encode() +} + +// ParseURI parses a connection URI and returns the connection type, host/path, and parameters. +func ParseURI(uri string, connConfigPath string) (*Connection, error) { + split := strings.SplitN(uri, "://", 2) + var scheme string + var rest string + if len(split) > 1 { + scheme = split[0] + rest = split[1] + } else { + rest = split[0] + } + if scheme == "" { + scheme = "wsh" + } + + var host string + var path string + var params url.Values + if strings.HasPrefix(rest, "//") { + rest = strings.TrimPrefix(rest, "//") + split = strings.SplitN(rest, "/", 2) + if len(split) > 1 { + host = split[0] + path = "/" + split[1] + } else { + host = split[0] + path = "/" + } + } else if strings.HasPrefix(rest, "/~") { + host = "local" + path = strings.TrimPrefix(rest, "/") + } else if stat, _ := os.Stat(rest); stat != nil { + host = "current" + path = rest + } else { + parsedUrl, err := url.Parse("http://" + rest) + if err != nil { + return nil, err + } + host = parsedUrl.Host + if parsedUrl.User != nil { + host = parsedUrl.User.Username() + "@" + host + } + log.Printf("parsedUrl: %v", parsedUrl) + log.Printf("parsedUrl.Host: %v", parsedUrl.Host) + log.Printf("parsedUrl.Path: %v", parsedUrl.Path) + log.Printf("parsedUrl.User: %v", parsedUrl.User) + params = parsedUrl.Query() + path = parsedUrl.Path + } + + log.Printf("scheme: %v", scheme) + return &Connection{ + Scheme: scheme, + Host: host, + Path: path, + Params: ¶ms, + }, nil +} diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go new file mode 100644 index 000000000..ea813826a --- /dev/null +++ b/pkg/remote/connparse/connparse_test.go @@ -0,0 +1,189 @@ +package connparse_test + +import ( + "testing" + + "github.com/wavetermdev/waveterm/pkg/remote/connparse" +) + +func TestParseURI_BasicWSH(t *testing.T) { + t.Parallel() + cstr := "wsh://localhost:8080/path/to/file" + c, err := connparse.ParseURI(cstr, "") + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := "/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "localhost:8080" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "localhost:8080/path/to/file" + pathWithHost := c.GetPathWithHost() + if pathWithHost != expected { + t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } + if len(c.GetSchemeParts()) != 1 { + t.Fatalf("expected scheme parts to be 1, got %d", len(c.GetSchemeParts())) + } + if len(*c.Params) != 0 { + t.Fatalf("expected params to be empty, got %v", *c.Params) + } +} + +func TestParseURI_FullConnectionWSH(t *testing.T) { + t.Parallel() + cstr := "wsh://user@192.168.0.1:22/path/to/file?foo=bar&baz=qux" + c, err := connparse.ParseURI(cstr, "") + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := "/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "user@192.168.0.1:22" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "user@192.168.0.1:22/path/to/file" + pathWithHost := c.GetPathWithHost() + if pathWithHost != expected { + t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) + } + expected = "wsh" + if c.GetType() != expected { + t.Fatalf("expected conn type to be %q, got %q", expected, c.Scheme) + } + if len(c.GetSchemeParts()) != 1 { + t.Fatalf("expected scheme parts to be 1, got %d", len(c.GetSchemeParts())) + } + if len(*c.Params) != 2 { + t.Fatalf("expected params to have 2 items, got %v", *c.Params) + } + got := c.GetFullPath() + if len(got) != len(cstr) { + t.Fatalf("expected full path to be %q, got %q (looking at string length since params can be out of order)", cstr, got) + } +} + +func TestParseURI_MissingScheme(t *testing.T) { + t.Parallel() + cstr := "localhost:8080/path/to/file" + c, err := connparse.ParseURI(cstr, "") + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := "/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "localhost:8080" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } +} + +func TestParseURI_WSHShorthand(t *testing.T) { + t.Parallel() + cstr := "//conn/path/to/file" + c, err := connparse.ParseURI(cstr, "") + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := "/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + if c.Host != "conn" { + t.Fatalf("expected host to be empty, got %q", c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } +} + +func TestParseURI_WSHLocalHomeShorthand(t *testing.T) { + t.Parallel() + cstr := "/~/path/to/file" + c, err := connparse.ParseURI(cstr, "") + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := "~/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + if c.Host != "local" { + t.Fatalf("expected host to be empty, got %q", c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } +} + +func TestParseURI_WSHCurrentAbsolutePath(t *testing.T) { + t.Parallel() + cstr := t.TempDir() + c, err := connparse.ParseURI(cstr, "") + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := cstr + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "current" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } +} + +func TestParseURI_BasicS3(t *testing.T) { + t.Parallel() + cstr := "profile:s3://bucket/path/to/file" + c, err := connparse.ParseURI(cstr, "") + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := "/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "bucket" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "bucket/path/to/file" + pathWithHost := c.GetPathWithHost() + if pathWithHost != expected { + t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) + } + expected = "s3" + if c.GetType() != expected { + t.Fatalf("expected conn type to be %q, got %q", expected, c.GetType()) + } + if len(c.GetSchemeParts()) != 2 { + t.Fatalf("expected scheme parts to be 2, got %d", len(c.GetSchemeParts())) + } + if len(*c.Params) != 0 { + t.Fatalf("expected params to be empty, got %v", *c.Params) + } +} diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index acd440aed..07d6ca0f5 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -2,28 +2,100 @@ package fileshare import ( "context" + "fmt" "log" - "github.com/wavetermdev/waveterm/pkg/remote" "github.com/wavetermdev/waveterm/pkg/remote/awsconn" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/s3fs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" + "github.com/wavetermdev/waveterm/pkg/wshrpc" ) -func CreateFileShareClient(ctx context.Context, connection string) fstype.FileShareClient { - connType := remote.ParseConnectionType(connection) - if connType == remote.ConnectionTypeAws { +// CreateFileShareClient creates a fileshare client based on the connection string +// Returns the client and the parsed connection +func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileShareClient, *connparse.Connection) { + conn, err := connparse.ParseURI(connection) + if err != nil { + log.Printf("error parsing connection: %v", err) + return nil, nil + } + conntype := conn.GetType() + if conntype == connparse.ConnectionTypeS3 { config, err := awsconn.GetConfigForConnection(ctx, connection) if err != nil { log.Printf("error getting aws config: %v", err) - return nil + return nil, nil } - return s3fs.NewS3Client(config) - } else if connType == remote.ConnectionTypeWave { - return wavefs.NewWaveClient() + return s3fs.NewS3Client(config), conn + } else if conntype == connparse.ConnectionTypeWave { + return wavefs.NewWaveClient(), conn } else { - return wshfs.NewWshClient(connection) + return wshfs.NewWshClient(connection), conn + } +} + +func Read(ctx context.Context, path string) (*fstype.FullFile, error) { + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + } + return client.Read(ctx, conn) +} + +func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + } + return client.Stat(ctx, conn) +} + +func PutFile(ctx context.Context, data wshrpc.FileData) error { + client, conn := CreateFileShareClient(ctx, data.Path) + if conn == nil || client == nil { + return fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Path) + } + return client.PutFile(ctx, fstype.FileData{ + Conn: conn, + Data64: data.Data64, + Opts: data.Opts, + At: data.At, + }) +} + +func Mkdir(ctx context.Context, path string) error { + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + } + return client.Mkdir(ctx, conn) +} + +// TODO: Implement move across different fileshare types +func Move(ctx context.Context, srcPath, destPath string, recursive bool) error { + srcClient, srcConn := CreateFileShareClient(ctx, srcPath) + if srcConn == nil || srcClient == nil { + return fmt.Errorf("error creating fileshare client, could not parse connection %s or %s", srcPath, destPath) + } + destConn, err := connparse.ParseURI(destPath) + if err != nil { + return fmt.Errorf("error parsing destination connection %s: %v", destPath, err) + } + return srcClient.Move(ctx, srcConn, destConn, recursive) +} + +// TODO: Implement copy across different fileshare types +func Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { + return nil +} + +func Delete(ctx context.Context, path string) error { + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return fmt.Errorf("error creating fileshare client, could not parse connection %s", path) } + return client.Delete(ctx, conn) } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index d3d95d690..46d9e4395 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -3,6 +3,7 @@ package fstype import ( "context" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -11,21 +12,28 @@ type FullFile struct { Data64 string `json:"data64"` // base64 encoded } +type FileData struct { + Conn *connparse.Connection + Data64 string + Opts *wshrpc.FileOptsType + At *wshrpc.FileDataAt // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) +} + type FileShareClient interface { - // Stat returns the file info at the given path - Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) + // Stat returns the file info at the given parsed connection path + Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo - Read(ctx context.Context, path string) (*FullFile, error) + Read(ctx context.Context, conn *connparse.Connection) (*FullFile, error) // PutFile writes the given data to the file at the given path - PutFile(ctx context.Context, data wshrpc.FileData) error + PutFile(ctx context.Context, data FileData) error // Mkdir creates a directory at the given path - Mkdir(ctx context.Context, path string) error + Mkdir(ctx context.Context, conn *connparse.Connection) error // Move moves the file from srcPath to destPath - Move(ctx context.Context, srcPath, destPath string, recursive bool) error + Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error // Copy copies the file from srcPath to destPath - Copy(ctx context.Context, srcPath, destPath string, recursive bool) error + Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error // Delete deletes the entry at the given path - Delete(ctx context.Context, path string) error + Delete(ctx context.Context, conn *connparse.Connection) error // GetConnectionType returns the type of connection for the fileshare GetConnectionType() string } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index fb8b9bdd9..51728212d 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -8,7 +8,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/wavetermdev/waveterm/pkg/remote" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -25,34 +25,34 @@ func NewS3Client(config *aws.Config) *S3Client { } } -func (c S3Client) Read(ctx context.Context, path string) (*fstype.FullFile, error) { +func (c S3Client) Read(ctx context.Context, conn *connparse.Connection) (*fstype.FullFile, error) { return nil, nil } -func (c S3Client) Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { +func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { return nil, nil } -func (c S3Client) PutFile(ctx context.Context, data wshrpc.FileData) error { +func (c S3Client) PutFile(ctx context.Context, data fstype.FileData) error { return nil } -func (c S3Client) Mkdir(ctx context.Context, path string) error { +func (c S3Client) Mkdir(ctx context.Context, conn *connparse.Connection) error { return nil } -func (c S3Client) Move(ctx context.Context, srcPath, destPath string, recursive bool) error { +func (c S3Client) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { return nil } -func (c S3Client) Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { +func (c S3Client) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { return nil } -func (c S3Client) Delete(ctx context.Context, path string) error { +func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection) error { return nil } func (c S3Client) GetConnectionType() string { - return remote.ConnectionTypeWsh + return connparse.ConnectionTypeS3 } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 0f6989f83..9cee085ed 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -8,10 +8,11 @@ import ( "encoding/base64" "fmt" "io/fs" + "regexp" "strings" "github.com/wavetermdev/waveterm/pkg/filestore" - "github.com/wavetermdev/waveterm/pkg/remote" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" @@ -22,19 +23,21 @@ type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} +var wavefilePathRe = regexp.MustCompile(`^wavefile:\/\/([^?]+)(?:\?(?:([^=]+)=([^&]+))(?:&([^=]+)=([^&]+))*)$`) + func NewWaveClient() *WaveClient { return &WaveClient{} } -func (c WaveClient) Read(ctx context.Context, path string) (*fstype.FullFile, error) { +func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection) (*fstype.FullFile, error) { return nil, nil } -func (c WaveClient) Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { +func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { return nil, nil } -func (c WaveClient) PutFile(ctx context.Context, data wshrpc.FileData) error { +func (c WaveClient) PutFile(ctx context.Context, data fstype.FileData) error { dataBuf, err := base64.StdEncoding.DecodeString(data.Data64) if err != nil { return fmt.Errorf("error decoding data64: %w", err) @@ -72,19 +75,19 @@ func (c WaveClient) PutFile(ctx context.Context, data wshrpc.FileData) error { return nil } -func (c WaveClient) Mkdir(ctx context.Context, path string) error { +func (c WaveClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { return nil } -func (c WaveClient) Move(ctx context.Context, srcPath, destPath string, recursive bool) error { +func (c WaveClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { return nil } -func (c WaveClient) Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { +func (c WaveClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { return nil } -func (c WaveClient) Delete(ctx context.Context, path string) error { +func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) error { err := filestore.WFS.DeleteFile(ctx, wshrpc.RpcContext.BlockId, path) if err != nil { return fmt.Errorf("error deleting blockfile: %w", err) @@ -102,10 +105,10 @@ func (c WaveClient) Delete(ctx context.Context, path string) error { } func (c WaveClient) GetConnectionType() string { - return remote.ConnectionTypeWave + return connparse.ConnectionTypeWave } -func parseFilePath(path string) (zoneId, fileName string, err error) { +func parseFilePath(conn *connparse.Connection) (zoneId, fileName string, err error) { parts := strings.SplitN(path, "/", 2) if len(parts) < 2 { return "", "", fmt.Errorf("invalid path: %s", path) diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 47d789d87..5c2828fa1 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -11,30 +11,24 @@ import ( "fmt" "io" - "github.com/wavetermdev/waveterm/pkg/remote" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" - "github.com/wavetermdev/waveterm/pkg/wshrpc/wshserver" - "github.com/wavetermdev/waveterm/pkg/wshutil" ) -type WshClient struct { - connRoute string -} +type WshClient struct{} var _ fstype.FileShareClient = WshClient{} -func NewWshClient(connection string) *WshClient { - return &WshClient{ - connRoute: wshutil.MakeConnectionRouteId(connection), - } +func NewWshClient() *WshClient { + return &WshClient{} } -func (c WshClient) Read(ctx context.Context, path string) (*fstype.FullFile, error) { - client := wshserver.GetMainRpcClient() - streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: c.connRoute}) +func (c WshClient) Read(ctx context.Context, conn *connparse.Connection) (*fstype.FullFile, error) { + client := wshclient.GetBareRpcClient() + streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path} + // rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: conn.}) fullFile := &fstype.FullFile{} firstPk := true isDir := false @@ -86,36 +80,36 @@ func (c WshClient) Read(ctx context.Context, path string) (*fstype.FullFile, err return fullFile, nil } -func (c WshClient) Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { - client := wshserver.GetMainRpcClient() +func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { + client := wshclient.GetBareRpcClient() return wshclient.RemoteFileInfoCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) PutFile(ctx context.Context, data wshrpc.FileData) error { - client := wshserver.GetMainRpcClient() - writeData := wshrpc.CommandRemoteWriteFileData{Path: data.Path, Data64: data.Data64} +func (c WshClient) PutFile(ctx context.Context, data fstype.FileData) error { + client := wshclient.GetBareRpcClient() + writeData := wshrpc.CommandRemoteWriteFileData{Path: data.Conn.Path, Data64: data.Data64} return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) Mkdir(ctx context.Context, path string) error { - client := wshserver.GetMainRpcClient() +func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { + client := wshclient.GetBareRpcClient() return wshclient.RemoteMkdirCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) Move(ctx context.Context, srcPath, destPath string, recursive bool) error { - client := wshserver.GetMainRpcClient() - return wshclient.RemoteFileRenameCommand(client, [2]string{srcPath, destPath}, &wshrpc.RpcOpts{Route: c.connRoute}) +func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { + client := wshclient.GetBareRpcClient() + return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn, destPath}, &wshrpc.RpcOpts{Route: c.connRoute}) } -func (c WshClient) Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { +func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { return nil } -func (c WshClient) Delete(ctx context.Context, path string) error { - client := wshserver.GetMainRpcClient() +func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { + client := wshclient.GetBareRpcClient() return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) } func (c WshClient) GetConnectionType() string { - return remote.ConnectionTypeWsh + return connparse.ConnectionTypeWsh } diff --git a/pkg/service/fileservice/fileservice.go b/pkg/service/fileservice/fileservice.go index d447d072b..96e2e9c4d 100644 --- a/pkg/service/fileservice/fileservice.go +++ b/pkg/service/fileservice/fileservice.go @@ -26,11 +26,7 @@ func (fs *FileService) SaveFile_Meta() tsgenmeta.MethodMeta { } func (fs *FileService) SaveFile(ctx context.Context, connection string, path string, data64 string) error { - if connection == "" { - connection = wshrpc.LocalConnName - } - fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.PutFile(ctx, wshrpc.FileData{Path: path, Data64: data64}) + return fileshare.PutFile(ctx, wshrpc.FileData{Path: path, Data64: data64}) } func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta { @@ -41,35 +37,19 @@ func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta { } func (fs *FileService) StatFile(ctx context.Context, connection string, path string) (*wshrpc.FileInfo, error) { - if connection == "" { - connection = wshrpc.LocalConnName - } - fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Stat(ctx, path) + return fileshare.Stat(ctx, path) } func (fs *FileService) Mkdir(ctx context.Context, connection string, path string) error { - if connection == "" { - connection = wshrpc.LocalConnName - } - fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Mkdir(ctx, path) + return fileshare.Mkdir(ctx, path) } func (fs *FileService) TouchFile(ctx context.Context, connection string, path string) error { - if connection == "" { - connection = wshrpc.LocalConnName - } - fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.PutFile(ctx, wshrpc.FileData{Path: path, Data64: ""}) + return fileshare.PutFile(ctx, wshrpc.FileData{Path: path, Data64: ""}) } func (fs *FileService) Rename(ctx context.Context, connection string, path string, newPath string) error { - if connection == "" { - connection = wshrpc.LocalConnName - } - fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Move(ctx, path, newPath, false) + return fileshare.Move(ctx, path, newPath, false) } func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { @@ -80,11 +60,7 @@ func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { } func (fs *FileService) ReadFile(ctx context.Context, connection string, path string) (*fstype.FullFile, error) { - if connection == "" { - connection = wshrpc.LocalConnName - } - fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Read(ctx, path) + return fileshare.Read(ctx, path) } func (fs *FileService) GetWaveFile(id string, path string) (any, error) { @@ -105,11 +81,7 @@ func (fs *FileService) DeleteFile_Meta() tsgenmeta.MethodMeta { } func (fs *FileService) DeleteFile(ctx context.Context, connection string, path string) error { - if connection == "" { - connection = wshrpc.LocalConnName - } - fsclient := fileshare.CreateFileShareClient(ctx, connection) - return fsclient.Delete(ctx, path) + return fileshare.Delete(ctx, path) } func (fs *FileService) GetFullConfig() wconfig.FullConfigType { From d59497a24e289b6ff30eb332830791ea3b2f75b0 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 14:37:56 -0800 Subject: [PATCH 014/126] add back parseprofiles, make more modular --- pkg/remote/awsconn/awsconn.go | 85 +++++++++++++------------- pkg/remote/connparse/connparse.go | 4 +- pkg/remote/connparse/connparse_test.go | 18 +++--- pkg/remote/connutil.go | 18 ++++++ pkg/remote/fileshare/fileshare.go | 4 +- pkg/remote/fileshare/wavefs/wavefs.go | 29 ++++----- pkg/remote/fileshare/wshfs/wshfs.go | 15 ++--- pkg/wconfig/settingsconfig.go | 1 + 8 files changed, 94 insertions(+), 80 deletions(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index dc06ae07d..f1e756945 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -11,44 +11,53 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" - "github.com/wavetermdev/waveterm/pkg/util/iterfn" "github.com/wavetermdev/waveterm/pkg/wconfig" "gopkg.in/ini.v1" ) +const ( + ProfileConfigKey = "profile:config" + ProfilePrefix = "aws:" + TempFilePattern = "waveterm-awsconfig-%s" +) + var connectionRe = regexp.MustCompile(`^aws:\/\/(.*)$`) var tempfiles map[string]string = make(map[string]string) -func GetConfigForConnection(ctx context.Context, connection string) (*aws.Config, error) { - connMatch := connectionRe.FindStringSubmatch(connection) - if connMatch == nil { - return nil, fmt.Errorf("invalid connection string: %s)", connection) - } - connection = connMatch[1] - connfile, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ConnectionsFile) - if len(cerrs) > 0 { - return nil, fmt.Errorf("error reading config file: %v", cerrs[0]) - } +func GetConfig(ctx context.Context, profile string) (*aws.Config, error) { optfns := []func(*config.LoadOptions) error{} - if connfile[connection] != nil { - connectionconfig := connfile.GetMap(connection) - if connectionconfig["aws:config"] != "" { - var tempfile string - if tempfiles[connection] != "" { - tempfile = tempfiles[connection] - } else { - awsConfig := connectionconfig.GetString("aws:config", "") - tempfile, err := os.CreateTemp("", fmt.Sprintf("waveterm-awsconfig-%s", connection)) - if err != nil { - return nil, fmt.Errorf("error creating temp file: %v", err) + // If profile is empty, use default config + if profile != "" { + connMatch := connectionRe.FindStringSubmatch(profile) + if connMatch == nil { + return nil, fmt.Errorf("invalid connection string: %s)", profile) + } + profile = connMatch[1] + profiles, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ProfilesFile) + if len(cerrs) > 0 { + return nil, fmt.Errorf("error reading config file: %v", cerrs[0]) + } + if profiles[profile] != nil { + connectionconfig := profiles.GetMap(profile) + if connectionconfig[ProfileConfigKey] != "" { + var tempfile string + if tempfiles[profile] != "" { + tempfile = tempfiles[profile] + } else { + awsConfig := connectionconfig.GetString(ProfileConfigKey, "") + tempfile, err := os.CreateTemp("", fmt.Sprintf(TempFilePattern, profile)) + if err != nil { + return nil, fmt.Errorf("error creating temp file: %v", err) + } + tempfile.WriteString(awsConfig) } - tempfile.WriteString(awsConfig) + optfns = append(optfns, config.WithSharedCredentialsFiles([]string{tempfile})) } - optfns = append(optfns, config.WithSharedCredentialsFiles([]string{tempfile})) } + trimmedProfile := strings.TrimPrefix(profile, ProfilePrefix) + optfns = append(optfns, config.WithSharedConfigProfile(trimmedProfile)) } - optfns = append(optfns, config.WithSharedConfigProfile(connection)) cfg, err := config.LoadDefaultConfig(ctx, optfns...) if err != nil { return nil, fmt.Errorf("error loading config: %v", err) @@ -56,31 +65,19 @@ func GetConfigForConnection(ctx context.Context, connection string) (*aws.Config return &cfg, nil } -func ParseProfiles() []string { - connfile, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ConnectionsFile) - profiles := map[string]any{} - if len(cerrs) > 0 { - log.Printf("error reading wave connections file: %v", cerrs[0]) - } else { - for k, _ := range connfile { - connMatch := connectionRe.FindStringSubmatch(k) - if connMatch != nil { - profiles[connMatch[1]] = struct{}{} - } - } - } - +func ParseProfiles() map[string]struct{} { + profiles := make(map[string]struct{}) fname := config.DefaultSharedConfigFilename() // Get aws.config default shared configuration file name f, err := ini.Load(fname) // Load ini file if err != nil { log.Printf("error reading aws config file: %v", err) - return iterfn.MapKeysToSorted(profiles) + return nil } for _, v := range f.Sections() { if len(v.Keys()) != 0 { // Get only the sections having Keys parts := strings.Split(v.Name(), " ") if len(parts) == 2 && parts[0] == "profile" { // skip default - profiles[parts[1]] = struct{}{} + profiles[ProfilePrefix+parts[1]] = struct{}{} } } } @@ -89,10 +86,10 @@ func ParseProfiles() []string { f, err = ini.Load(fname) if err != nil { log.Printf("error reading aws credentials file: %v", err) - return iterfn.MapKeysToSorted(profiles) + return profiles } for _, v := range f.Sections() { - profiles[v.Name()] = struct{}{} + profiles[ProfilePrefix+v.Name()] = struct{}{} } - return iterfn.MapKeysToSorted(profiles) + return profiles } diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index 32fcbe2b3..f95ae436b 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -53,12 +53,12 @@ func (c *Connection) GetPathWithHost() string { return c.Host + "/" + c.Path } -func (c *Connection) GetFullPath() string { +func (c *Connection) GetFullURI() string { return c.Scheme + "://" + c.GetPathWithHost() + "?" + c.Params.Encode() } // ParseURI parses a connection URI and returns the connection type, host/path, and parameters. -func ParseURI(uri string, connConfigPath string) (*Connection, error) { +func ParseURI(uri string) (*Connection, error) { split := strings.SplitN(uri, "://", 2) var scheme string var rest string diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index ea813826a..db8fc375c 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -9,7 +9,7 @@ import ( func TestParseURI_BasicWSH(t *testing.T) { t.Parallel() cstr := "wsh://localhost:8080/path/to/file" - c, err := connparse.ParseURI(cstr, "") + c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } @@ -41,7 +41,7 @@ func TestParseURI_BasicWSH(t *testing.T) { func TestParseURI_FullConnectionWSH(t *testing.T) { t.Parallel() cstr := "wsh://user@192.168.0.1:22/path/to/file?foo=bar&baz=qux" - c, err := connparse.ParseURI(cstr, "") + c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } @@ -68,16 +68,16 @@ func TestParseURI_FullConnectionWSH(t *testing.T) { if len(*c.Params) != 2 { t.Fatalf("expected params to have 2 items, got %v", *c.Params) } - got := c.GetFullPath() + got := c.GetFullURI() if len(got) != len(cstr) { - t.Fatalf("expected full path to be %q, got %q (looking at string length since params can be out of order)", cstr, got) + t.Fatalf("expected full uri to be %q, got %q (looking at string length since params can be out of order)", cstr, got) } } func TestParseURI_MissingScheme(t *testing.T) { t.Parallel() cstr := "localhost:8080/path/to/file" - c, err := connparse.ParseURI(cstr, "") + c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } @@ -98,7 +98,7 @@ func TestParseURI_MissingScheme(t *testing.T) { func TestParseURI_WSHShorthand(t *testing.T) { t.Parallel() cstr := "//conn/path/to/file" - c, err := connparse.ParseURI(cstr, "") + c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } @@ -118,7 +118,7 @@ func TestParseURI_WSHShorthand(t *testing.T) { func TestParseURI_WSHLocalHomeShorthand(t *testing.T) { t.Parallel() cstr := "/~/path/to/file" - c, err := connparse.ParseURI(cstr, "") + c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } @@ -138,7 +138,7 @@ func TestParseURI_WSHLocalHomeShorthand(t *testing.T) { func TestParseURI_WSHCurrentAbsolutePath(t *testing.T) { t.Parallel() cstr := t.TempDir() - c, err := connparse.ParseURI(cstr, "") + c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } @@ -159,7 +159,7 @@ func TestParseURI_WSHCurrentAbsolutePath(t *testing.T) { func TestParseURI_BasicS3(t *testing.T) { t.Parallel() cstr := "profile:s3://bucket/path/to/file" - c, err := connparse.ParseURI(cstr, "") + c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } diff --git a/pkg/remote/connutil.go b/pkg/remote/connutil.go index 079593606..ec60c64ee 100644 --- a/pkg/remote/connutil.go +++ b/pkg/remote/connutil.go @@ -17,8 +17,11 @@ import ( "text/template" "github.com/wavetermdev/waveterm/pkg/genconn" + "github.com/wavetermdev/waveterm/pkg/remote/awsconn" + "github.com/wavetermdev/waveterm/pkg/util/iterfn" "github.com/wavetermdev/waveterm/pkg/util/shellutil" "github.com/wavetermdev/waveterm/pkg/wavebase" + "github.com/wavetermdev/waveterm/pkg/wconfig" "golang.org/x/crypto/ssh" ) @@ -282,3 +285,18 @@ func NormalizeConfigPattern(pattern string) string { } return fmt.Sprintf("%s%s%s", userName, pattern, port) } + +func ParseProfiles() []string { + connfile, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ProfilesFile) + if len(cerrs) > 0 { + log.Printf("error reading config file: %v", cerrs[0]) + return nil + } + + awsProfiles := awsconn.ParseProfiles() + for profile := range awsProfiles { + connfile[profile] = struct{}{} + } + + return iterfn.MapKeysToSorted(connfile) +} diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 07d6ca0f5..9cc05a58b 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -24,7 +24,7 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } conntype := conn.GetType() if conntype == connparse.ConnectionTypeS3 { - config, err := awsconn.GetConfigForConnection(ctx, connection) + config, err := awsconn.GetConfig(ctx, connection) if err != nil { log.Printf("error getting aws config: %v", err) return nil, nil @@ -33,7 +33,7 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } else if conntype == connparse.ConnectionTypeWave { return wavefs.NewWaveClient(), conn } else { - return wshfs.NewWshClient(connection), conn + return wshfs.NewWshClient(), conn } } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 9cee085ed..e4d02c181 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -9,7 +9,6 @@ import ( "fmt" "io/fs" "regexp" - "strings" "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/connparse" @@ -42,10 +41,11 @@ func (c WaveClient) PutFile(ctx context.Context, data fstype.FileData) error { if err != nil { return fmt.Errorf("error decoding data64: %w", err) } - zoneId, fileName, err := parseFilePath(data.Path) - if err != nil { - return fmt.Errorf("error parsing path: %w", err) + zoneId := data.Conn.GetParam("zoneid") + if zoneId == "" { + return fmt.Errorf("zoneid not found in connection") } + fileName := data.Conn.GetPathWithHost() if data.At != nil { err = filestore.WFS.WriteAt(ctx, zoneId, fileName, data.At.Offset, dataBuf) if err == fs.ErrNotExist { @@ -88,16 +88,21 @@ func (c WaveClient) Copy(ctx context.Context, srcConn, destConn *connparse.Conne } func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) error { - err := filestore.WFS.DeleteFile(ctx, wshrpc.RpcContext.BlockId, path) + zoneId := conn.GetParam("zoneid") + if zoneId == "" { + return fmt.Errorf("zoneid not found in connection") + } + fileName := conn.GetPathWithHost() + err := filestore.WFS.DeleteFile(ctx, zoneId, fileName) if err != nil { return fmt.Errorf("error deleting blockfile: %w", err) } wps.Broker.Publish(wps.WaveEvent{ Event: wps.Event_BlockFile, - Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, data.ZoneId).String()}, + Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, zoneId).String()}, Data: &wps.WSFileEventData{ - ZoneId: data.ZoneId, - FileName: data.FileName, + ZoneId: zoneId, + FileName: fileName, FileOp: wps.FileOp_Delete, }, }) @@ -107,11 +112,3 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) erro func (c WaveClient) GetConnectionType() string { return connparse.ConnectionTypeWave } - -func parseFilePath(conn *connparse.Connection) (zoneId, fileName string, err error) { - parts := strings.SplitN(path, "/", 2) - if len(parts) < 2 { - return "", "", fmt.Errorf("invalid path: %s", path) - } - return parts[0], parts[1], nil -} diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 5c2828fa1..b265ee5bb 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -15,6 +15,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" + "github.com/wavetermdev/waveterm/pkg/wshutil" ) type WshClient struct{} @@ -28,7 +29,7 @@ func NewWshClient() *WshClient { func (c WshClient) Read(ctx context.Context, conn *connparse.Connection) (*fstype.FullFile, error) { client := wshclient.GetBareRpcClient() streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path} - // rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: conn.}) + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) fullFile := &fstype.FullFile{} firstPk := true isDir := false @@ -70,7 +71,7 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection) (*fstyp if isDir { fiBytes, err := json.Marshal(fileInfoArr) if err != nil { - return nil, fmt.Errorf("unable to serialize files %s", path) + return nil, fmt.Errorf("unable to serialize files %s", conn.Path) } fullFile.Data64 = base64.StdEncoding.EncodeToString(fiBytes) } else { @@ -82,23 +83,23 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection) (*fstyp func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileInfoCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) + return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) PutFile(ctx context.Context, data fstype.FileData) error { client := wshclient.GetBareRpcClient() writeData := wshrpc.CommandRemoteWriteFileData{Path: data.Conn.Path, Data64: data.Data64} - return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: c.connRoute}) + return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(data.Conn.Host)}) } func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteMkdirCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) + return wshclient.RemoteMkdirCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn, destPath}, &wshrpc.RpcOpts{Route: c.connRoute}) + return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) } func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { @@ -107,7 +108,7 @@ func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connec func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileDeleteCommand(client, path, &wshrpc.RpcOpts{Route: c.connRoute}) + return wshclient.RemoteFileDeleteCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) GetConnectionType() string { diff --git a/pkg/wconfig/settingsconfig.go b/pkg/wconfig/settingsconfig.go index 12db5422a..3fcfb2bd8 100644 --- a/pkg/wconfig/settingsconfig.go +++ b/pkg/wconfig/settingsconfig.go @@ -24,6 +24,7 @@ import ( const SettingsFile = "settings.json" const ConnectionsFile = "connections.json" +const ProfilesFile = "profiles.json" const AnySchema = ` { From 1191597cd7e43177850ddd7e6fc7db0a0363ab99 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 14:39:42 -0800 Subject: [PATCH 015/126] generate --- frontend/app/store/wshclientapi.ts | 14 +++---- frontend/types/gotypes.d.ts | 59 ++++++++++-------------------- pkg/wshrpc/wshclient/wshclient.go | 18 ++++----- 3 files changed, 36 insertions(+), 55 deletions(-) diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 2d97a1424..04ebe180e 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -138,7 +138,7 @@ class RpcApiType { } // command "fileappend" [call] - FileAppendCommand(client: WshClient, data: CommandFileData, opts?: RpcOpts): Promise { + FileAppendCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("fileappend", data, opts); } @@ -148,32 +148,32 @@ class RpcApiType { } // command "filecreate" [call] - FileCreateCommand(client: WshClient, data: CommandFileCreateData, opts?: RpcOpts): Promise { + FileCreateCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("filecreate", data, opts); } // command "filedelete" [call] - FileDeleteCommand(client: WshClient, data: CommandFileData, opts?: RpcOpts): Promise { + FileDeleteCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("filedelete", data, opts); } // command "fileinfo" [call] - FileInfoCommand(client: WshClient, data: CommandFileData, opts?: RpcOpts): Promise { + FileInfoCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("fileinfo", data, opts); } // command "filelist" [call] - FileListCommand(client: WshClient, data: CommandFileListData, opts?: RpcOpts): Promise { + FileListCommand(client: WshClient, data: CommandFileListData, opts?: RpcOpts): Promise { return client.wshRpcCall("filelist", data, opts); } // command "fileread" [call] - FileReadCommand(client: WshClient, data: CommandFileData, opts?: RpcOpts): Promise { + FileReadCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("fileread", data, opts); } // command "filewrite" [call] - FileWriteCommand(client: WshClient, data: CommandFileData, opts?: RpcOpts): Promise { + FileWriteCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("filewrite", data, opts); } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index befec4888..49a8a7812 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -74,7 +74,7 @@ declare global { tabid: string; workspaceid: string; block: Block; - files: WaveFile[]; + files: FileInfo[]; }; // webcmd.BlockInputWSCommand @@ -171,32 +171,9 @@ declare global { maxitems: number; }; - // wshrpc.CommandFileCreateData - type CommandFileCreateData = { - zoneid: string; - filename: string; - meta?: {[key: string]: any}; - opts?: FileOptsType; - }; - - // wshrpc.CommandFileData - type CommandFileData = { - zoneid: string; - filename: string; - data64?: string; - at?: CommandFileDataAt; - }; - - // wshrpc.CommandFileDataAt - type CommandFileDataAt = { - offset: number; - size?: number; - }; - // wshrpc.CommandFileListData type CommandFileListData = { - zoneid: string; - prefix?: string; + path: string; all?: boolean; offset?: number; limit?: number; @@ -364,6 +341,20 @@ declare global { height: number; }; + // wshrpc.FileData + type FileData = { + path: string; + data64?: string; + opts?: FileOptsType; + at?: FileDataAt; + }; + + // wshrpc.FileDataAt + type FileDataAt = { + offset: number; + size?: number; + }; + // waveobj.FileDef type FileDef = { content?: string; @@ -376,7 +367,9 @@ declare global { dir: string; name: string; notfound?: boolean; + opts?: FileOptsType; size: number; + meta?: {[key: string]: any}; mode: number; modestr: string; modtime: number; @@ -385,7 +378,7 @@ declare global { readonly?: boolean; }; - // filestore.FileOptsType + // wshrpc.FileOptsType type FileOptsType = { maxsize?: number; circular?: boolean; @@ -405,7 +398,7 @@ declare global { configerrors: ConfigError[]; }; - // fileshare.FullFile + // fstype.FullFile type FullFile = { info: FileInfo; data64: string; @@ -1060,18 +1053,6 @@ declare global { meta: {[key: string]: any}; }; - // wshrpc.WaveFileInfo - type WaveFileInfo = { - zoneid: string; - name: string; - opts?: FileOptsType; - size?: number; - createdts?: number; - modts?: number; - meta?: {[key: string]: any}; - isdir?: boolean; - }; - // wshrpc.WaveInfoData type WaveInfoData = { version: string; diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index da524bf5a..cb0bccba1 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -170,7 +170,7 @@ func EventUnsubAllCommand(w *wshutil.WshRpc, opts *wshrpc.RpcOpts) error { } // command "fileappend", wshserver.FileAppendCommand -func FileAppendCommand(w *wshutil.WshRpc, data wshrpc.CommandFileData, opts *wshrpc.RpcOpts) error { +func FileAppendCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "fileappend", data, opts) return err } @@ -182,37 +182,37 @@ func FileAppendIJsonCommand(w *wshutil.WshRpc, data wshrpc.CommandAppendIJsonDat } // command "filecreate", wshserver.FileCreateCommand -func FileCreateCommand(w *wshutil.WshRpc, data wshrpc.CommandFileCreateData, opts *wshrpc.RpcOpts) error { +func FileCreateCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "filecreate", data, opts) return err } // command "filedelete", wshserver.FileDeleteCommand -func FileDeleteCommand(w *wshutil.WshRpc, data wshrpc.CommandFileData, opts *wshrpc.RpcOpts) error { +func FileDeleteCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "filedelete", data, opts) return err } // command "fileinfo", wshserver.FileInfoCommand -func FileInfoCommand(w *wshutil.WshRpc, data wshrpc.CommandFileData, opts *wshrpc.RpcOpts) (*wshrpc.WaveFileInfo, error) { - resp, err := sendRpcRequestCallHelper[*wshrpc.WaveFileInfo](w, "fileinfo", data, opts) +func FileInfoCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) (*wshrpc.FileInfo, error) { + resp, err := sendRpcRequestCallHelper[*wshrpc.FileInfo](w, "fileinfo", data, opts) return resp, err } // command "filelist", wshserver.FileListCommand -func FileListCommand(w *wshutil.WshRpc, data wshrpc.CommandFileListData, opts *wshrpc.RpcOpts) ([]*wshrpc.WaveFileInfo, error) { - resp, err := sendRpcRequestCallHelper[[]*wshrpc.WaveFileInfo](w, "filelist", data, opts) +func FileListCommand(w *wshutil.WshRpc, data wshrpc.CommandFileListData, opts *wshrpc.RpcOpts) ([]*wshrpc.FileInfo, error) { + resp, err := sendRpcRequestCallHelper[[]*wshrpc.FileInfo](w, "filelist", data, opts) return resp, err } // command "fileread", wshserver.FileReadCommand -func FileReadCommand(w *wshutil.WshRpc, data wshrpc.CommandFileData, opts *wshrpc.RpcOpts) (string, error) { +func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) (string, error) { resp, err := sendRpcRequestCallHelper[string](w, "fileread", data, opts) return resp, err } // command "filewrite", wshserver.FileWriteCommand -func FileWriteCommand(w *wshutil.WshRpc, data wshrpc.CommandFileData, opts *wshrpc.RpcOpts) error { +func FileWriteCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "filewrite", data, opts) return err } From 1d2f1fe79e81d7b06d1b27d44d0d51fbc7a94ed5 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 15:14:02 -0800 Subject: [PATCH 016/126] save --- pkg/remote/connparse/connparse.go | 7 +- pkg/remote/fileshare/fstype/fstype.go | 2 +- pkg/remote/fileshare/s3fs/s3fs.go | 2 +- pkg/remote/fileshare/wavefs/wavefs.go | 117 ++++++++++++++++++++++++-- pkg/remote/fileshare/wshfs/wshfs.go | 8 +- pkg/wshrpc/wshserver/wshserver.go | 117 ++------------------------ 6 files changed, 125 insertions(+), 128 deletions(-) diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index f95ae436b..798540716 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -4,7 +4,6 @@ package connparse import ( - "log" "net/url" "os" "strings" @@ -92,6 +91,7 @@ func ParseURI(uri string) (*Connection, error) { host = "current" path = rest } else { + // trick url parser into parsing the rest of the string as a URL parsedUrl, err := url.Parse("http://" + rest) if err != nil { return nil, err @@ -100,15 +100,10 @@ func ParseURI(uri string) (*Connection, error) { if parsedUrl.User != nil { host = parsedUrl.User.Username() + "@" + host } - log.Printf("parsedUrl: %v", parsedUrl) - log.Printf("parsedUrl.Host: %v", parsedUrl.Host) - log.Printf("parsedUrl.Path: %v", parsedUrl.Path) - log.Printf("parsedUrl.User: %v", parsedUrl.User) params = parsedUrl.Query() path = parsedUrl.Path } - log.Printf("scheme: %v", scheme) return &Connection{ Scheme: scheme, Host: host, diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 46d9e4395..6322c3db4 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -23,7 +23,7 @@ type FileShareClient interface { // Stat returns the file info at the given parsed connection path Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo - Read(ctx context.Context, conn *connparse.Connection) (*FullFile, error) + Read(ctx context.Context, data FileData) (*FullFile, error) // PutFile writes the given data to the file at the given path PutFile(ctx context.Context, data FileData) error // Mkdir creates a directory at the given path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 51728212d..28b410ed4 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -25,7 +25,7 @@ func NewS3Client(config *aws.Config) *S3Client { } } -func (c S3Client) Read(ctx context.Context, conn *connparse.Connection) (*fstype.FullFile, error) { +func (c S3Client) Read(ctx context.Context, data fstype.FileData) (*fstype.FullFile, error) { return nil, nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index e4d02c181..5fe605d4e 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -8,7 +8,7 @@ import ( "encoding/base64" "fmt" "io/fs" - "regexp" + "strings" "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/connparse" @@ -22,18 +22,111 @@ type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} -var wavefilePathRe = regexp.MustCompile(`^wavefile:\/\/([^?]+)(?:\?(?:([^=]+)=([^&]+))(?:&([^=]+)=([^&]+))*)$`) - func NewWaveClient() *WaveClient { return &WaveClient{} } -func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection) (*fstype.FullFile, error) { - return nil, nil +func (c WaveClient) Read(ctx context.Context, data fstype.FileData) (*fstype.FullFile, error) { + zoneId := data.Conn.GetParam("zoneid") + if zoneId == "" { + return nil, fmt.Errorf("zoneid not found in connection") + } + fileName := data.Conn.GetPathWithHost() + if data.At != nil { + _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, data.At.Size) + if err == nil { + return &fstype.FullFile{Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil + } else if err == fs.ErrNotExist { + return nil, fmt.Errorf("NOTFOUND: %w", err) + } else { + return nil, fmt.Errorf("error reading blockfile: %w", err) + } + } else { + _, dataBuf, err := filestore.WFS.ReadFile(ctx, zoneId, fileName) + if err == nil { + return &fstype.FullFile{Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil + } else if err != fs.ErrNotExist { + return nil, fmt.Errorf("error reading blockfile: %w", err) + } + } + prefix := fileName + fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) + if err != nil { + return nil, fmt.Errorf("error listing blockfiles: %w", err) + } + var fileList []*filestore.WaveFile + for _, wf := range fileListOrig { + fileList = append(fileList, wf) + } + if prefix != "" { + var filteredList []*filestore.WaveFile + for _, file := range fileList { + if strings.HasPrefix(file.Name, prefix) { + filteredList = append(filteredList, file) + } + } + fileList = filteredList + } + if !data.All { + var filteredList []*wshrpc.FileInfo + dirMap := make(map[string]int64) // the value is max modtime + for _, file := range fileList { + // if there is an extra "/" after the prefix, don't include it + // first strip the prefix + relPath := strings.TrimPrefix(file.Name, prefix) + // then check if there is a "/" after the prefix + if strings.Contains(relPath, "/") { + dirPath := strings.Split(relPath, "/")[0] + modTime := dirMap[dirPath] + if file.ModTs > modTime { + dirMap[dirPath] = file.ModTs + } + continue + } + filteredList = append(filteredList, waveFileToFileInfo(file)) + } + for dir := range dirMap { + filteredList = append(filteredList, &wshrpc.FileInfo{ + ZoneId: data.ZoneId, + Name: data.Prefix + dir + "/", + Size: 0, + Meta: nil, + ModTs: dirMap[dir], + CreatedTs: dirMap[dir], + IsDir: true, + }) + } + fileList = filteredList + } + if data.Offset > 0 { + if data.Offset >= len(fileList) { + fileList = nil + } else { + fileList = fileList[data.Offset:] + } + } + if data.Limit > 0 { + if data.Limit < len(fileList) { + fileList = fileList[:data.Limit] + } + } + return fileList, nil } func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { - return nil, nil + zoneId := conn.GetParam("zoneid") + if zoneId == "" { + return nil, fmt.Errorf("zoneid not found in connection") + } + fileName := conn.GetPathWithHost() + fileInfo, err := filestore.WFS.Stat(ctx, zoneId, fileName) + if err != nil { + if err == fs.ErrNotExist { + return nil, fmt.Errorf("NOTFOUND: %w", err) + } + return nil, fmt.Errorf("error getting file info: %w", err) + } + return waveFileToFileInfo(fileInfo), nil } func (c WaveClient) PutFile(ctx context.Context, data fstype.FileData) error { @@ -75,6 +168,7 @@ func (c WaveClient) PutFile(ctx context.Context, data fstype.FileData) error { return nil } +// WaveFile does not support directories, only prefix-based listing func (c WaveClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { return nil } @@ -112,3 +206,14 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) erro func (c WaveClient) GetConnectionType() string { return connparse.ConnectionTypeWave } + +func waveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { + path := fmt.Sprintf("wavefile://%s?zoneid=%s", wf.Name, wf.ZoneId) + return &wshrpc.FileInfo{ + Path: path, + Name: wf.Name, + Opts: wf.Opts, + Size: wf.Size, + Meta: wf.Meta, + } +} diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index b265ee5bb..ad8847167 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -26,10 +26,10 @@ func NewWshClient() *WshClient { return &WshClient{} } -func (c WshClient) Read(ctx context.Context, conn *connparse.Connection) (*fstype.FullFile, error) { +func (c WshClient) Read(ctx context.Context, data fstype.FileData) (*fstype.FullFile, error) { client := wshclient.GetBareRpcClient() - streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path} - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + streamFileData := wshrpc.CommandRemoteStreamFileData{Path: data.Conn.Path} + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(data.Conn.Host)}) fullFile := &fstype.FullFile{} firstPk := true isDir := false @@ -71,7 +71,7 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection) (*fstyp if isDir { fiBytes, err := json.Marshal(fileInfoArr) if err != nil { - return nil, fmt.Errorf("unable to serialize files %s", conn.Path) + return nil, fmt.Errorf("unable to serialize files %s", data.Conn.Path) } fullFile.Data64 = base64.StdEncoding.EncodeToString(fiBytes) } else { diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 928bf4130..be38adb98 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -281,8 +281,7 @@ func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.FileCrea if data.Opts != nil { fileOpts = *data.Opts } - client := fileshare.CreateFileShareClient(ctx, remote.ConnectionTypeWave) - err := client.PutFile(ctx, wshrpc.FileData{ + err := fileshare.PutFile(ctx, wshrpc.FileData{ Path: data.Path, Data64: "", Opts: &fileOpts, @@ -294,99 +293,15 @@ func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.FileCrea } func (ws *WshServer) FileDeleteCommand(ctx context.Context, data wshrpc.FileData) error { - client := fileshare.CreateFileShareClient(ctx, remote.ConnectionTypeWave) - err := client.Delete(ctx, data.Path) - if err != nil { - return fmt.Errorf("error deleting file: %w", err) - } - return nil + return fileshare.Delete(ctx, data.Path) } -func waveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { - return &wshrpc.FileInfo{ - ZoneId: wf.ZoneId, - Name: wf.Name, - Opts: wf.Opts, - Size: wf.Size, - CreatedTs: wf.CreatedTs, - ModTs: wf.ModTs, - Meta: wf.Meta, - } -} - -func (ws *WshServer) FileInfoCommand(ctx context.Context, data wshrpc.CommandFileData) (*wshrpc.FileInfo, error) { - fileInfo, err := filestore.WFS.Stat(ctx, data.ZoneId, data.FileName) - if err != nil { - if err == fs.ErrNotExist { - return nil, fmt.Errorf("NOTFOUND: %w", err) - } - return nil, fmt.Errorf("error getting file info: %w", err) - } - return waveFileToFileInfo(fileInfo), nil +func (ws *WshServer) FileInfoCommand(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileInfo, error) { + return fileshare.Stat(ctx, data.Path) } func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.CommandFileListData) ([]*wshrpc.FileInfo, error) { - fileListOrig, err := filestore.WFS.ListFiles(ctx, data.ZoneId) - if err != nil { - return nil, fmt.Errorf("error listing blockfiles: %w", err) - } - var fileList []*wshrpc.FileInfo - for _, wf := range fileListOrig { - fileList = append(fileList, waveFileToFileInfo(wf)) - } - if data.Prefix != "" { - var filteredList []*wshrpc.FileInfo - for _, file := range fileList { - if strings.HasPrefix(file.Name, data.Prefix) { - filteredList = append(filteredList, file) - } - } - fileList = filteredList - } - if !data.All { - var filteredList []*wshrpc.FileInfo - dirMap := make(map[string]int64) // the value is max modtime - for _, file := range fileList { - // if there is an extra "/" after the prefix, don't include it - // first strip the prefix - relPath := strings.TrimPrefix(file.Name, data.Prefix) - // then check if there is a "/" after the prefix - if strings.Contains(relPath, "/") { - dirPath := strings.Split(relPath, "/")[0] - modTime := dirMap[dirPath] - if file.ModTs > modTime { - dirMap[dirPath] = file.ModTs - } - continue - } - filteredList = append(filteredList, file) - } - for dir := range dirMap { - filteredList = append(filteredList, &wshrpc.FileInfo{ - ZoneId: data.ZoneId, - Name: data.Prefix + dir + "/", - Size: 0, - Meta: nil, - ModTs: dirMap[dir], - CreatedTs: dirMap[dir], - IsDir: true, - }) - } - fileList = filteredList - } - if data.Offset > 0 { - if data.Offset >= len(fileList) { - fileList = nil - } else { - fileList = fileList[data.Offset:] - } - } - if data.Limit > 0 { - if data.Limit < len(fileList) { - fileList = fileList[:data.Limit] - } - } - return fileList, nil + return fileshare.Read(ctx, data.Path) } func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.CommandFileData) error { @@ -423,26 +338,8 @@ func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.CommandFi return nil } -func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.CommandFileData) (string, error) { - if data.At != nil { - _, dataBuf, err := filestore.WFS.ReadAt(ctx, data.ZoneId, data.FileName, data.At.Offset, data.At.Size) - if err == fs.ErrNotExist { - return "", fmt.Errorf("NOTFOUND: %w", err) - } - if err != nil { - return "", fmt.Errorf("error reading blockfile: %w", err) - } - return base64.StdEncoding.EncodeToString(dataBuf), nil - } else { - _, dataBuf, err := filestore.WFS.ReadFile(ctx, data.ZoneId, data.FileName) - if err == fs.ErrNotExist { - return "", fmt.Errorf("NOTFOUND: %w", err) - } - if err != nil { - return "", fmt.Errorf("error reading blockfile: %w", err) - } - return base64.StdEncoding.EncodeToString(dataBuf), nil - } +func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) (string, error) { + } func (ws *WshServer) FileAppendCommand(ctx context.Context, data wshrpc.CommandFileData) error { From 3dd5070cb1d63b4a6d51968cab82a9aa2871b705 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 18:07:35 -0800 Subject: [PATCH 017/126] save --- pkg/remote/connparse/connparse.go | 31 +++------ pkg/remote/connparse/connparse_test.go | 15 +---- pkg/remote/fileshare/fstype/fstype.go | 12 +--- pkg/remote/fileshare/wavefs/wavefs.go | 89 +++++++++++++++----------- pkg/remote/fileshare/wshfs/wshfs.go | 33 ++++++---- pkg/wshrpc/wshclient/wshclient.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 75 +++++++++++++++++++++- pkg/wshrpc/wshrpctypes.go | 86 +++++++++++++++---------- pkg/wshrpc/wshserver/wshserver.go | 59 +---------------- 9 files changed, 211 insertions(+), 191 deletions(-) diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index 798540716..c1f2323e6 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -4,7 +4,6 @@ package connparse import ( - "net/url" "os" "strings" ) @@ -19,15 +18,6 @@ type Connection struct { Scheme string Host string Path string - Params *url.Values -} - -func (c *Connection) GetParam(key string) string { - return c.Params.Get(key) -} - -func (c *Connection) SetParam(key, value string) { - c.Params.Set(key, value) } func (c *Connection) GetSchemeParts() []string { @@ -53,7 +43,7 @@ func (c *Connection) GetPathWithHost() string { } func (c *Connection) GetFullURI() string { - return c.Scheme + "://" + c.GetPathWithHost() + "?" + c.Params.Encode() + return c.Scheme + "://" + c.GetPathWithHost() } // ParseURI parses a connection URI and returns the connection type, host/path, and parameters. @@ -73,7 +63,6 @@ func ParseURI(uri string) (*Connection, error) { var host string var path string - var params url.Values if strings.HasPrefix(rest, "//") { rest = strings.TrimPrefix(rest, "//") split = strings.SplitN(rest, "/", 2) @@ -91,23 +80,19 @@ func ParseURI(uri string) (*Connection, error) { host = "current" path = rest } else { - // trick url parser into parsing the rest of the string as a URL - parsedUrl, err := url.Parse("http://" + rest) - if err != nil { - return nil, err - } - host = parsedUrl.Host - if parsedUrl.User != nil { - host = parsedUrl.User.Username() + "@" + host + split = strings.SplitN(rest, "/", 2) + if len(split) > 1 { + host = split[0] + path = "/" + split[1] + } else { + host = split[0] + path = "/" } - params = parsedUrl.Query() - path = parsedUrl.Path } return &Connection{ Scheme: scheme, Host: host, Path: path, - Params: ¶ms, }, nil } diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index db8fc375c..ca697f823 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -33,14 +33,11 @@ func TestParseURI_BasicWSH(t *testing.T) { if len(c.GetSchemeParts()) != 1 { t.Fatalf("expected scheme parts to be 1, got %d", len(c.GetSchemeParts())) } - if len(*c.Params) != 0 { - t.Fatalf("expected params to be empty, got %v", *c.Params) - } } func TestParseURI_FullConnectionWSH(t *testing.T) { t.Parallel() - cstr := "wsh://user@192.168.0.1:22/path/to/file?foo=bar&baz=qux" + cstr := "wsh://user@192.168.0.1:22/path/to/file" c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) @@ -65,12 +62,9 @@ func TestParseURI_FullConnectionWSH(t *testing.T) { if len(c.GetSchemeParts()) != 1 { t.Fatalf("expected scheme parts to be 1, got %d", len(c.GetSchemeParts())) } - if len(*c.Params) != 2 { - t.Fatalf("expected params to have 2 items, got %v", *c.Params) - } got := c.GetFullURI() - if len(got) != len(cstr) { - t.Fatalf("expected full uri to be %q, got %q (looking at string length since params can be out of order)", cstr, got) + if got != cstr { + t.Fatalf("expected full URI to be %q, got %q", cstr, got) } } @@ -183,7 +177,4 @@ func TestParseURI_BasicS3(t *testing.T) { if len(c.GetSchemeParts()) != 2 { t.Fatalf("expected scheme parts to be 2, got %d", len(c.GetSchemeParts())) } - if len(*c.Params) != 0 { - t.Fatalf("expected params to be empty, got %v", *c.Params) - } } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 6322c3db4..fb7d8181b 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -12,20 +12,14 @@ type FullFile struct { Data64 string `json:"data64"` // base64 encoded } -type FileData struct { - Conn *connparse.Connection - Data64 string - Opts *wshrpc.FileOptsType - At *wshrpc.FileDataAt // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) -} - type FileShareClient interface { // Stat returns the file info at the given parsed connection path Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo - Read(ctx context.Context, data FileData) (*FullFile, error) + Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) + ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) // PutFile writes the given data to the file at the given path - PutFile(ctx context.Context, data FileData) error + PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error // Mkdir creates a directory at the given path Mkdir(ctx context.Context, conn *connparse.Connection) error // Move moves the file from srcPath to destPath diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 5fe605d4e..c4e357481 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -18,6 +18,10 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshrpc" ) +const ( + WaveFilePathPattern = "wavefile://%s/%s" +) + type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -26,16 +30,16 @@ func NewWaveClient() *WaveClient { return &WaveClient{} } -func (c WaveClient) Read(ctx context.Context, data fstype.FileData) (*fstype.FullFile, error) { - zoneId := data.Conn.GetParam("zoneid") +func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { + zoneId := conn.Host if zoneId == "" { return nil, fmt.Errorf("zoneid not found in connection") } - fileName := data.Conn.GetPathWithHost() + fileName := conn.Path if data.At != nil { _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, data.At.Size) if err == nil { - return &fstype.FullFile{Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil + return &wshrpc.FileData{Info: data.Info, Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil } else if err == fs.ErrNotExist { return nil, fmt.Errorf("NOTFOUND: %w", err) } else { @@ -44,22 +48,35 @@ func (c WaveClient) Read(ctx context.Context, data fstype.FileData) (*fstype.Ful } else { _, dataBuf, err := filestore.WFS.ReadFile(ctx, zoneId, fileName) if err == nil { - return &fstype.FullFile{Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil + return &wshrpc.FileData{Info: data.Info, Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil } else if err != fs.ErrNotExist { return nil, fmt.Errorf("error reading blockfile: %w", err) } } + list, err := c.ListEntries(ctx, conn, nil) + if err != nil { + return nil, fmt.Errorf("error listing blockfiles: %w", err) + } + return &wshrpc.FileData{Info: data.Info, Entries: list}, nil +} + +func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { + zoneId := conn.Host + if zoneId == "" { + return nil, fmt.Errorf("zoneid not found in connection") + } + fileName := conn.Path prefix := fileName fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) if err != nil { return nil, fmt.Errorf("error listing blockfiles: %w", err) } - var fileList []*filestore.WaveFile + var fileList []*wshrpc.FileInfo for _, wf := range fileListOrig { - fileList = append(fileList, wf) + fileList = append(fileList, waveFileToFileInfo(wf)) } if prefix != "" { - var filteredList []*filestore.WaveFile + var filteredList []*wshrpc.FileInfo for _, file := range fileList { if strings.HasPrefix(file.Name, prefix) { filteredList = append(filteredList, file) @@ -67,9 +84,9 @@ func (c WaveClient) Read(ctx context.Context, data fstype.FileData) (*fstype.Ful } fileList = filteredList } - if !data.All { + if !opts.All { var filteredList []*wshrpc.FileInfo - dirMap := make(map[string]int64) // the value is max modtime + dirMap := make(map[string]any) // the value is max modtime for _, file := range fileList { // if there is an extra "/" after the prefix, don't include it // first strip the prefix @@ -77,48 +94,44 @@ func (c WaveClient) Read(ctx context.Context, data fstype.FileData) (*fstype.Ful // then check if there is a "/" after the prefix if strings.Contains(relPath, "/") { dirPath := strings.Split(relPath, "/")[0] - modTime := dirMap[dirPath] - if file.ModTs > modTime { - dirMap[dirPath] = file.ModTs - } + dirMap[dirPath] = struct{}{} continue } - filteredList = append(filteredList, waveFileToFileInfo(file)) + filteredList = append(filteredList, file) } for dir := range dirMap { + dirName := prefix + dir + "/" filteredList = append(filteredList, &wshrpc.FileInfo{ - ZoneId: data.ZoneId, - Name: data.Prefix + dir + "/", - Size: 0, - Meta: nil, - ModTs: dirMap[dir], - CreatedTs: dirMap[dir], - IsDir: true, + Path: fmt.Sprintf(WaveFilePathPattern, zoneId, dirName), + Name: dirName, + Dir: dirName, + Size: 0, + IsDir: true, }) } fileList = filteredList } - if data.Offset > 0 { - if data.Offset >= len(fileList) { + if opts.Offset > 0 { + if opts.Offset >= len(fileList) { fileList = nil } else { - fileList = fileList[data.Offset:] + fileList = fileList[opts.Offset:] } } - if data.Limit > 0 { - if data.Limit < len(fileList) { - fileList = fileList[:data.Limit] + if opts.Limit > 0 { + if opts.Limit < len(fileList) { + fileList = fileList[:opts.Limit] } } return fileList, nil } func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { - zoneId := conn.GetParam("zoneid") + zoneId := conn.Host if zoneId == "" { return nil, fmt.Errorf("zoneid not found in connection") } - fileName := conn.GetPathWithHost() + fileName := conn.Path fileInfo, err := filestore.WFS.Stat(ctx, zoneId, fileName) if err != nil { if err == fs.ErrNotExist { @@ -129,16 +142,16 @@ func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshr return waveFileToFileInfo(fileInfo), nil } -func (c WaveClient) PutFile(ctx context.Context, data fstype.FileData) error { +func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { dataBuf, err := base64.StdEncoding.DecodeString(data.Data64) if err != nil { return fmt.Errorf("error decoding data64: %w", err) } - zoneId := data.Conn.GetParam("zoneid") + zoneId := conn.Host if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - fileName := data.Conn.GetPathWithHost() + fileName := conn.Path if data.At != nil { err = filestore.WFS.WriteAt(ctx, zoneId, fileName, data.At.Offset, dataBuf) if err == fs.ErrNotExist { @@ -182,11 +195,11 @@ func (c WaveClient) Copy(ctx context.Context, srcConn, destConn *connparse.Conne } func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) error { - zoneId := conn.GetParam("zoneid") + zoneId := conn.Host if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - fileName := conn.GetPathWithHost() + fileName := conn.Path err := filestore.WFS.DeleteFile(ctx, zoneId, fileName) if err != nil { return fmt.Errorf("error deleting blockfile: %w", err) @@ -208,12 +221,12 @@ func (c WaveClient) GetConnectionType() string { } func waveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { - path := fmt.Sprintf("wavefile://%s?zoneid=%s", wf.Name, wf.ZoneId) + path := fmt.Sprintf(WaveFilePathPattern, wf.ZoneId, wf.Name) return &wshrpc.FileInfo{ Path: path, Name: wf.Name, - Opts: wf.Opts, + Opts: &wf.Opts, Size: wf.Size, - Meta: wf.Meta, + Meta: &wf.Meta, } } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index ad8847167..39e5720f3 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -7,7 +7,6 @@ import ( "bytes" "context" "encoding/base64" - "encoding/json" "fmt" "io" @@ -26,11 +25,11 @@ func NewWshClient() *WshClient { return &WshClient{} } -func (c WshClient) Read(ctx context.Context, data fstype.FileData) (*fstype.FullFile, error) { +func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { client := wshclient.GetBareRpcClient() - streamFileData := wshrpc.CommandRemoteStreamFileData{Path: data.Conn.Path} - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(data.Conn.Host)}) - fullFile := &fstype.FullFile{} + streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path} + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + fullFile := &wshrpc.FileData{} firstPk := true isDir := false var fileBuf bytes.Buffer @@ -69,11 +68,7 @@ func (c WshClient) Read(ctx context.Context, data fstype.FileData) (*fstype.Full } } if isDir { - fiBytes, err := json.Marshal(fileInfoArr) - if err != nil { - return nil, fmt.Errorf("unable to serialize files %s", data.Conn.Path) - } - fullFile.Data64 = base64.StdEncoding.EncodeToString(fiBytes) + } else { // we can avoid this re-encoding if we ensure the remote side always encodes chunks of 3 bytes so we don't get padding chars fullFile.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) @@ -81,15 +76,27 @@ func (c WshClient) Read(ctx context.Context, data fstype.FileData) (*fstype.Full return fullFile, nil } +func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { + +} + +func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opts *wshrpc.FileListOpts, data *wshrpc.FileData) ([]*wshrpc.FileInfo, error) { + entries := data.Entries + if opts.All { + + } + return entries, nil +} + func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { client := wshclient.GetBareRpcClient() return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } -func (c WshClient) PutFile(ctx context.Context, data fstype.FileData) error { +func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { client := wshclient.GetBareRpcClient() - writeData := wshrpc.CommandRemoteWriteFileData{Path: data.Conn.Path, Data64: data.Data64} - return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(data.Conn.Host)}) + writeData := wshrpc.CommandRemoteWriteFileData{Path: conn.Path, Data64: data.Data64} + return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index cb0bccba1..ebbc275e0 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -200,7 +200,7 @@ func FileInfoCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOp } // command "filelist", wshserver.FileListCommand -func FileListCommand(w *wshutil.WshRpc, data wshrpc.CommandFileListData, opts *wshrpc.RpcOpts) ([]*wshrpc.FileInfo, error) { +func FileListCommand(w *wshutil.WshRpc, data wshrpc.FileListData, opts *wshrpc.RpcOpts) ([]*wshrpc.FileInfo, error) { resp, err := sendRpcRequestCallHelper[[]*wshrpc.FileInfo](w, "filelist", data, opts) return resp, err } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index cd3cd5d39..c9421787c 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -44,8 +44,8 @@ func (impl *ServerImpl) MessageCommand(ctx context.Context, data wshrpc.CommandM return nil } -func respErr(err error) wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData] { - return wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData]{Error: err} +func respErr[T any](err error) wshrpc.RespOrErrorUnion[T] { + return wshrpc.RespOrErrorUnion[T]{Error: err} } type ByteRangeType struct { @@ -199,12 +199,81 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData]{Response: resp} }) if err != nil { - ch <- respErr(err) + ch <- respErr[wshrpc.CommandRemoteStreamFileRtnData](err) } }() return ch } +func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrpc.CommandRemoteListEntriesData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) + go func() { + defer close(ch) + path, err := wavebase.ExpandHomeDir(data.Path) + if err != nil { + ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](err) + return + } + innerFilesEntries := []*fs.DirEntry{} + seen := 0 + if data.Opts.Limit == 0 { + data.Opts.Limit = MaxDirSize + } + err = listFilesInternal(path, innerFilesEntries, &seen, data.Opts) + if err != nil { + ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("cannot list entries in dir %q: %w", path, err)) + return + } + var fileInfoArr []*wshrpc.FileInfo + for _, innerFileEntry := range innerFilesEntries { + if ctx.Err() != nil { + return + } + innerFileInfoInt, err := (*innerFileEntry).Info() + if err != nil { + continue + } + innerFileInfo := statToFileInfo(filepath.Join(path, innerFileInfoInt.Name()), innerFileInfoInt, false) + fileInfoArr = append(fileInfoArr, innerFileInfo) + if len(fileInfoArr) >= DirChunkSize { + resp := wshrpc.CommandRemoteListEntriesRtnData{FileInfo: fileInfoArr} + ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: resp} + fileInfoArr = nil + } + } + if len(fileInfoArr) > 0 { + resp := wshrpc.CommandRemoteListEntriesRtnData{FileInfo: fileInfoArr} + ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: resp} + } + }() + return ch +} + +func listFilesInternal(path string, fileInfoArr []*fs.DirEntry, seen *int, opts *wshrpc.FileListOpts) error { + innerFilesEntries, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("cannot open dir %q: %w", path, err) + } + for _, innerFileEntry := range innerFilesEntries { + if err != nil { + continue + } + if innerFileEntry.IsDir() && opts.All { + err = listFilesInternal(filepath.Join(path, innerFileEntry.Name()), fileInfoArr, seen, opts) + if err != nil { + return err + } + } else { + fileInfoArr = append(fileInfoArr, &innerFileEntry) + } + *seen++ + if *seen >= opts.Offset || *seen >= opts.Offset+opts.Limit { + break + } + } + return nil +} + func statToFileInfo(fullPath string, finfo fs.FileInfo, extended bool) *wshrpc.FileInfo { mimeType := utilfn.DetectMimeType(fullPath, finfo, extended) rtn := &wshrpc.FileInfo{ diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index ea7a1dd8a..3a6c51dcd 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -131,7 +131,7 @@ type WshRpcInterface interface { FileWriteCommand(ctx context.Context, data FileData) error FileReadCommand(ctx context.Context, data FileData) (string, error) FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) - FileListCommand(ctx context.Context, data CommandFileListData) ([]*FileInfo, error) + FileListCommand(ctx context.Context, data FileListData) ([]*FileInfo, error) EventPublishCommand(ctx context.Context, data wps.WaveEvent) error EventSubCommand(ctx context.Context, data wps.SubscriptionRequest) error EventUnsubCommand(ctx context.Context, data string) error @@ -168,6 +168,7 @@ type WshRpcInterface interface { // remotes RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[CommandRemoteStreamFileRtnData] + RemoteListEntriesCommand(ctx context.Context, data CommandRemoteListEntriesData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) RemoteFileTouchCommand(ctx context.Context, path string) error RemoteFileRenameCommand(ctx context.Context, pathTuple [2]string) error @@ -329,17 +330,48 @@ type FileDataAt struct { } type FileData struct { - Path string `json:"path"` - Data64 string `json:"data64,omitempty"` - Opts *FileOptsType `json:"opts,omitempty"` - At *FileDataAt `json:"at,omitempty"` // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) + Info *FileInfo `json:"info,omitempty"` + Data64 string `json:"data64,omitempty"` + Entries []*FileInfo `json:"entries,omitempty"` + Opts *FileOptsType `json:"opts,omitempty"` + At *FileDataAt `json:"at,omitempty"` // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) } -type CommandFileListData struct { - Path string `json:"path"` - All bool `json:"all,omitempty"` - Offset int `json:"offset,omitempty"` - Limit int `json:"limit,omitempty"` +type FileInfo struct { + Path string `json:"path"` // cleaned path (may have "~") + Dir string `json:"dir"` // returns the directory part of the path (if this is a a directory, it will be equal to Path). "~" will be expanded, and separators will be normalized to "/" + Name string `json:"name"` + NotFound bool `json:"notfound,omitempty"` + Opts *FileOptsType `json:"opts,omitempty"` + Size int64 `json:"size"` + Meta *FileMeta `json:"meta,omitempty"` + Mode os.FileMode `json:"mode"` + ModeStr string `json:"modestr"` + ModTime int64 `json:"modtime"` + IsDir bool `json:"isdir,omitempty"` + SupportsMkdir *bool `json:"supportsmkdir,omitempty"` + MimeType string `json:"mimetype,omitempty"` + ReadOnly bool `json:"readonly,omitempty"` // this is not set for fileinfo's returned from directory listings +} + +type FileOptsType struct { + MaxSize int64 `json:"maxsize,omitempty"` + Circular bool `json:"circular,omitempty"` + IJson bool `json:"ijson,omitempty"` + IJsonBudget int `json:"ijsonbudget,omitempty"` +} + +type FileMeta = map[string]any + +type FileListData struct { + Path string `json:"path"` + Opts *FileListOpts `json:"opts,omitempty"` +} + +type FileListOpts struct { + All bool `json:"all,omitempty"` + Offset int `json:"offset,omitempty"` + Limit int `json:"limit,omitempty"` } type FileCreateData struct { @@ -420,31 +452,6 @@ type CpuDataType struct { Value float64 `json:"value"` } -type FileInfo struct { - Path string `json:"path"` // cleaned path (may have "~") - Dir string `json:"dir"` // returns the directory part of the path (if this is a a directory, it will be equal to Path). "~" will be expanded, and separators will be normalized to "/" - Name string `json:"name"` - NotFound bool `json:"notfound,omitempty"` - Opts FileOptsType `json:"opts,omitempty"` - Size int64 `json:"size"` - Meta FileMeta `json:"meta,omitempty"` - Mode os.FileMode `json:"mode"` - ModeStr string `json:"modestr"` - ModTime int64 `json:"modtime"` - IsDir bool `json:"isdir,omitempty"` - MimeType string `json:"mimetype,omitempty"` - ReadOnly bool `json:"readonly,omitempty"` // this is not set for fileinfo's returned from directory listings -} - -type FileOptsType struct { - MaxSize int64 `json:"maxsize,omitempty"` - Circular bool `json:"circular,omitempty"` - IJson bool `json:"ijson,omitempty"` - IJsonBudget int `json:"ijsonbudget,omitempty"` -} - -type FileMeta = map[string]any - type CommandRemoteStreamFileData struct { Path string `json:"path"` ByteRange string `json:"byterange,omitempty"` @@ -455,6 +462,15 @@ type CommandRemoteStreamFileRtnData struct { Data64 string `json:"data64,omitempty"` } +type CommandRemoteListEntriesData struct { + Path string `json:"path"` + Opts *FileListOpts `json:"opts,omitempty"` +} + +type CommandRemoteListEntriesRtnData struct { + FileInfo []*FileInfo `json:"fileinfo,omitempty"` +} + type CommandRemoteWriteFileData struct { Path string `json:"path"` Data64 string `json:"data64"` diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index be38adb98..e8fa0213d 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -304,69 +304,14 @@ func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.CommandFil return fileshare.Read(ctx, data.Path) } -func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.CommandFileData) error { - dataBuf, err := base64.StdEncoding.DecodeString(data.Data64) - if err != nil { - return fmt.Errorf("error decoding data64: %w", err) - } - if data.At != nil { - err = filestore.WFS.WriteAt(ctx, data.ZoneId, data.FileName, data.At.Offset, dataBuf) - if err == fs.ErrNotExist { - return fmt.Errorf("NOTFOUND: %w", err) - } - if err != nil { - return fmt.Errorf("error writing to blockfile: %w", err) - } - } else { - err = filestore.WFS.WriteFile(ctx, data.ZoneId, data.FileName, dataBuf) - if err == fs.ErrNotExist { - return fmt.Errorf("NOTFOUND: %w", err) - } - if err != nil { - return fmt.Errorf("error writing to blockfile: %w", err) - } - } - wps.Broker.Publish(wps.WaveEvent{ - Event: wps.Event_BlockFile, - Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, data.ZoneId).String()}, - Data: &wps.WSFileEventData{ - ZoneId: data.ZoneId, - FileName: data.FileName, - FileOp: wps.FileOp_Invalidate, - }, - }) - return nil +func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.FileData) error { + return fileshare.PutFile(ctx, data) } func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) (string, error) { } -func (ws *WshServer) FileAppendCommand(ctx context.Context, data wshrpc.CommandFileData) error { - dataBuf, err := base64.StdEncoding.DecodeString(data.Data64) - if err != nil { - return fmt.Errorf("error decoding data64: %w", err) - } - err = filestore.WFS.AppendData(ctx, data.ZoneId, data.FileName, dataBuf) - if err == fs.ErrNotExist { - return fmt.Errorf("NOTFOUND: %w", err) - } - if err != nil { - return fmt.Errorf("error appending to blockfile: %w", err) - } - wps.Broker.Publish(wps.WaveEvent{ - Event: wps.Event_BlockFile, - Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, data.ZoneId).String()}, - Data: &wps.WSFileEventData{ - ZoneId: data.ZoneId, - FileName: data.FileName, - FileOp: wps.FileOp_Append, - Data64: base64.StdEncoding.EncodeToString(dataBuf), - }, - }) - return nil -} - func (ws *WshServer) FileAppendIJsonCommand(ctx context.Context, data wshrpc.CommandAppendIJsonData) error { tryCreate := true if data.FileName == blockcontroller.BlockFile_VDom && tryCreate { From 0753a7bd263e57e23bf2aab53b92680f6161f826 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 18:24:06 -0800 Subject: [PATCH 018/126] got listentries implemented --- frontend/app/store/services.ts | 34 ----------------------- frontend/app/store/wshclientapi.ts | 7 ++++- frontend/types/gotypes.d.ts | 42 ++++++++++++++++++----------- pkg/remote/fileshare/fileshare.go | 11 ++++---- pkg/remote/fileshare/s3fs/s3fs.go | 8 ++++-- pkg/remote/fileshare/wshfs/wshfs.go | 35 ++++++++++++++++++++---- pkg/wshrpc/wshclient/wshclient.go | 5 ++++ 7 files changed, 79 insertions(+), 63 deletions(-) diff --git a/frontend/app/store/services.ts b/frontend/app/store/services.ts index 81d4e1925..5211271f6 100644 --- a/frontend/app/store/services.ts +++ b/frontend/app/store/services.ts @@ -49,40 +49,6 @@ export const ClientService = new ClientServiceType(); // fileservice.FileService (file) class FileServiceType { - // delete file - DeleteFile(connection: string, path: string): Promise { - return WOS.callBackendService("file", "DeleteFile", Array.from(arguments)) - } - GetFullConfig(): Promise { - return WOS.callBackendService("file", "GetFullConfig", Array.from(arguments)) - } - GetWaveFile(arg1: string, arg2: string): Promise { - return WOS.callBackendService("file", "GetWaveFile", Array.from(arguments)) - } - Mkdir(arg2: string, arg3: string): Promise { - return WOS.callBackendService("file", "Mkdir", Array.from(arguments)) - } - - // read file - ReadFile(connection: string, path: string): Promise { - return WOS.callBackendService("file", "ReadFile", Array.from(arguments)) - } - Rename(arg2: string, arg3: string, arg4: string): Promise { - return WOS.callBackendService("file", "Rename", Array.from(arguments)) - } - - // save file - SaveFile(connection: string, path: string, data64: string): Promise { - return WOS.callBackendService("file", "SaveFile", Array.from(arguments)) - } - - // get file info - StatFile(connection: string, path: string): Promise { - return WOS.callBackendService("file", "StatFile", Array.from(arguments)) - } - TouchFile(arg2: string, arg3: string): Promise { - return WOS.callBackendService("file", "TouchFile", Array.from(arguments)) - } } export const FileService = new FileServiceType(); diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 04ebe180e..9ec6d36f1 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -163,7 +163,7 @@ class RpcApiType { } // command "filelist" [call] - FileListCommand(client: WshClient, data: CommandFileListData, opts?: RpcOpts): Promise { + FileListCommand(client: WshClient, data: FileListData, opts?: RpcOpts): Promise { return client.wshRpcCall("filelist", data, opts); } @@ -237,6 +237,11 @@ class RpcApiType { return client.wshRpcCall("remotefiletouch", data, opts); } + // command "remotelistentries" [responsestream] + RemoteListEntriesCommand(client: WshClient, data: CommandRemoteListEntriesData, opts?: RpcOpts): AsyncGenerator { + return client.wshRpcStream("remotelistentries", data, opts); + } + // command "remotemkdir" [call] RemoteMkdirCommand(client: WshClient, data: string, opts?: RpcOpts): Promise { return client.wshRpcCall("remotemkdir", data, opts); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 49a8a7812..025fd9d43 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -171,14 +171,6 @@ declare global { maxitems: number; }; - // wshrpc.CommandFileListData - type CommandFileListData = { - path: string; - all?: boolean; - offset?: number; - limit?: number; - }; - // wshrpc.CommandGetMetaData type CommandGetMetaData = { oref: ORef; @@ -190,6 +182,17 @@ declare global { message: string; }; + // wshrpc.CommandRemoteListEntriesData + type CommandRemoteListEntriesData = { + path: string; + opts?: FileListOpts; + }; + + // wshrpc.CommandRemoteListEntriesRtnData + type CommandRemoteListEntriesRtnData = { + fileinfo?: FileInfo[]; + }; + // wshrpc.CommandRemoteStreamFileData type CommandRemoteStreamFileData = { path: string; @@ -343,8 +346,9 @@ declare global { // wshrpc.FileData type FileData = { - path: string; + info?: FileInfo; data64?: string; + entries?: FileInfo[]; opts?: FileOptsType; at?: FileDataAt; }; @@ -374,10 +378,24 @@ declare global { modestr: string; modtime: number; isdir?: boolean; + supportsmkdir?: boolean; mimetype?: string; readonly?: boolean; }; + // wshrpc.FileListData + type FileListData = { + path: string; + opts?: FileListOpts; + }; + + // wshrpc.FileListOpts + type FileListOpts = { + all?: boolean; + offset?: number; + limit?: number; + }; + // wshrpc.FileOptsType type FileOptsType = { maxsize?: number; @@ -398,12 +416,6 @@ declare global { configerrors: ConfigError[]; }; - // fstype.FullFile - type FullFile = { - info: FileInfo; - data64: string; - }; - // waveobj.LayoutActionData type LayoutActionData = { actiontype: string; diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 9cc05a58b..bd01b76c9 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -37,12 +37,12 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } } -func Read(ctx context.Context, path string) (*fstype.FullFile, error) { +func Read(ctx context.Context, path string) (*wshrpc.FileData, error) { client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) } - return client.Read(ctx, conn) + return client.Read(ctx, conn, wshrpc.FileData{}) } func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { @@ -54,12 +54,11 @@ func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { } func PutFile(ctx context.Context, data wshrpc.FileData) error { - client, conn := CreateFileShareClient(ctx, data.Path) + client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { - return fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Path) + return fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path) } - return client.PutFile(ctx, fstype.FileData{ - Conn: conn, + return client.PutFile(ctx, conn, wshrpc.FileData{ Data64: data.Data64, Opts: data.Opts, At: data.At, diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 28b410ed4..a4aaee8f9 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -25,7 +25,11 @@ func NewS3Client(config *aws.Config) *S3Client { } } -func (c S3Client) Read(ctx context.Context, data fstype.FileData) (*fstype.FullFile, error) { +func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { + return nil, nil +} + +func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { return nil, nil } @@ -33,7 +37,7 @@ func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc return nil, nil } -func (c S3Client) PutFile(ctx context.Context, data fstype.FileData) error { +func (c S3Client) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { return nil } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 39e5720f3..f143c1777 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -68,7 +68,11 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws } } if isDir { - + entries, err := listEntriesInternal(client, conn, &wshrpc.FileListOpts{}, &wshrpc.FileData{Entries: fileInfoArr}) + if err != nil { + return nil, fmt.Errorf("stream file, failed to list entries: %w", err) + } + fullFile.Entries = entries } else { // we can avoid this re-encoding if we ensure the remote side always encodes chunks of 3 bytes so we don't get padding chars fullFile.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) @@ -77,13 +81,34 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws } func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { - + return listEntriesInternal(wshclient.GetBareRpcClient(), conn, opts, nil) } func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opts *wshrpc.FileListOpts, data *wshrpc.FileData) ([]*wshrpc.FileInfo, error) { - entries := data.Entries - if opts.All { - + var entries []*wshrpc.FileInfo + if opts.All || data == nil { + rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + for respUnion := range rtnCh { + if respUnion.Error != nil { + return nil, respUnion.Error + } + resp := respUnion.Response + entries = append(entries, resp.FileInfo...) + } + } else { + entries = append(entries, data.Entries...) + if opts.Offset > 0 { + if opts.Offset >= len(entries) { + entries = nil + } else { + entries = entries[opts.Offset:] + } + } + if opts.Limit > 0 { + if opts.Limit < len(entries) { + entries = entries[:opts.Limit] + } + } } return entries, nil } diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index ebbc275e0..8c86d84d7 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -289,6 +289,11 @@ func RemoteFileTouchCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts return err } +// command "remotelistentries", wshserver.RemoteListEntriesCommand +func RemoteListEntriesCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteListEntriesData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + return sendRpcRequestResponseStreamHelper[wshrpc.CommandRemoteListEntriesRtnData](w, "remotelistentries", data, opts) +} + // command "remotemkdir", wshserver.RemoteMkdirCommand func RemoteMkdirCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "remotemkdir", data, opts) From d491ffa32783eafb32a7f3aad800f0bd4687437f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 18:26:17 -0800 Subject: [PATCH 019/126] save --- pkg/remote/fileshare/fileshare.go | 8 ++++++++ pkg/wshrpc/wshserver/wshserver.go | 15 ++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index bd01b76c9..f14a36ae2 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -45,6 +45,14 @@ func Read(ctx context.Context, path string) (*wshrpc.FileData, error) { return client.Read(ctx, conn, wshrpc.FileData{}) } +func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + } + return client.ListEntries(ctx, conn, opts) +} + func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index e8fa0213d..61f7412d3 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -282,7 +282,7 @@ func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.FileCrea fileOpts = *data.Opts } err := fileshare.PutFile(ctx, wshrpc.FileData{ - Path: data.Path, + Info: &wshrpc.FileInfo{Path: data.Path}, Data64: "", Opts: &fileOpts, }) @@ -293,15 +293,15 @@ func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.FileCrea } func (ws *WshServer) FileDeleteCommand(ctx context.Context, data wshrpc.FileData) error { - return fileshare.Delete(ctx, data.Path) + return fileshare.Delete(ctx, data.Info.Path) } func (ws *WshServer) FileInfoCommand(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileInfo, error) { - return fileshare.Stat(ctx, data.Path) + return fileshare.Stat(ctx, data.Info.Path) } -func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.CommandFileListData) ([]*wshrpc.FileInfo, error) { - return fileshare.Read(ctx, data.Path) +func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.FileListData) ([]*wshrpc.FileInfo, error) { + return fileshare.ListEntries(ctx, data.Path, data.Opts) } func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.FileData) error { @@ -309,13 +309,14 @@ func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.FileData) } func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) (string, error) { - + // TODO: implement this + return "", nil } func (ws *WshServer) FileAppendIJsonCommand(ctx context.Context, data wshrpc.CommandAppendIJsonData) error { tryCreate := true if data.FileName == blockcontroller.BlockFile_VDom && tryCreate { - err := filestore.WFS.MakeFile(ctx, data.ZoneId, data.FileName, nil, filestore.FileOptsType{MaxSize: blockcontroller.DefaultHtmlMaxFileSize, IJson: true}) + err := filestore.WFS.MakeFile(ctx, data.ZoneId, data.FileName, nil, wshrpc.FileOptsType{MaxSize: blockcontroller.DefaultHtmlMaxFileSize, IJson: true}) if err != nil && err != fs.ErrExist { return fmt.Errorf("error creating blockfile[vdom]: %w", err) } From d7f3a182e47af7b88c87f19f309ade3124edfd0f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 18:37:09 -0800 Subject: [PATCH 020/126] comment --- pkg/remote/fileshare/fstype/fstype.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index fb7d8181b..217c54591 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -15,8 +15,9 @@ type FullFile struct { type FileShareClient interface { // Stat returns the file info at the given parsed connection path Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) - // Read returns the file info at the given path, if it's a dir, then the file data will be a serialized array of FileInfo + // Read returns the file info at the given path, if it's a dir, then the list of entries Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) + // ListEntries returns the list of entries at the given path, or nothing if the path is a file ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) // PutFile writes the given data to the file at the given path PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error From 6219bb7eb87ba78b142238d34e3531a252c55234 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 13 Jan 2025 18:40:54 -0800 Subject: [PATCH 021/126] add supportsmkdir --- pkg/remote/fileshare/wavefs/wavefs.go | 22 +++++++++++---------- pkg/wshrpc/wshremote/wshremote.go | 28 ++++++++++++++------------- pkg/wshrpc/wshrpctypes.go | 2 +- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index c4e357481..39e243106 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -102,11 +102,12 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, for dir := range dirMap { dirName := prefix + dir + "/" filteredList = append(filteredList, &wshrpc.FileInfo{ - Path: fmt.Sprintf(WaveFilePathPattern, zoneId, dirName), - Name: dirName, - Dir: dirName, - Size: 0, - IsDir: true, + Path: fmt.Sprintf(WaveFilePathPattern, zoneId, dirName), + Name: dirName, + Dir: dirName, + Size: 0, + IsDir: true, + SupportsMkdir: false, }) } fileList = filteredList @@ -223,10 +224,11 @@ func (c WaveClient) GetConnectionType() string { func waveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { path := fmt.Sprintf(WaveFilePathPattern, wf.ZoneId, wf.Name) return &wshrpc.FileInfo{ - Path: path, - Name: wf.Name, - Opts: &wf.Opts, - Size: wf.Size, - Meta: &wf.Meta, + Path: path, + Name: wf.Name, + Opts: &wf.Opts, + Size: wf.Size, + Meta: &wf.Meta, + SupportsMkdir: false, } } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index c9421787c..d4c47f3a1 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -277,15 +277,16 @@ func listFilesInternal(path string, fileInfoArr []*fs.DirEntry, seen *int, opts func statToFileInfo(fullPath string, finfo fs.FileInfo, extended bool) *wshrpc.FileInfo { mimeType := utilfn.DetectMimeType(fullPath, finfo, extended) rtn := &wshrpc.FileInfo{ - Path: wavebase.ReplaceHomeDir(fullPath), - Dir: computeDirPart(fullPath, finfo.IsDir()), - Name: finfo.Name(), - Size: finfo.Size(), - Mode: finfo.Mode(), - ModeStr: finfo.Mode().String(), - ModTime: finfo.ModTime().UnixMilli(), - IsDir: finfo.IsDir(), - MimeType: mimeType, + Path: wavebase.ReplaceHomeDir(fullPath), + Dir: computeDirPart(fullPath, finfo.IsDir()), + Name: finfo.Name(), + Size: finfo.Size(), + Mode: finfo.Mode(), + ModeStr: finfo.Mode().String(), + ModTime: finfo.ModTime().UnixMilli(), + IsDir: finfo.IsDir(), + MimeType: mimeType, + SupportsMkdir: true, } if finfo.IsDir() { rtn.Size = -1 @@ -338,10 +339,11 @@ func (*ServerImpl) fileInfoInternal(path string, extended bool) (*wshrpc.FileInf finfo, err := os.Stat(cleanedPath) if os.IsNotExist(err) { return &wshrpc.FileInfo{ - Path: wavebase.ReplaceHomeDir(path), - Dir: computeDirPart(path, false), - NotFound: true, - ReadOnly: checkIsReadOnly(cleanedPath, finfo, false), + Path: wavebase.ReplaceHomeDir(path), + Dir: computeDirPart(path, false), + NotFound: true, + ReadOnly: checkIsReadOnly(cleanedPath, finfo, false), + SupportsMkdir: true, }, nil } if err != nil { diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 3a6c51dcd..f83e19448 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -349,7 +349,7 @@ type FileInfo struct { ModeStr string `json:"modestr"` ModTime int64 `json:"modtime"` IsDir bool `json:"isdir,omitempty"` - SupportsMkdir *bool `json:"supportsmkdir,omitempty"` + SupportsMkdir bool `json:"supportsmkdir,omitempty"` MimeType string `json:"mimetype,omitempty"` ReadOnly bool `json:"readonly,omitempty"` // this is not set for fileinfo's returned from directory listings } From 6414fbae89c66b4f4f966f2aceb0299bb3800f78 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 12:58:19 -0800 Subject: [PATCH 022/126] move getfullconfig to wshrpc --- cmd/wsh/cmd/wshcmd-file-util.go | 34 ++++---- cmd/wsh/cmd/wshcmd-getvar.go | 8 +- cmd/wsh/cmd/wshcmd-readfile.go | 4 +- emain/emain-tabview.ts | 5 +- emain/emain-window.ts | 12 +-- emain/emain-wsh.ts | 5 +- emain/emain.ts | 17 ++-- emain/updater.ts | 3 +- frontend/app/store/services.ts | 6 -- frontend/app/store/wshclientapi.ts | 5 ++ frontend/types/gotypes.d.ts | 2 +- frontend/wave.ts | 3 +- pkg/blockcontroller/blockcontroller.go | 4 +- pkg/remote/conncontroller/conncontroller.go | 12 +-- pkg/remote/fileshare/fstype/fstype.go | 5 -- pkg/remote/fileshare/wavefs/wavefs.go | 23 +----- pkg/remote/sshclient.go | 23 +++--- pkg/service/fileservice/fileservice.go | 90 --------------------- pkg/service/service.go | 2 - pkg/util/wavefileutil/wavefileutil.go | 32 ++++++++ pkg/wconfig/settingsconfig.go | 35 +++++++- pkg/wshrpc/wshclient/wshclient.go | 6 ++ pkg/wshrpc/wshrpctypes.go | 41 ++-------- pkg/wshrpc/wshserver/wshserver.go | 15 +++- 24 files changed, 163 insertions(+), 229 deletions(-) delete mode 100644 pkg/service/fileservice/fileservice.go create mode 100644 pkg/util/wavefileutil/wavefileutil.go diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index ca8051338..3d2f1d56d 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -10,6 +10,7 @@ import ( "io/fs" "strings" + "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" ) @@ -24,15 +25,11 @@ func convertNotFoundErr(err error) error { return err } -func ensureWaveFile(origName string, fileData wshrpc.CommandFileData) (*wshrpc.WaveFileInfo, error) { +func ensureWaveFile(origName string, fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { - createData := wshrpc.CommandFileCreateData{ - ZoneId: fileData.ZoneId, - FileName: fileData.FileName, - } - err = wshclient.FileCreateCommand(RpcClient, createData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + err = wshclient.FileCreateCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) if err != nil { return nil, fmt.Errorf("creating file: %w", err) } @@ -48,7 +45,7 @@ func ensureWaveFile(origName string, fileData wshrpc.CommandFileData) (*wshrpc.W return info, nil } -func streamWriteToWaveFile(fileData wshrpc.CommandFileData, reader io.Reader) error { +func streamWriteToWaveFile(fileData wshrpc.FileData, reader io.Reader) error { // First truncate the file with an empty write emptyWrite := fileData emptyWrite.Data64 = "" @@ -90,7 +87,7 @@ func streamWriteToWaveFile(fileData wshrpc.CommandFileData, reader io.Reader) er return nil } -func streamReadFromWaveFile(fileData wshrpc.CommandFileData, size int64, writer io.Writer) error { +func streamReadFromWaveFile(fileData wshrpc.FileData, size int64, writer io.Writer) error { const chunkSize = 32 * 1024 // 32KB chunks for offset := int64(0); offset < size; offset += chunkSize { // Calculate the length of this chunk @@ -127,7 +124,7 @@ func streamReadFromWaveFile(fileData wshrpc.CommandFileData, size int64, writer } type fileListResult struct { - info *wshrpc.WaveFileInfo + info *wshrpc.FileInfo err error } @@ -139,9 +136,9 @@ func streamFileList(zoneId string, path string, recursive bool, filesOnly bool) go func() { defer close(resultChan) - fileData := wshrpc.CommandFileData{ - ZoneId: zoneId, - FileName: path, + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, zoneId, path)}, } info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) @@ -169,13 +166,12 @@ func streamFileList(zoneId string, path string, recursive bool, filesOnly bool) foundAny := false for { - listData := wshrpc.CommandFileListData{ - ZoneId: zoneId, - Prefix: prefix, - All: recursive, - Offset: offset, - Limit: 100, - } + listData := wshrpc.FileListData{ + Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, zoneId, prefix), + Opts: &wshrpc.FileListOpts{ + All: recursive, + Offset: offset, + Limit: 100}} files, err := wshclient.FileListCommand(RpcClient, listData, &wshrpc.RpcOpts{Timeout: 2000}) if err != nil { diff --git a/cmd/wsh/cmd/wshcmd-getvar.go b/cmd/wsh/cmd/wshcmd-getvar.go index b88f0b4b2..693e1b797 100644 --- a/cmd/wsh/cmd/wshcmd-getvar.go +++ b/cmd/wsh/cmd/wshcmd-getvar.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/wavetermdev/waveterm/pkg/util/envutil" + "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" ) @@ -112,10 +113,9 @@ func getVarRun(cmd *cobra.Command, args []string) error { } func getAllVariables(zoneId string) error { - fileData := wshrpc.CommandFileData{ - ZoneId: zoneId, - FileName: getVarFileName, - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, zoneId, getVarFileName)}} envStr64, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) err = convertNotFoundErr(err) diff --git a/cmd/wsh/cmd/wshcmd-readfile.go b/cmd/wsh/cmd/wshcmd-readfile.go index c0afb9ed5..652e7d49e 100644 --- a/cmd/wsh/cmd/wshcmd-readfile.go +++ b/cmd/wsh/cmd/wshcmd-readfile.go @@ -5,8 +5,10 @@ package cmd import ( "encoding/base64" + "fmt" "github.com/spf13/cobra" + "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" ) @@ -29,7 +31,7 @@ func runReadFile(cmd *cobra.Command, args []string) { WriteStderr("[error] %v\n", err) return } - resp64, err := wshclient.FileReadCommand(RpcClient, wshrpc.CommandFileData{ZoneId: fullORef.OID, FileName: args[0]}, &wshrpc.RpcOpts{Timeout: 5000}) + resp64, err := wshclient.FileReadCommand(RpcClient, wshrpc.FileData{Info: &wshrpc.FileInfo{Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, fullORef.OID, args[0])}}, &wshrpc.RpcOpts{Timeout: 5000}) if err != nil { WriteStderr("[error] reading file: %v\n", err) return diff --git a/emain/emain-tabview.ts b/emain/emain-tabview.ts index 501c87795..7a9326988 100644 --- a/emain/emain-tabview.ts +++ b/emain/emain-tabview.ts @@ -1,7 +1,7 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 -import { FileService } from "@/app/store/services"; +import { RpcApi } from "@/app/store/wshclientapi"; import { adaptFromElectronKeyEvent } from "@/util/keyutil"; import { Rectangle, shell, WebContentsView } from "electron"; import { getWaveWindowById } from "emain/emain-window"; @@ -9,6 +9,7 @@ import path from "path"; import { configureAuthKeyRequestInjection } from "./authkey"; import { setWasActive } from "./emain-activity"; import { handleCtrlShiftFocus, handleCtrlShiftState, shFrameNavHandler, shNavHandler } from "./emain-util"; +import { ElectronWshClient } from "./emain-wsh"; import { getElectronAppBasePath, isDevVite } from "./platform"; function computeBgColor(fullConfig: FullConfigType): string { @@ -200,7 +201,7 @@ export async function getOrCreateWebViewForTab(waveWindowId: string, tabId: stri if (tabView) { return [tabView, true]; } - const fullConfig = await FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); tabView = getSpareTab(fullConfig); tabView.waveWindowId = waveWindowId; tabView.lastUsedTs = Date.now(); diff --git a/emain/emain-window.ts b/emain/emain-window.ts index b0ea19c46..4f95313e1 100644 --- a/emain/emain-window.ts +++ b/emain/emain-window.ts @@ -1,7 +1,8 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 -import { ClientService, FileService, ObjectService, WindowService, WorkspaceService } from "@/app/store/services"; +import { ClientService, ObjectService, WindowService, WorkspaceService } from "@/app/store/services"; +import { RpcApi } from "@/app/store/wshclientapi"; import { fireAndForget } from "@/util/util"; import { BaseWindow, BaseWindowConstructorOptions, dialog, globalShortcut, ipcMain, screen } from "electron"; import path from "path"; @@ -15,6 +16,7 @@ import { } from "./emain-activity"; import { getOrCreateWebViewForTab, getWaveTabViewByWebContentsId, WaveTabView } from "./emain-tabview"; import { delay, ensureBoundsAreVisible, waveKeyToElectronKey } from "./emain-util"; +import { ElectronWshClient } from "./emain-wsh"; import { log } from "./log"; import { getElectronAppBasePath, unamePlatform } from "./platform"; import { updater } from "./updater"; @@ -255,7 +257,7 @@ export class WaveBrowserWindow extends BaseWindow { e.preventDefault(); fireAndForget(async () => { const numWindows = waveWindowMap.size; - const fullConfig = await FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); if (numWindows > 1 || !fullConfig.settings["window:savelastwindow"]) { console.log("numWindows > 1 or user does not want last window saved", numWindows); if (fullConfig.settings["window:confirmclose"]) { @@ -621,7 +623,7 @@ export async function createWindowForWorkspace(workspaceId: string) { if (!newWin) { console.log("error creating new window", this.waveWindowId); } - const newBwin = await createBrowserWindow(newWin, await FileService.GetFullConfig(), { + const newBwin = await createBrowserWindow(newWin, await RpcApi.GetFullConfigCommand(ElectronWshClient), { unamePlatform, }); newBwin.show(); @@ -743,7 +745,7 @@ ipcMain.on("delete-workspace", (event, workspaceId) => { export async function createNewWaveWindow() { log("createNewWaveWindow"); const clientData = await ClientService.GetClientData(); - const fullConfig = await FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); let recreatedWindow = false; const allWindows = getAllWaveWindows(); if (allWindows.length === 0 && clientData?.windowids?.length >= 1) { @@ -780,7 +782,7 @@ export async function relaunchBrowserWindows() { setGlobalIsRelaunching(false); const clientData = await ClientService.GetClientData(); - const fullConfig = await FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); const wins: WaveBrowserWindow[] = []; for (const windowId of clientData.windowids.slice().reverse()) { const windowData: WaveWindow = await WindowService.GetWindow(windowId); diff --git a/emain/emain-wsh.ts b/emain/emain-wsh.ts index 20e4f235f..fe84e030e 100644 --- a/emain/emain-wsh.ts +++ b/emain/emain-wsh.ts @@ -1,7 +1,8 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 -import { FileService, WindowService } from "@/app/store/services"; +import { WindowService } from "@/app/store/services"; +import { RpcApi } from "@/app/store/wshclientapi"; import { Notification } from "electron"; import { getResolvedUpdateChannel } from "emain/updater"; import { RpcResponseHelper, WshClient } from "../frontend/app/store/wshclient"; @@ -44,7 +45,7 @@ export class ElectronWshClientType extends WshClient { async handle_focuswindow(rh: RpcResponseHelper, windowId: string) { console.log(`focuswindow ${windowId}`); - const fullConfig = await FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); let ww = getWaveWindowById(windowId); if (ww == null) { const window = await WindowService.GetWindow(windowId); diff --git a/emain/emain.ts b/emain/emain.ts index 969e25822..3cd3eb0ba 100644 --- a/emain/emain.ts +++ b/emain/emain.ts @@ -94,7 +94,7 @@ function handleWSEvent(evtMsg: WSEventType) { if (windowData == null) { return; } - const fullConfig = await services.FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); const newWin = await createBrowserWindow(windowData, fullConfig, { unamePlatform }); newWin.show(); } else if (evtMsg.eventtype == "electron:closewindow") { @@ -311,7 +311,7 @@ if (unamePlatform !== "darwin") { electron.ipcMain.on("update-window-controls-overlay", async (event, rect: Dimensions) => { // Bail out if the user requests the native titlebar - const fullConfig = await services.FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); if (fullConfig.settings["window:nativetitlebar"]) return; const zoomFactor = event.sender.getZoomFactor(); @@ -588,18 +588,19 @@ async function appMain() { console.log("wavesrv ready signal received", ready, Date.now() - startTs, "ms"); await electronApp.whenReady(); configureAuthKeyRequestInjection(electron.session.defaultSession); - const fullConfig = await services.FileService.GetFullConfig(); - checkIfRunningUnderARM64Translation(fullConfig); - ensureHotSpareTab(fullConfig); - await relaunchBrowserWindows(); - await initDocsite(); - setTimeout(runActiveTimer, 5000); // start active timer, wait 5s just to be safe try { initElectronWshClient(); initElectronWshrpc(ElectronWshClient, { authKey: AuthKey }); } catch (e) { console.log("error initializing wshrpc", e); } + const fullConfig = await RpcApi.GetFullConfigCommand(ElectronWshClient); + checkIfRunningUnderARM64Translation(fullConfig); + ensureHotSpareTab(fullConfig); + await relaunchBrowserWindows(); + await initDocsite(); + setTimeout(runActiveTimer, 5000); // start active timer, wait 5s just to be safe + makeAppMenu(); await configureAutoUpdater(); setGlobalIsStarting(false); diff --git a/emain/updater.ts b/emain/updater.ts index bd6e1b240..68dab9cf7 100644 --- a/emain/updater.ts +++ b/emain/updater.ts @@ -6,7 +6,6 @@ import { autoUpdater } from "electron-updater"; import { readFileSync } from "fs"; import path from "path"; import YAML from "yaml"; -import { FileService } from "../frontend/app/store/services"; import { RpcApi } from "../frontend/app/store/wshclientapi"; import { isDev } from "../frontend/util/isdev"; import { fireAndForget } from "../frontend/util/util"; @@ -240,7 +239,7 @@ export async function configureAutoUpdater() { try { console.log("Configuring updater"); - const settings = (await FileService.GetFullConfig()).settings; + const settings = (await RpcApi.GetFullConfigCommand(ElectronWshClient)).settings; updater = new Updater(settings); await updater.start(); } catch (e) { diff --git a/frontend/app/store/services.ts b/frontend/app/store/services.ts index 5211271f6..0d3e887ca 100644 --- a/frontend/app/store/services.ts +++ b/frontend/app/store/services.ts @@ -47,12 +47,6 @@ class ClientServiceType { export const ClientService = new ClientServiceType(); -// fileservice.FileService (file) -class FileServiceType { -} - -export const FileService = new FileServiceType(); - // objectservice.ObjectService (object) class ObjectServiceType { // @returns blockId (and object updates) diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 9ec6d36f1..e64342060 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -182,6 +182,11 @@ class RpcApiType { return client.wshRpcCall("focuswindow", data, opts); } + // command "getfullconfig" [call] + GetFullConfigCommand(client: WshClient, opts?: RpcOpts): Promise { + return client.wshRpcCall("getfullconfig", null, opts); + } + // command "getmeta" [call] GetMetaCommand(client: WshClient, data: CommandGetMetaData, opts?: RpcOpts): Promise { return client.wshRpcCall("getmeta", data, opts); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 025fd9d43..da6a06f8f 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -278,7 +278,7 @@ declare global { logblockid?: string; }; - // wshrpc.ConnKeywords + // wconfig.ConnKeywords type ConnKeywords = { "conn:wshenabled"?: boolean; "conn:askbeforewshinstall"?: boolean; diff --git a/frontend/wave.ts b/frontend/wave.ts index 463412b76..bc3688bd0 100644 --- a/frontend/wave.ts +++ b/frontend/wave.ts @@ -9,7 +9,6 @@ import { registerGlobalKeys, } from "@/app/store/keymodel"; import { modalsModel } from "@/app/store/modalmodel"; -import { FileService } from "@/app/store/services"; import { RpcApi } from "@/app/store/wshclientapi"; import { initWshrpc, TabRpcClient } from "@/app/store/wshrpcutil"; import { loadMonaco } from "@/app/view/codeeditor/codeeditor"; @@ -185,7 +184,7 @@ async function initWave(initOpts: WaveInitOpts) { registerElectronReinjectKeyHandler(); registerControlShiftStateUpdateHandler(); setTimeout(loadMonaco, 30); - const fullConfig = await FileService.GetFullConfig(); + const fullConfig = await RpcApi.GetFullConfigCommand(TabRpcClient); console.log("fullconfig", fullConfig); globalStore.set(atoms.fullConfigAtom, fullConfig); console.log("Wave First Render"); diff --git a/pkg/blockcontroller/blockcontroller.go b/pkg/blockcontroller/blockcontroller.go index d9214ca87..71fe7635f 100644 --- a/pkg/blockcontroller/blockcontroller.go +++ b/pkg/blockcontroller/blockcontroller.go @@ -356,7 +356,7 @@ func (bc *BlockController) setupAndStartShellProcess(rc *RunShellOpts, blockMeta if err != nil { return nil, err } - conn := conncontroller.GetConn(credentialCtx, opts, false, &wshrpc.ConnKeywords{}) + conn := conncontroller.GetConn(credentialCtx, opts, false, &wconfig.ConnKeywords{}) connStatus := conn.DeriveConnStatus() if connStatus.Status != conncontroller.Status_Connected { return nil, fmt.Errorf("not connected, cannot start shellproc") @@ -708,7 +708,7 @@ func CheckConnStatus(blockId string) error { if err != nil { return fmt.Errorf("error parsing connection name: %w", err) } - conn := conncontroller.GetConn(context.Background(), opts, false, &wshrpc.ConnKeywords{}) + conn := conncontroller.GetConn(context.Background(), opts, false, &wconfig.ConnKeywords{}) connStatus := conn.DeriveConnStatus() if connStatus.Status != conncontroller.Status_Connected { return fmt.Errorf("not connected: %s", connStatus.Status) diff --git a/pkg/remote/conncontroller/conncontroller.go b/pkg/remote/conncontroller/conncontroller.go index 057b19c57..3f3b65597 100644 --- a/pkg/remote/conncontroller/conncontroller.go +++ b/pkg/remote/conncontroller/conncontroller.go @@ -455,7 +455,7 @@ func (conn *SSHConn) Reconnect(ctx context.Context) error { if err != nil { return err } - return conn.Connect(ctx, &wshrpc.ConnKeywords{}) + return conn.Connect(ctx, &wconfig.ConnKeywords{}) } func (conn *SSHConn) WaitForConnect(ctx context.Context) error { @@ -483,7 +483,7 @@ func (conn *SSHConn) WaitForConnect(ctx context.Context) error { } // does not return an error since that error is stored inside of SSHConn -func (conn *SSHConn) Connect(ctx context.Context, connFlags *wshrpc.ConnKeywords) error { +func (conn *SSHConn) Connect(ctx context.Context, connFlags *wconfig.ConnKeywords) error { blocklogger.Infof(ctx, "\n") var connectAllowed bool conn.WithLock(func() { @@ -677,7 +677,7 @@ func (conn *SSHConn) persistWshInstalled(ctx context.Context, result WshCheckRes } // returns (connect-error) -func (conn *SSHConn) connectInternal(ctx context.Context, connFlags *wshrpc.ConnKeywords) error { +func (conn *SSHConn) connectInternal(ctx context.Context, connFlags *wconfig.ConnKeywords) error { conn.Infof(ctx, "connectInternal %s\n", conn.GetName()) client, _, err := remote.ConnectToClient(ctx, conn.Opts, nil, 0, connFlags) if err != nil { @@ -754,7 +754,7 @@ func getConnInternal(opts *remote.SSHOpts) *SSHConn { return rtn } -func GetConn(ctx context.Context, opts *remote.SSHOpts, shouldConnect bool, connFlags *wshrpc.ConnKeywords) *SSHConn { +func GetConn(ctx context.Context, opts *remote.SSHOpts, shouldConnect bool, connFlags *wconfig.ConnKeywords) *SSHConn { conn := getConnInternal(opts) if conn.Client == nil && shouldConnect { conn.Connect(ctx, connFlags) @@ -771,7 +771,7 @@ func EnsureConnection(ctx context.Context, connName string) error { if err != nil { return fmt.Errorf("error parsing connection name: %w", err) } - conn := GetConn(ctx, connOpts, false, &wshrpc.ConnKeywords{}) + conn := GetConn(ctx, connOpts, false, &wconfig.ConnKeywords{}) if conn == nil { return fmt.Errorf("connection not found: %s", connName) } @@ -782,7 +782,7 @@ func EnsureConnection(ctx context.Context, connName string) error { case Status_Connecting: return conn.WaitForConnect(ctx) case Status_Init, Status_Disconnected: - return conn.Connect(ctx, &wshrpc.ConnKeywords{}) + return conn.Connect(ctx, &wconfig.ConnKeywords{}) case Status_Error: return fmt.Errorf("connection error: %s", connStatus.Error) default: diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 217c54591..0347a0a9f 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -7,11 +7,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshrpc" ) -type FullFile struct { - Info *wshrpc.FileInfo `json:"info"` - Data64 string `json:"data64"` // base64 encoded -} - type FileShareClient interface { // Stat returns the file info at the given parsed connection path Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 39e243106..bd39f3410 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -13,15 +13,12 @@ import ( "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) -const ( - WaveFilePathPattern = "wavefile://%s/%s" -) - type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -73,7 +70,7 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, } var fileList []*wshrpc.FileInfo for _, wf := range fileListOrig { - fileList = append(fileList, waveFileToFileInfo(wf)) + fileList = append(fileList, wavefileutil.WaveFileToFileInfo(wf)) } if prefix != "" { var filteredList []*wshrpc.FileInfo @@ -102,7 +99,7 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, for dir := range dirMap { dirName := prefix + dir + "/" filteredList = append(filteredList, &wshrpc.FileInfo{ - Path: fmt.Sprintf(WaveFilePathPattern, zoneId, dirName), + Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, zoneId, dirName), Name: dirName, Dir: dirName, Size: 0, @@ -140,7 +137,7 @@ func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshr } return nil, fmt.Errorf("error getting file info: %w", err) } - return waveFileToFileInfo(fileInfo), nil + return wavefileutil.WaveFileToFileInfo(fileInfo), nil } func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { @@ -220,15 +217,3 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) erro func (c WaveClient) GetConnectionType() string { return connparse.ConnectionTypeWave } - -func waveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { - path := fmt.Sprintf(WaveFilePathPattern, wf.ZoneId, wf.Name) - return &wshrpc.FileInfo{ - Path: path, - Name: wf.Name, - Opts: &wf.Opts, - Size: wf.Size, - Meta: &wf.Meta, - SupportsMkdir: false, - } -} diff --git a/pkg/remote/sshclient.go b/pkg/remote/sshclient.go index f135b1932..6115782d0 100644 --- a/pkg/remote/sshclient.go +++ b/pkg/remote/sshclient.go @@ -31,7 +31,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wconfig" - "github.com/wavetermdev/waveterm/pkg/wshrpc" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" xknownhosts "golang.org/x/crypto/ssh/knownhosts" @@ -115,7 +114,7 @@ func createDummySigner() ([]ssh.Signer, error) { // they were successes. An error in this function prevents any other // keys from being attempted. But if there's an error because of a dummy // file, the library can still try again with a new key. -func createPublicKeyCallback(connCtx context.Context, sshKeywords *wshrpc.ConnKeywords, authSockSignersExt []ssh.Signer, agentClient agent.ExtendedAgent, debugInfo *ConnectionDebugInfo) func() ([]ssh.Signer, error) { +func createPublicKeyCallback(connCtx context.Context, sshKeywords *wconfig.ConnKeywords, authSockSignersExt []ssh.Signer, agentClient agent.ExtendedAgent, debugInfo *ConnectionDebugInfo) func() ([]ssh.Signer, error) { var identityFiles []string existingKeys := make(map[string][]byte) @@ -402,7 +401,7 @@ func lineContainsMatch(line []byte, matches [][]byte) bool { return false } -func createHostKeyCallback(sshKeywords *wshrpc.ConnKeywords) (ssh.HostKeyCallback, HostKeyAlgorithms, error) { +func createHostKeyCallback(sshKeywords *wconfig.ConnKeywords) (ssh.HostKeyCallback, HostKeyAlgorithms, error) { globalKnownHostsFiles := sshKeywords.SshGlobalKnownHostsFile userKnownHostsFiles := sshKeywords.SshUserKnownHostsFile @@ -568,7 +567,7 @@ func createHostKeyCallback(sshKeywords *wshrpc.ConnKeywords) (ssh.HostKeyCallbac return waveHostKeyCallback, hostKeyAlgorithms, nil } -func createClientConfig(connCtx context.Context, sshKeywords *wshrpc.ConnKeywords, debugInfo *ConnectionDebugInfo) (*ssh.ClientConfig, error) { +func createClientConfig(connCtx context.Context, sshKeywords *wconfig.ConnKeywords, debugInfo *ConnectionDebugInfo) (*ssh.ClientConfig, error) { chosenUser := utilfn.SafeDeref(sshKeywords.SshUser) chosenHostName := utilfn.SafeDeref(sshKeywords.SshHostName) chosenPort := utilfn.SafeDeref(sshKeywords.SshPort) @@ -660,7 +659,7 @@ func connectInternal(ctx context.Context, networkAddr string, clientConfig *ssh. return ssh.NewClient(c, chans, reqs), nil } -func ConnectToClient(connCtx context.Context, opts *SSHOpts, currentClient *ssh.Client, jumpNum int32, connFlags *wshrpc.ConnKeywords) (*ssh.Client, int32, error) { +func ConnectToClient(connCtx context.Context, opts *SSHOpts, currentClient *ssh.Client, jumpNum int32, connFlags *wconfig.ConnKeywords) (*ssh.Client, int32, error) { blocklogger.Infof(connCtx, "[conndebug] ConnectToClient %s (jump:%d)...\n", opts.String(), jumpNum) debugInfo := &ConnectionDebugInfo{ CurrentClient: currentClient, @@ -676,7 +675,7 @@ func ConnectToClient(connCtx context.Context, opts *SSHOpts, currentClient *ssh. return nil, debugInfo.JumpNum, ConnectionError{ConnectionDebugInfo: debugInfo, Err: err} } - parsedKeywords := &wshrpc.ConnKeywords{} + parsedKeywords := &wconfig.ConnKeywords{} if opts.SSHUser != "" { parsedKeywords.SshUser = &opts.SSHUser } @@ -688,7 +687,7 @@ func ConnectToClient(connCtx context.Context, opts *SSHOpts, currentClient *ssh. fullConfig := wconfig.GetWatcher().GetFullConfig() internalSshConfigKeywords, ok := fullConfig.Connections[rawName] if !ok { - internalSshConfigKeywords = wshrpc.ConnKeywords{} + internalSshConfigKeywords = wconfig.ConnKeywords{} } // cascade order: @@ -721,7 +720,7 @@ func ConnectToClient(connCtx context.Context, opts *SSHOpts, currentClient *ssh. } // do not apply supplied keywords to proxies - ssh config must be used for that - debugInfo.CurrentClient, jumpNum, err = ConnectToClient(connCtx, proxyOpts, debugInfo.CurrentClient, jumpNum, &wshrpc.ConnKeywords{}) + debugInfo.CurrentClient, jumpNum, err = ConnectToClient(connCtx, proxyOpts, debugInfo.CurrentClient, jumpNum, &wconfig.ConnKeywords{}) if err != nil { // do not add a context on a recursive call // (this can cause a recursive nested context that's arbitrarily deep) @@ -743,7 +742,7 @@ func ConnectToClient(connCtx context.Context, opts *SSHOpts, currentClient *ssh. // note that a `var == "yes"` will default to false // but `var != "no"` will default to true // when given unexpected strings -func findSshConfigKeywords(hostPattern string) (connKeywords *wshrpc.ConnKeywords, outErr error) { +func findSshConfigKeywords(hostPattern string) (connKeywords *wconfig.ConnKeywords, outErr error) { defer func() { panicErr := panichandler.PanicHandler("sshclient:find-ssh-config-keywords", recover()) if panicErr != nil { @@ -751,7 +750,7 @@ func findSshConfigKeywords(hostPattern string) (connKeywords *wshrpc.ConnKeyword } }() WaveSshConfigUserSettings().ReloadConfigs() - sshKeywords := &wshrpc.ConnKeywords{} + sshKeywords := &wconfig.ConnKeywords{} var err error userRaw, err := WaveSshConfigUserSettings().GetStrict(hostPattern, "User") @@ -893,9 +892,9 @@ func (opts SSHOpts) String() string { return stringRepr } -func mergeKeywords(oldKeywords *wshrpc.ConnKeywords, newKeywords *wshrpc.ConnKeywords) *wshrpc.ConnKeywords { +func mergeKeywords(oldKeywords *wconfig.ConnKeywords, newKeywords *wconfig.ConnKeywords) *wconfig.ConnKeywords { if oldKeywords == nil { - oldKeywords = &wshrpc.ConnKeywords{} + oldKeywords = &wconfig.ConnKeywords{} } if newKeywords == nil { return oldKeywords diff --git a/pkg/service/fileservice/fileservice.go b/pkg/service/fileservice/fileservice.go deleted file mode 100644 index 96e2e9c4d..000000000 --- a/pkg/service/fileservice/fileservice.go +++ /dev/null @@ -1,90 +0,0 @@ -package fileservice - -import ( - "context" - "fmt" - "time" - - "github.com/wavetermdev/waveterm/pkg/filestore" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" - "github.com/wavetermdev/waveterm/pkg/tsgen/tsgenmeta" - "github.com/wavetermdev/waveterm/pkg/wconfig" - "github.com/wavetermdev/waveterm/pkg/wshrpc" -) - -const MaxFileSize = 10 * 1024 * 1024 // 10M -const DefaultTimeout = 2 * time.Second - -type FileService struct{} - -func (fs *FileService) SaveFile_Meta() tsgenmeta.MethodMeta { - return tsgenmeta.MethodMeta{ - Desc: "save file", - ArgNames: []string{"ctx", "connection", "path", "data64"}, - } -} - -func (fs *FileService) SaveFile(ctx context.Context, connection string, path string, data64 string) error { - return fileshare.PutFile(ctx, wshrpc.FileData{Path: path, Data64: data64}) -} - -func (fs *FileService) StatFile_Meta() tsgenmeta.MethodMeta { - return tsgenmeta.MethodMeta{ - Desc: "get file info", - ArgNames: []string{"ctx", "connection", "path"}, - } -} - -func (fs *FileService) StatFile(ctx context.Context, connection string, path string) (*wshrpc.FileInfo, error) { - return fileshare.Stat(ctx, path) -} - -func (fs *FileService) Mkdir(ctx context.Context, connection string, path string) error { - return fileshare.Mkdir(ctx, path) -} - -func (fs *FileService) TouchFile(ctx context.Context, connection string, path string) error { - return fileshare.PutFile(ctx, wshrpc.FileData{Path: path, Data64: ""}) -} - -func (fs *FileService) Rename(ctx context.Context, connection string, path string, newPath string) error { - return fileshare.Move(ctx, path, newPath, false) -} - -func (fs *FileService) ReadFile_Meta() tsgenmeta.MethodMeta { - return tsgenmeta.MethodMeta{ - Desc: "read file", - ArgNames: []string{"ctx", "connection", "path"}, - } -} - -func (fs *FileService) ReadFile(ctx context.Context, connection string, path string) (*fstype.FullFile, error) { - return fileshare.Read(ctx, path) -} - -func (fs *FileService) GetWaveFile(id string, path string) (any, error) { - ctx, cancelFn := context.WithTimeout(context.Background(), DefaultTimeout) - defer cancelFn() - file, err := filestore.WFS.Stat(ctx, id, path) - if err != nil { - return nil, fmt.Errorf("error getting file: %w", err) - } - return file, nil -} - -func (fs *FileService) DeleteFile_Meta() tsgenmeta.MethodMeta { - return tsgenmeta.MethodMeta{ - Desc: "delete file", - ArgNames: []string{"ctx", "connection", "path"}, - } -} - -func (fs *FileService) DeleteFile(ctx context.Context, connection string, path string) error { - return fileshare.Delete(ctx, path) -} - -func (fs *FileService) GetFullConfig() wconfig.FullConfigType { - watcher := wconfig.GetWatcher() - return watcher.GetFullConfig() -} diff --git a/pkg/service/service.go b/pkg/service/service.go index a51955bb3..b0ab2f975 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -11,7 +11,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/service/blockservice" "github.com/wavetermdev/waveterm/pkg/service/clientservice" - "github.com/wavetermdev/waveterm/pkg/service/fileservice" "github.com/wavetermdev/waveterm/pkg/service/objectservice" "github.com/wavetermdev/waveterm/pkg/service/userinputservice" "github.com/wavetermdev/waveterm/pkg/service/windowservice" @@ -25,7 +24,6 @@ import ( var ServiceMap = map[string]any{ "block": blockservice.BlockServiceInstance, "object": &objectservice.ObjectService{}, - "file": &fileservice.FileService{}, "client": &clientservice.ClientService{}, "window": &windowservice.WindowService{}, "workspace": &workspaceservice.WorkspaceService{}, diff --git a/pkg/util/wavefileutil/wavefileutil.go b/pkg/util/wavefileutil/wavefileutil.go new file mode 100644 index 000000000..81b09cf28 --- /dev/null +++ b/pkg/util/wavefileutil/wavefileutil.go @@ -0,0 +1,32 @@ +package wavefileutil + +import ( + "fmt" + + "github.com/wavetermdev/waveterm/pkg/filestore" + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) + +const ( + WaveFilePathPattern = "wavefile://%s/%s" +) + +func WaveFileToFileInfo(wf *filestore.WaveFile) *wshrpc.FileInfo { + path := fmt.Sprintf(WaveFilePathPattern, wf.ZoneId, wf.Name) + return &wshrpc.FileInfo{ + Path: path, + Name: wf.Name, + Opts: &wf.Opts, + Size: wf.Size, + Meta: &wf.Meta, + SupportsMkdir: false, + } +} + +func WaveFileListToFileInfoList(wfList []*filestore.WaveFile) []*wshrpc.FileInfo { + var fileInfoList []*wshrpc.FileInfo + for _, wf := range wfList { + fileInfoList = append(fileInfoList, WaveFileToFileInfo(wf)) + } + return fileInfoList +} diff --git a/pkg/wconfig/settingsconfig.go b/pkg/wconfig/settingsconfig.go index 3fcfb2bd8..066fae639 100644 --- a/pkg/wconfig/settingsconfig.go +++ b/pkg/wconfig/settingsconfig.go @@ -19,7 +19,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wconfig/defaultconfig" - "github.com/wavetermdev/waveterm/pkg/wshrpc" ) const SettingsFile = "settings.json" @@ -133,10 +132,42 @@ type FullConfigType struct { Widgets map[string]WidgetConfigType `json:"widgets"` Presets map[string]waveobj.MetaMapType `json:"presets"` TermThemes map[string]TermThemeType `json:"termthemes"` - Connections map[string]wshrpc.ConnKeywords `json:"connections"` + Connections map[string]ConnKeywords `json:"connections"` ConfigErrors []ConfigError `json:"configerrors" configfile:"-"` } +type ConnKeywords struct { + ConnWshEnabled *bool `json:"conn:wshenabled,omitempty"` + ConnAskBeforeWshInstall *bool `json:"conn:askbeforewshinstall,omitempty"` + ConnOverrideConfig bool `json:"conn:overrideconfig,omitempty"` + ConnWshPath string `json:"conn:wshpath,omitempty"` + + DisplayHidden *bool `json:"display:hidden,omitempty"` + DisplayOrder float32 `json:"display:order,omitempty"` + + TermClear bool `json:"term:*,omitempty"` + TermFontSize float64 `json:"term:fontsize,omitempty"` + TermFontFamily string `json:"term:fontfamily,omitempty"` + TermTheme string `json:"term:theme,omitempty"` + + SshUser *string `json:"ssh:user,omitempty"` + SshHostName *string `json:"ssh:hostname,omitempty"` + SshPort *string `json:"ssh:port,omitempty"` + SshIdentityFile []string `json:"ssh:identityfile,omitempty"` + SshBatchMode *bool `json:"ssh:batchmode,omitempty"` + SshPubkeyAuthentication *bool `json:"ssh:pubkeyauthentication,omitempty"` + SshPasswordAuthentication *bool `json:"ssh:passwordauthentication,omitempty"` + SshKbdInteractiveAuthentication *bool `json:"ssh:kbdinteractiveauthentication,omitempty"` + SshPreferredAuthentications []string `json:"ssh:preferredauthentications,omitempty"` + SshAddKeysToAgent *bool `json:"ssh:addkeystoagent,omitempty"` + SshIdentityAgent *string `json:"ssh:identityagent,omitempty"` + SshProxyJump []string `json:"ssh:proxyjump,omitempty"` + SshUserKnownHostsFile []string `json:"ssh:userknownhostsfile,omitempty"` + SshGlobalKnownHostsFile []string `json:"ssh:globalknownhostsfile,omitempty"` + + AwsConfig *string `json:"aws:config,omitempty"` +} + func DefaultBoolPtr(arg *bool, def bool) bool { if arg == nil { return def diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 8c86d84d7..523cbf18b 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -223,6 +223,12 @@ func FocusWindowCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) er return err } +// command "getfullconfig", wshserver.GetFullConfigCommand +func GetFullConfigCommand(w *wshutil.WshRpc, opts *wshrpc.RpcOpts) (wconfig.FullConfigType, error) { + resp, err := sendRpcRequestCallHelper[wconfig.FullConfigType](w, "getfullconfig", nil, opts) + return resp, err +} + // command "getmeta", wshserver.GetMetaCommand func GetMetaCommand(w *wshutil.WshRpc, data wshrpc.CommandGetMetaData, opts *wshrpc.RpcOpts) (waveobj.MetaMapType, error) { resp, err := sendRpcRequestCallHelper[waveobj.MetaMapType](w, "getmeta", data, opts) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index f83e19448..3b2781f44 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -15,6 +15,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/ijson" "github.com/wavetermdev/waveterm/pkg/vdom" "github.com/wavetermdev/waveterm/pkg/waveobj" + "github.com/wavetermdev/waveterm/pkg/wconfig" "github.com/wavetermdev/waveterm/pkg/wps" ) @@ -60,6 +61,7 @@ const ( Command_Test = "test" Command_SetConfig = "setconfig" Command_SetConnectionsConfig = "connectionsconfig" + Command_GetFullConfig = "getfullconfig" Command_RemoteStreamFile = "remotestreamfile" Command_RemoteFileInfo = "remotefileinfo" Command_RemoteFileTouch = "remotefiletouch" @@ -143,6 +145,7 @@ type WshRpcInterface interface { TestCommand(ctx context.Context, data string) error SetConfigCommand(ctx context.Context, data MetaSettingsType) error SetConnectionsConfigCommand(ctx context.Context, data ConnConfigRequest) error + GetFullConfigCommand(ctx context.Context) (wconfig.FullConfigType, error) BlockInfoCommand(ctx context.Context, blockId string) (*BlockInfoData, error) WaveInfoCommand(ctx context.Context) (*WaveInfoData, error) WshActivityCommand(ct context.Context, data map[string]int) error @@ -477,42 +480,10 @@ type CommandRemoteWriteFileData struct { CreateMode os.FileMode `json:"createmode,omitempty"` } -type ConnKeywords struct { - ConnWshEnabled *bool `json:"conn:wshenabled,omitempty"` - ConnAskBeforeWshInstall *bool `json:"conn:askbeforewshinstall,omitempty"` - ConnOverrideConfig bool `json:"conn:overrideconfig,omitempty"` - ConnWshPath string `json:"conn:wshpath,omitempty"` - - DisplayHidden *bool `json:"display:hidden,omitempty"` - DisplayOrder float32 `json:"display:order,omitempty"` - - TermClear bool `json:"term:*,omitempty"` - TermFontSize float64 `json:"term:fontsize,omitempty"` - TermFontFamily string `json:"term:fontfamily,omitempty"` - TermTheme string `json:"term:theme,omitempty"` - - SshUser *string `json:"ssh:user,omitempty"` - SshHostName *string `json:"ssh:hostname,omitempty"` - SshPort *string `json:"ssh:port,omitempty"` - SshIdentityFile []string `json:"ssh:identityfile,omitempty"` - SshBatchMode *bool `json:"ssh:batchmode,omitempty"` - SshPubkeyAuthentication *bool `json:"ssh:pubkeyauthentication,omitempty"` - SshPasswordAuthentication *bool `json:"ssh:passwordauthentication,omitempty"` - SshKbdInteractiveAuthentication *bool `json:"ssh:kbdinteractiveauthentication,omitempty"` - SshPreferredAuthentications []string `json:"ssh:preferredauthentications,omitempty"` - SshAddKeysToAgent *bool `json:"ssh:addkeystoagent,omitempty"` - SshIdentityAgent *string `json:"ssh:identityagent,omitempty"` - SshProxyJump []string `json:"ssh:proxyjump,omitempty"` - SshUserKnownHostsFile []string `json:"ssh:userknownhostsfile,omitempty"` - SshGlobalKnownHostsFile []string `json:"ssh:globalknownhostsfile,omitempty"` - - AwsConfig *string `json:"aws:config,omitempty"` -} - type ConnRequest struct { - Host string `json:"host"` - Keywords ConnKeywords `json:"keywords,omitempty"` - LogBlockId string `json:"logblockid,omitempty"` + Host string `json:"host"` + Keywords wconfig.ConnKeywords `json:"keywords,omitempty"` + LogBlockId string `json:"logblockid,omitempty"` } const ( diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 61f7412d3..58409e709 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -28,6 +28,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/util/envutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" + "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/waveai" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/waveobj" @@ -433,6 +434,11 @@ func (ws *WshServer) SetConnectionsConfigCommand(ctx context.Context, data wshrp return wconfig.SetConnectionsConfigValue(data.Host, data.MetaMapType) } +func (ws *WshServer) GetFullConfigCommand(ctx context.Context) (wconfig.FullConfigType, error) { + watcher := wconfig.GetWatcher() + return watcher.GetFullConfig(), nil +} + func (ws *WshServer) ConnStatusCommand(ctx context.Context) ([]wshrpc.ConnStatus, error) { rtn := conncontroller.GetAllConnStatus() return rtn, nil @@ -480,7 +486,7 @@ func (ws *WshServer) ConnDisconnectCommand(ctx context.Context, connName string) if err != nil { return fmt.Errorf("error parsing connection name: %w", err) } - conn := conncontroller.GetConn(ctx, connOpts, false, &wshrpc.ConnKeywords{}) + conn := conncontroller.GetConn(ctx, connOpts, false, &wconfig.ConnKeywords{}) if conn == nil { return fmt.Errorf("connection not found: %s", connName) } @@ -524,7 +530,7 @@ func (ws *WshServer) ConnReinstallWshCommand(ctx context.Context, data wshrpc.Co if err != nil { return fmt.Errorf("error parsing connection name: %w", err) } - conn := conncontroller.GetConn(ctx, connOpts, false, &wshrpc.ConnKeywords{}) + conn := conncontroller.GetConn(ctx, connOpts, false, &wconfig.ConnKeywords{}) if conn == nil { return fmt.Errorf("connection not found: %s", connName) } @@ -596,12 +602,13 @@ func (ws *WshServer) BlockInfoCommand(ctx context.Context, blockId string) (*wsh if err != nil { return nil, fmt.Errorf("error listing blockfiles: %w", err) } + fileInfoList := wavefileutil.WaveFileListToFileInfoList(fileList) return &wshrpc.BlockInfoData{ BlockId: blockId, TabId: tabId, WorkspaceId: workspaceId, Block: blockData, - Files: fileList, + Files: fileInfoList, }, nil } @@ -684,7 +691,7 @@ func (ws *WshServer) SetVarCommand(ctx context.Context, data wshrpc.CommandVarDa _, fileData, err := filestore.WFS.ReadFile(ctx, data.ZoneId, data.FileName) if err == fs.ErrNotExist { fileData = []byte{} - err = filestore.WFS.MakeFile(ctx, data.ZoneId, data.FileName, nil, filestore.FileOptsType{}) + err = filestore.WFS.MakeFile(ctx, data.ZoneId, data.FileName, nil, wshrpc.FileOptsType{}) if err != nil { return fmt.Errorf("error creating blockfile: %w", err) } From cf775833bd8b41b6b03716b980a18954371a7645 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 13:58:31 -0800 Subject: [PATCH 023/126] fix wsh errors --- cmd/generatego/main-generatego.go | 1 + cmd/wsh/cmd/wshcmd-file-util.go | 6 +- cmd/wsh/cmd/wshcmd-file.go | 148 +++++------------- cmd/wsh/cmd/wshcmd-ssh.go | 3 +- .../app/view/preview/directorypreview.tsx | 1 - pkg/wshrpc/wshclient/wshclient.go | 1 + 6 files changed, 42 insertions(+), 118 deletions(-) diff --git a/cmd/generatego/main-generatego.go b/cmd/generatego/main-generatego.go index 70c216ccd..c93faa352 100644 --- a/cmd/generatego/main-generatego.go +++ b/cmd/generatego/main-generatego.go @@ -26,6 +26,7 @@ func GenerateWshClient() error { gogen.GenerateBoilerplate(&buf, "wshclient", []string{ "github.com/wavetermdev/waveterm/pkg/wshutil", "github.com/wavetermdev/waveterm/pkg/wshrpc", + "github.com/wavetermdev/waveterm/pkg/wconfig", "github.com/wavetermdev/waveterm/pkg/waveobj", "github.com/wavetermdev/waveterm/pkg/wps", "github.com/wavetermdev/waveterm/pkg/vdom", diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 3d2f1d56d..d805f1e6b 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -25,7 +25,7 @@ func convertNotFoundErr(err error) error { return err } -func ensureWaveFile(origName string, fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { +func ensureFile(origName string, fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { @@ -45,7 +45,7 @@ func ensureWaveFile(origName string, fileData wshrpc.FileData) (*wshrpc.FileInfo return info, nil } -func streamWriteToWaveFile(fileData wshrpc.FileData, reader io.Reader) error { +func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { // First truncate the file with an empty write emptyWrite := fileData emptyWrite.Data64 = "" @@ -87,7 +87,7 @@ func streamWriteToWaveFile(fileData wshrpc.FileData, reader io.Reader) error { return nil } -func streamReadFromWaveFile(fileData wshrpc.FileData, size int64, writer io.Writer) error { +func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) error { const chunkSize = 32 * 1024 // 32KB chunks for offset := int64(0); offset < size; offset += chunkSize { // Calculate the length of this chunk diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 8d1280515..8bc45e213 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -166,20 +166,10 @@ Exactly one of source or destination must be a wavefile:// URL.`, } func fileCatRun(cmd *cobra.Command, args []string) error { - ref, err := parseWaveFileURL(args[0]) - if err != nil { - return err - } - - fullORef, err := resolveWaveFile(ref) - if err != nil { - return err - } - fileData := wshrpc.CommandFileData{ - ZoneId: fullORef.OID, - FileName: ref.fileName, - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: args[0]}} // Get file info first to check existence and get size info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) @@ -191,7 +181,7 @@ func fileCatRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("getting file info: %w", err) } - err = streamReadFromWaveFile(fileData, info.Size, os.Stdout) + err = streamReadFromFile(fileData, info.Size, os.Stdout) if err != nil { return fmt.Errorf("reading file: %w", err) } @@ -200,20 +190,9 @@ func fileCatRun(cmd *cobra.Command, args []string) error { } func fileInfoRun(cmd *cobra.Command, args []string) error { - ref, err := parseWaveFileURL(args[0]) - if err != nil { - return err - } - - fullORef, err := resolveWaveFile(ref) - if err != nil { - return err - } - - fileData := wshrpc.CommandFileData{ - ZoneId: fullORef.OID, - FileName: ref.fileName, - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: args[0]}} info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) @@ -226,11 +205,10 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { WriteStdout("filename: %s\n", info.Name) WriteStdout("size: %d\n", info.Size) - WriteStdout("ctime: %s\n", time.Unix(info.CreatedTs/1000, 0).Format(time.DateTime)) - WriteStdout("mtime: %s\n", time.Unix(info.ModTs/1000, 0).Format(time.DateTime)) - if len(info.Meta) > 0 { + WriteStdout("mtime: %s\n", time.Unix(info.ModTime/1000, 0).Format(time.DateTime)) + if len(*info.Meta) > 0 { WriteStdout("metadata:\n") - for k, v := range info.Meta { + for k, v := range *info.Meta { WriteStdout(" %s: %v\n", k, v) } } @@ -238,22 +216,11 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { } func fileRmRun(cmd *cobra.Command, args []string) error { - ref, err := parseWaveFileURL(args[0]) - if err != nil { - return err - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: args[0]}} - fullORef, err := resolveWaveFile(ref) - if err != nil { - return err - } - - fileData := wshrpc.CommandFileData{ - ZoneId: fullORef.OID, - FileName: ref.fileName, - } - - _, err = wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + _, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { return fmt.Errorf("%s: no such file", args[0]) @@ -271,27 +238,16 @@ func fileRmRun(cmd *cobra.Command, args []string) error { } func fileWriteRun(cmd *cobra.Command, args []string) error { - ref, err := parseWaveFileURL(args[0]) - if err != nil { - return err - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: args[0]}} - fullORef, err := resolveWaveFile(ref) + _, err := ensureFile(args[0], fileData) if err != nil { return err } - fileData := wshrpc.CommandFileData{ - ZoneId: fullORef.OID, - FileName: ref.fileName, - } - - _, err = ensureWaveFile(args[0], fileData) - if err != nil { - return err - } - - err = streamWriteToWaveFile(fileData, WrappedStdin) + err = streamWriteToFile(fileData, WrappedStdin) if err != nil { return fmt.Errorf("writing file: %w", err) } @@ -300,22 +256,11 @@ func fileWriteRun(cmd *cobra.Command, args []string) error { } func fileAppendRun(cmd *cobra.Command, args []string) error { - ref, err := parseWaveFileURL(args[0]) - if err != nil { - return err - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: args[0]}} - fullORef, err := resolveWaveFile(ref) - if err != nil { - return err - } - - fileData := wshrpc.CommandFileData{ - ZoneId: fullORef.OID, - FileName: ref.fileName, - } - - info, err := ensureWaveFile(args[0], fileData) + info, err := ensureFile(args[0], fileData) if err != nil { return err } @@ -415,20 +360,9 @@ func fileCpRun(cmd *cobra.Command, args []string) error { } func copyFromWaveToLocal(src, dst string) error { - ref, err := parseWaveFileURL(src) - if err != nil { - return err - } - - fullORef, err := resolveWaveFile(ref) - if err != nil { - return err - } - - fileData := wshrpc.CommandFileData{ - ZoneId: fullORef.OID, - FileName: ref.fileName, - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: src}} // Get file info first to check existence and get size info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) @@ -447,7 +381,7 @@ func copyFromWaveToLocal(src, dst string) error { } defer f.Close() - err = streamReadFromWaveFile(fileData, info.Size, f) + err = streamReadFromFile(fileData, info.Size, f) if err != nil { return fmt.Errorf("reading wave file: %w", err) } @@ -456,15 +390,9 @@ func copyFromWaveToLocal(src, dst string) error { } func copyFromLocalToWave(src, dst string) error { - ref, err := parseWaveFileURL(dst) - if err != nil { - return err - } - - fullORef, err := resolveWaveFile(ref) - if err != nil { - return err - } + fileData := wshrpc.FileData{ + Info: &wshrpc.FileInfo{ + Path: dst}} // stat local file stat, err := os.Stat(src) @@ -477,13 +405,7 @@ func copyFromLocalToWave(src, dst string) error { if stat.IsDir() { return fmt.Errorf("%s: is a directory", src) } - - fileData := wshrpc.CommandFileData{ - ZoneId: fullORef.OID, - FileName: ref.fileName, - } - - _, err = ensureWaveFile(dst, fileData) + _, err = ensureFile(dst, fileData) if err != nil { return err } @@ -494,7 +416,7 @@ func copyFromLocalToWave(src, dst string) error { } defer file.Close() - err = streamWriteToWaveFile(fileData, file) + err = streamWriteToFile(fileData, file) if err != nil { return fmt.Errorf("writing wave file: %w", err) } @@ -530,7 +452,7 @@ func filePrintColumns(filesChan <-chan fileListResult) error { func filePrintLong(filesChan <-chan fileListResult) error { // Sample first 100 files to determine name width maxNameLen := 0 - var samples []*wshrpc.WaveFileInfo + var samples []*wshrpc.FileInfo for f := range filesChan { if f.err != nil { @@ -555,7 +477,7 @@ func filePrintLong(filesChan <-chan fileListResult) error { // Print samples for _, f := range samples { name := f.Name - t := time.Unix(f.ModTs/1000, 0) + t := time.Unix(f.ModTime/1000, 0) timestamp := utilfn.FormatLsTime(t) if f.Size == 0 && strings.HasSuffix(name, "/") { fmt.Fprintf(os.Stdout, "%-*s %8s %s\n", nameWidth, name, "-", timestamp) @@ -570,7 +492,7 @@ func filePrintLong(filesChan <-chan fileListResult) error { return f.err } name := f.info.Name - timestamp := time.Unix(f.info.ModTs/1000, 0).Format("Jan 02 15:04") + timestamp := time.Unix(f.info.ModTime/1000, 0).Format("Jan 02 15:04") if f.info.Size == 0 && strings.HasSuffix(name, "/") { fmt.Fprintf(os.Stdout, "%-*s %8s %s\n", nameWidth, name, "-", timestamp) } else { diff --git a/cmd/wsh/cmd/wshcmd-ssh.go b/cmd/wsh/cmd/wshcmd-ssh.go index ab5b552bb..36e827931 100644 --- a/cmd/wsh/cmd/wshcmd-ssh.go +++ b/cmd/wsh/cmd/wshcmd-ssh.go @@ -8,6 +8,7 @@ import ( "github.com/spf13/cobra" "github.com/wavetermdev/waveterm/pkg/waveobj" + "github.com/wavetermdev/waveterm/pkg/wconfig" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" ) @@ -41,7 +42,7 @@ func sshRun(cmd *cobra.Command, args []string) (rtnErr error) { connOpts := wshrpc.ConnRequest{ Host: sshArg, LogBlockId: blockId, - Keywords: wshrpc.ConnKeywords{ + Keywords: wconfig.ConnKeywords{ SshIdentityFile: identityFiles, }, } diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index c1c13e019..51fc1f90d 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -5,7 +5,6 @@ import { Button } from "@/app/element/button"; import { Input } from "@/app/element/input"; import { ContextMenuModel } from "@/app/store/contextmenu"; import { PLATFORM, atoms, createBlock, getApi, globalStore } from "@/app/store/global"; -import { FileService } from "@/app/store/services"; import type { PreviewModel } from "@/app/view/preview/preview"; import { checkKeyPressed, isCharacterKeyEvent } from "@/util/keyutil"; import { base64ToString, fireAndForget, isBlank } from "@/util/util"; diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 523cbf18b..18ba2f9a3 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -8,6 +8,7 @@ package wshclient import ( "github.com/wavetermdev/waveterm/pkg/wshutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wconfig" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" "github.com/wavetermdev/waveterm/pkg/vdom" From 4022182b5eefcd4604a27ad6742474ac76a5d853 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 15:11:24 -0800 Subject: [PATCH 024/126] more plumbing --- cmd/wsh/cmd/wshcmd-file.go | 127 ++++++++++---------------- emain/emain.ts | 4 +- frontend/app/store/wshclientapi.ts | 61 +++---------- frontend/util/util.ts | 5 + pkg/remote/connparse/connparse.go | 9 +- pkg/remote/fileshare/fileshare.go | 16 ++++ pkg/remote/fileshare/fstype/fstype.go | 2 + pkg/remote/fileshare/s3fs/s3fs.go | 4 + pkg/remote/fileshare/wavefs/wavefs.go | 18 ++++ pkg/remote/fileshare/wshfs/wshfs.go | 8 ++ pkg/util/colprint/colprint.go | 63 +++++++++++++ pkg/wshrpc/wshserver/wshserver.go | 6 +- 12 files changed, 189 insertions(+), 134 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 8bc45e213..6accefd79 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -6,11 +6,12 @@ package cmd import ( "bufio" "bytes" + "context" "encoding/base64" "fmt" "io" "io/fs" - "net/url" + "log" "os" "path" "path/filepath" @@ -18,6 +19,7 @@ import ( "time" "github.com/spf13/cobra" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/util/colprint" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/waveobj" @@ -66,36 +68,6 @@ type waveFileRef struct { fileName string } -func parseWaveFileURL(fileURL string) (*waveFileRef, error) { - if !strings.HasPrefix(fileURL, WaveFilePrefix) { - return nil, fmt.Errorf("invalid file reference %q: must use wavefile:// URL format", fileURL) - } - - u, err := url.Parse(fileURL) - if err != nil { - return nil, fmt.Errorf("invalid wavefile URL: %w", err) - } - - if u.Scheme != WaveFileScheme { - return nil, fmt.Errorf("invalid URL scheme %q: must be wavefile://", u.Scheme) - } - - // Path must start with / - if !strings.HasPrefix(u.Path, "/") { - return nil, fmt.Errorf("invalid wavefile URL: path must start with /") - } - - // Must have a host (zone) - if u.Host == "" { - return nil, fmt.Errorf("invalid wavefile URL: must specify zone (e.g., wavefile://block/file.txt)") - } - - return &waveFileRef{ - zoneId: u.Host, - fileName: strings.TrimPrefix(u.Path, "/"), - }, nil -} - func resolveWaveFile(ref *waveFileRef) (*waveobj.ORef, error) { return resolveSimpleId(ref.zoneId) } @@ -424,7 +396,7 @@ func copyFromLocalToWave(src, dst string) error { return nil } -func filePrintColumns(filesChan <-chan fileListResult) error { +func filePrintColumns(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) error { width := 80 // default if we can't get terminal if w, _, err := term.GetSize(int(os.Stdout.Fd())); err == nil { width = w @@ -435,37 +407,35 @@ func filePrintColumns(filesChan <-chan fileListResult) error { numCols = 1 } - return colprint.PrintColumns( + return colprint.PrintColumnsArray( filesChan, numCols, 100, // sample size - func(f fileListResult) (string, error) { - if f.err != nil { - return "", f.err + func(respUnion wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) ([]string, error) { + if respUnion.Error != nil { + return []string{}, respUnion.Error } - return f.info.Name, nil + strs := make([]string, len(respUnion.Response.FileInfo)) + for i, f := range respUnion.Response.FileInfo { + strs[i] = f.Name + } + return strs, nil }, os.Stdout, ) } -func filePrintLong(filesChan <-chan fileListResult) error { +func filePrintLong(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) error { // Sample first 100 files to determine name width maxNameLen := 0 var samples []*wshrpc.FileInfo - for f := range filesChan { - if f.err != nil { - return f.err - } - samples = append(samples, f.info) - if len(f.info.Name) > maxNameLen { - maxNameLen = len(f.info.Name) - } - - if len(samples) >= 100 { - break + for respUnion := range filesChan { + if respUnion.Error != nil { + return respUnion.Error } + resp := respUnion.Response + samples = append(samples, resp.FileInfo...) } // Use sampled width, but cap it at 60 chars to prevent excessive width @@ -487,16 +457,19 @@ func filePrintLong(filesChan <-chan fileListResult) error { } // Continue with remaining files - for f := range filesChan { - if f.err != nil { - return f.err + for respUnion := range filesChan { + if respUnion.Error != nil { + return respUnion.Error } - name := f.info.Name - timestamp := time.Unix(f.info.ModTime/1000, 0).Format("Jan 02 15:04") - if f.info.Size == 0 && strings.HasSuffix(name, "/") { - fmt.Fprintf(os.Stdout, "%-*s %8s %s\n", nameWidth, name, "-", timestamp) - } else { - fmt.Fprintf(os.Stdout, "%-*s %8d %s\n", nameWidth, name, f.info.Size, timestamp) + for _, f := range respUnion.Response.FileInfo { + name := f.Name + t := time.Unix(f.ModTime/1000, 0) + timestamp := utilfn.FormatLsTime(t) + if f.Size == 0 && strings.HasSuffix(name, "/") { + fmt.Fprintf(os.Stdout, "%-*s %8s %s\n", nameWidth, name, "-", timestamp) + } else { + fmt.Fprintf(os.Stdout, "%-*s %8d %s\n", nameWidth, name, f.Size, timestamp) + } } } @@ -507,7 +480,6 @@ func fileListRun(cmd *cobra.Command, args []string) error { recursive, _ := cmd.Flags().GetBool("recursive") longForm, _ := cmd.Flags().GetBool("long") onePerLine, _ := cmd.Flags().GetBool("one") - filesOnly, _ := cmd.Flags().GetBool("files") // Check if we're in a pipe stat, _ := os.Stdout.Stat() @@ -516,24 +488,12 @@ func fileListRun(cmd *cobra.Command, args []string) error { onePerLine = true } - // Default to listing everything if no path specified - if len(args) == 0 { - args = append(args, "wavefile://client/") - } - - ref, err := parseWaveFileURL(args[0]) - if err != nil { - return err - } - - fullORef, err := resolveWaveFile(ref) - if err != nil { - return err - } - - filesChan, err := streamFileList(fullORef.OID, ref.fileName, recursive, filesOnly) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(fileTimeout)*time.Millisecond) + defer cancel() + filesChan, err := fileshare.ListEntriesStream(ctx, args[0], &wshrpc.FileListOpts{All: recursive}) + log.Printf("listEntriesStream: %v", err) if err != nil { - return err + return fmt.Errorf("listing files: %w", err) } if longForm { @@ -541,13 +501,18 @@ func fileListRun(cmd *cobra.Command, args []string) error { } if onePerLine { - for f := range filesChan { - if f.err != nil { - return f.err + log.Printf("onePerLine") + for respUnion := range filesChan { + if respUnion.Error != nil { + log.Printf("error: %v", respUnion.Error) + return respUnion.Error + } + for _, f := range respUnion.Response.FileInfo { + log.Printf("file: %s", f.Name) + fmt.Fprintln(os.Stdout, f.Name) } - fmt.Fprintln(os.Stdout, f.info.Name) + return nil } - return nil } return filePrintColumns(filesChan) diff --git a/emain/emain.ts b/emain/emain.ts index 3cd3eb0ba..42c8ed2c3 100644 --- a/emain/emain.ts +++ b/emain/emain.ts @@ -14,7 +14,7 @@ import * as services from "../frontend/app/store/services"; import { initElectronWshrpc, shutdownWshrpc } from "../frontend/app/store/wshrpcutil"; import { getWebServerEndpoint } from "../frontend/util/endpoints"; import * as keyutil from "../frontend/util/keyutil"; -import { fireAndForget } from "../frontend/util/util"; +import { fireAndForget, sleep } from "../frontend/util/util"; import { AuthKey, configureAuthKeyRequestInjection } from "./authkey"; import { initDocsite } from "./docsite"; import { @@ -588,6 +588,8 @@ async function appMain() { console.log("wavesrv ready signal received", ready, Date.now() - startTs, "ms"); await electronApp.whenReady(); configureAuthKeyRequestInjection(electron.session.defaultSession); + + await sleep(10); // wait a bit for wavesrv to be ready try { initElectronWshClient(); initElectronWshrpc(ElectronWshClient, { authKey: AuthKey }); diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 8302f5ba4..5d42826d5 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -63,11 +63,7 @@ class RpcApiType { } // command "controllerappendoutput" [call] - ControllerAppendOutputCommand( - client: WshClient, - data: CommandControllerAppendOutputData, - opts?: RpcOpts - ): Promise { + ControllerAppendOutputCommand(client: WshClient, data: CommandControllerAppendOutputData, opts?: RpcOpts): Promise { return client.wshRpcCall("controllerappendoutput", data, opts); } @@ -122,11 +118,7 @@ class RpcApiType { } // command "eventreadhistory" [call] - EventReadHistoryCommand( - client: WshClient, - data: CommandEventReadHistoryData, - opts?: RpcOpts - ): Promise { + EventReadHistoryCommand(client: WshClient, data: CommandEventReadHistoryData, opts?: RpcOpts): Promise { return client.wshRpcCall("eventreadhistory", data, opts); } @@ -266,11 +258,7 @@ class RpcApiType { } // command "remotelistentries" [responsestream] - RemoteListEntriesCommand( - client: WshClient, - data: CommandRemoteListEntriesData, - opts?: RpcOpts - ): AsyncGenerator { + RemoteListEntriesCommand(client: WshClient, data: CommandRemoteListEntriesData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotelistentries", data, opts); } @@ -280,16 +268,12 @@ class RpcApiType { } // command "remotestreamcpudata" [responsestream] - RemoteStreamCpuDataCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { + RemoteStreamCpuDataCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotestreamcpudata", null, opts); } // command "remotestreamfile" [responsestream] - RemoteStreamFileCommand( - client: WshClient, - data: CommandRemoteStreamFileData, - opts?: RpcOpts - ): AsyncGenerator { + RemoteStreamFileCommand(client: WshClient, data: CommandRemoteStreamFileData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotestreamfile", data, opts); } @@ -299,11 +283,7 @@ class RpcApiType { } // command "resolveids" [call] - ResolveIdsCommand( - client: WshClient, - data: CommandResolveIdsData, - opts?: RpcOpts - ): Promise { + ResolveIdsCommand(client: WshClient, data: CommandResolveIdsData, opts?: RpcOpts): Promise { return client.wshRpcCall("resolveids", data, opts); } @@ -343,25 +323,17 @@ class RpcApiType { } // command "streamcpudata" [responsestream] - StreamCpuDataCommand( - client: WshClient, - data: CpuDataRequest, - opts?: RpcOpts - ): AsyncGenerator { + StreamCpuDataCommand(client: WshClient, data: CpuDataRequest, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("streamcpudata", data, opts); } // command "streamtest" [responsestream] - StreamTestCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { + StreamTestCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("streamtest", null, opts); } // command "streamwaveai" [responsestream] - StreamWaveAiCommand( - client: WshClient, - data: WaveAIStreamRequest, - opts?: RpcOpts - ): AsyncGenerator { + StreamWaveAiCommand(client: WshClient, data: WaveAIStreamRequest, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("streamwaveai", data, opts); } @@ -381,20 +353,12 @@ class RpcApiType { } // command "vdomrender" [responsestream] - VDomRenderCommand( - client: WshClient, - data: VDomFrontendUpdate, - opts?: RpcOpts - ): AsyncGenerator { + VDomRenderCommand(client: WshClient, data: VDomFrontendUpdate, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("vdomrender", data, opts); } // command "vdomurlrequest" [responsestream] - VDomUrlRequestCommand( - client: WshClient, - data: VDomUrlRequestData, - opts?: RpcOpts - ): AsyncGenerator { + VDomUrlRequestCommand(client: WshClient, data: VDomUrlRequestData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("vdomurlrequest", data, opts); } @@ -419,7 +383,7 @@ class RpcApiType { } // command "wshactivity" [call] - WshActivityCommand(client: WshClient, data: { [key: string]: number }, opts?: RpcOpts): Promise { + WshActivityCommand(client: WshClient, data: {[key: string]: number}, opts?: RpcOpts): Promise { return client.wshRpcCall("wshactivity", data, opts); } @@ -437,6 +401,7 @@ class RpcApiType { WslStatusCommand(client: WshClient, opts?: RpcOpts): Promise { return client.wshRpcCall("wslstatus", null, opts); } + } export const RpcApi = new RpcApiType(); diff --git a/frontend/util/util.ts b/frontend/util/util.ts index 7bd20981f..a37575887 100644 --- a/frontend/util/util.ts +++ b/frontend/util/util.ts @@ -302,6 +302,10 @@ function makeConnRoute(conn: string): string { return "conn:" + conn; } +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + export { atomWithDebounce, atomWithThrottle, @@ -321,6 +325,7 @@ export { makeConnRoute, makeExternLink, makeIconClass, + sleep, stringToBase64, useAtomValueSafe, }; diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index c1f2323e6..56a604959 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -12,6 +12,9 @@ const ( ConnectionTypeWsh = "wsh" ConnectionTypeS3 = "s3" ConnectionTypeWave = "wavefile" + + ConnHostLocal = "local" + ConnHostWaveSrv = "wavesrv" ) type Connection struct { @@ -58,7 +61,7 @@ func ParseURI(uri string) (*Connection, error) { rest = split[0] } if scheme == "" { - scheme = "wsh" + scheme = ConnectionTypeWsh } var host string @@ -74,10 +77,10 @@ func ParseURI(uri string) (*Connection, error) { path = "/" } } else if strings.HasPrefix(rest, "/~") { - host = "local" + host = ConnHostWaveSrv path = strings.TrimPrefix(rest, "/") } else if stat, _ := os.Stat(rest); stat != nil { - host = "current" + host = ConnHostLocal path = rest } else { split = strings.SplitN(rest, "/", 2) diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index f14a36ae2..9a000076b 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -12,6 +12,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshutil" ) // CreateFileShareClient creates a fileshare client based on the connection string @@ -22,6 +23,13 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS log.Printf("error parsing connection: %v", err) return nil, nil } + if conn.Host == connparse.ConnHostLocal { + handler := wshutil.GetRpcResponseHandlerFromContext(ctx) + if handler == nil { + conn.Host = connparse.ConnHostWaveSrv + } + conn.Host = handler.GetSource() + } conntype := conn.GetType() if conntype == connparse.ConnectionTypeS3 { config, err := awsconn.GetConfig(ctx, connection) @@ -53,6 +61,14 @@ func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([ return client.ListEntries(ctx, conn, opts) } +func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) (<-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], error) { + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + } + return client.ListEntriesStream(ctx, conn, opts), nil +} + func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 0347a0a9f..5376d6955 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -14,6 +14,8 @@ type FileShareClient interface { Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) // ListEntries returns the list of entries at the given path, or nothing if the path is a file ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) + // ListEntriesStream returns a stream of entries at the given path + ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] // PutFile writes the given data to the file at the given path PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error // Mkdir creates a directory at the given path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index a4aaee8f9..f98275492 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -33,6 +33,10 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o return nil, nil } +func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + return nil +} + func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { return nil, nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index bd39f3410..8974958c2 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -19,6 +19,8 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshrpc" ) +const DirChunkSize = 128 + type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -57,6 +59,22 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w return &wshrpc.FileData{Info: data.Info, Entries: list}, nil } +func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) + go func() { + defer close(ch) + list, err := c.ListEntries(ctx, conn, opts) + if err != nil { + ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Error: err} + return + } + for i := 0; i < len(list); i += DirChunkSize { + ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: list[i:min(i+DirChunkSize, len(list))]}} + } + }() + return ch +} + func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { zoneId := conn.Host if zoneId == "" { diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index f143c1777..8b150067a 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "fmt" "io" + "log" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -88,6 +89,7 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt var entries []*wshrpc.FileInfo if opts.All || data == nil { rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + log.Printf("listEntriesInternal: remote list entries") for respUnion := range rtnCh { if respUnion.Error != nil { return nil, respUnion.Error @@ -95,6 +97,7 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt resp := respUnion.Response entries = append(entries, resp.FileInfo...) } + log.Printf("listEntriesInternal: remote list entries done") } else { entries = append(entries, data.Entries...) if opts.Offset > 0 { @@ -113,6 +116,11 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt return entries, nil } +func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + client := wshclient.GetBareRpcClient() + return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) +} + func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { client := wshclient.GetBareRpcClient() return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) diff --git a/pkg/util/colprint/colprint.go b/pkg/util/colprint/colprint.go index 2fe78e424..91445d123 100644 --- a/pkg/util/colprint/colprint.go +++ b/pkg/util/colprint/colprint.go @@ -11,6 +11,69 @@ import ( // formatFn is a function that converts a value of type T to its string representation type formatFn[T any] func(T) (string, error) +type formatFnArray[T any] func(T) ([]string, error) + +func PrintColumnsArray[T any](values <-chan T, numCols int, sampleSize int, format formatFnArray[T], w io.Writer) error { + // Get first batch and determine column width + maxLen := 0 + var samples []T + + for v := range values { + samples = append(samples, v) + str, err := format(v) + if err != nil { + return err + } + for _, s := range str { + if len(s) > maxLen { + maxLen = len(s) + } + } + if len(samples) >= sampleSize { + break + } + } + + colWidth := maxLen + 2 // Add minimum padding + if colWidth < 1 { + colWidth = 1 + } + + // Print in columns using our determined width + col := 0 + for _, v := range samples { + str, err := format(v) + if err != nil { + return err + } + for _, s := range str { + if err := printColHelper(s, colWidth, &col, numCols, w); err != nil { + return err + } + } + } + + // Continue with any remaining values + for v := range values { + str, err := format(v) + if err != nil { + return err + } + for _, s := range str { + if err := printColHelper(s, colWidth, &col, numCols, w); err != nil { + return err + } + } + } + + if col > 0 { + if _, err := fmt.Fprint(w, "\n"); err != nil { + return err + } + } + return nil +} + // PrintColumns prints values in columns, adapting to long values by letting them span multiple columns func PrintColumns[T any](values <-chan T, numCols int, sampleSize int, format formatFn[T], w io.Writer) error { // Get first batch and determine column width diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 6d3bad95b..fca5419ae 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -305,6 +305,10 @@ func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.FileListDa return fileshare.ListEntries(ctx, data.Path, data.Opts) } +func (ws *WshServer) FileListStreamCommand(ctx context.Context, data wshrpc.FileListData) (<-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], error) { + return fileshare.ListEntriesStream(ctx, data.Path, data.Opts) +} + func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.FileData) error { return fileshare.PutFile(ctx, data) } @@ -567,7 +571,7 @@ func (ws *WshServer) ConnUpdateWshCommand(ctx context.Context, remoteInfo wshrpc if err != nil { return false, fmt.Errorf("error parsing connection name: %w", err) } - conn := conncontroller.GetConn(ctx, connOpts, false, &wshrpc.ConnKeywords{}) + conn := conncontroller.GetConn(ctx, connOpts, false, &wconfig.ConnKeywords{}) if conn == nil { return false, fmt.Errorf("connection not found: %s", connName) } From 098730ca6fac85d82cc52004f925bda81b921014 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 18:01:26 -0800 Subject: [PATCH 025/126] save --- cmd/wsh/cmd/wshcmd-file.go | 80 +++++++++---- cmd/wsh/cmd/wshcmd-setbg.go | 4 +- frontend/app/store/wshclientapi.ts | 5 + pkg/remote/connparse/connparse.go | 47 ++++---- pkg/remote/connparse/connparse_test.go | 124 +++++++++++++++------ pkg/remote/fileshare/fileshare.go | 11 +- pkg/remote/fileshare/wshfs/wshfs.go | 26 +++-- pkg/util/fileutil/fileutil.go | 87 +++++++++++++++ pkg/util/{utilfn => fileutil}/mimetypes.go | 2 +- pkg/util/utilfn/utilfn.go | 61 ---------- pkg/wshrpc/wshclient/wshclient.go | 5 + pkg/wshrpc/wshremote/wshremote.go | 3 +- pkg/wshrpc/wshrpctypes.go | 3 + pkg/wshrpc/wshserver/wshserver.go | 2 +- pkg/wshutil/wshrouter.go | 38 ++++++- 15 files changed, 340 insertions(+), 158 deletions(-) create mode 100644 pkg/util/fileutil/fileutil.go rename pkg/util/{utilfn => fileutil}/mimetypes.go (99%) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 6accefd79..ace03c2d8 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -6,7 +6,6 @@ package cmd import ( "bufio" "bytes" - "context" "encoding/base64" "fmt" "io" @@ -19,8 +18,9 @@ import ( "time" "github.com/spf13/cobra" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/util/colprint" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -138,16 +138,19 @@ Exactly one of source or destination must be a wavefile:// URL.`, } func fileCatRun(cmd *cobra.Command, args []string) error { - + path, err := fixRelativePaths(args[0]) + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ - Path: args[0]}} + Path: path}} // Get file info first to check existence and get size info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { - return fmt.Errorf("%s: no such file", args[0]) + return fmt.Errorf("%s: no such file", path) } if err != nil { return fmt.Errorf("getting file info: %w", err) @@ -162,14 +165,18 @@ func fileCatRun(cmd *cobra.Command, args []string) error { } func fileInfoRun(cmd *cobra.Command, args []string) error { + path, err := fixRelativePaths(args[0]) + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ - Path: args[0]}} + Path: path}} info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { - return fmt.Errorf("%s: no such file", args[0]) + return fmt.Errorf("%s: no such file", path) } if err != nil { return fmt.Errorf("getting file info: %w", err) @@ -188,14 +195,18 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { } func fileRmRun(cmd *cobra.Command, args []string) error { + path, err := fixRelativePaths(args[0]) + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ - Path: args[0]}} + Path: path}} - _, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + _, err = wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { - return fmt.Errorf("%s: no such file", args[0]) + return fmt.Errorf("%s: no such file", path) } if err != nil { return fmt.Errorf("getting file info: %w", err) @@ -210,11 +221,15 @@ func fileRmRun(cmd *cobra.Command, args []string) error { } func fileWriteRun(cmd *cobra.Command, args []string) error { + path, err := fixRelativePaths(args[0]) + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ - Path: args[0]}} + Path: path}} - _, err := ensureFile(args[0], fileData) + _, err = ensureFile(path, fileData) if err != nil { return err } @@ -228,11 +243,15 @@ func fileWriteRun(cmd *cobra.Command, args []string) error { } func fileAppendRun(cmd *cobra.Command, args []string) error { + path, err := fixRelativePaths(args[0]) + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ - Path: args[0]}} + Path: path}} - info, err := ensureFile(args[0], fileData) + info, err := ensureFile(path, fileData) if err != nil { return err } @@ -332,9 +351,13 @@ func fileCpRun(cmd *cobra.Command, args []string) error { } func copyFromWaveToLocal(src, dst string) error { + path, err := fixRelativePaths(src) + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ - Path: src}} + Path: path}} // Get file info first to check existence and get size info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) @@ -362,9 +385,13 @@ func copyFromWaveToLocal(src, dst string) error { } func copyFromLocalToWave(src, dst string) error { + path, err := fixRelativePaths(dst) + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ - Path: dst}} + Path: path}} // stat local file stat, err := os.Stat(src) @@ -488,14 +515,14 @@ func fileListRun(cmd *cobra.Command, args []string) error { onePerLine = true } - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(fileTimeout)*time.Millisecond) - defer cancel() - filesChan, err := fileshare.ListEntriesStream(ctx, args[0], &wshrpc.FileListOpts{All: recursive}) - log.Printf("listEntriesStream: %v", err) + path, err := fixRelativePaths(args[0]) if err != nil { - return fmt.Errorf("listing files: %w", err) + return err } + filesChan := wshclient.FileListStreamCommand(RpcClient, wshrpc.FileListData{Path: path, Opts: &wshrpc.FileListOpts{All: recursive}}, &wshrpc.RpcOpts{Timeout: 2000}) + log.Printf("list entries stream") + if longForm { return filePrintLong(filesChan) } @@ -517,3 +544,14 @@ func fileListRun(cmd *cobra.Command, args []string) error { return filePrintColumns(filesChan) } + +func fixRelativePaths(path string) (string, error) { + conn, err := connparse.ParseURI(path) + if err != nil { + return "", err + } + if conn.Scheme == connparse.ConnectionTypeWsh && conn.Host == connparse.ConnHostCurrent { + return fileutil.FixPath(conn.Path) + } + return path, nil +} diff --git a/cmd/wsh/cmd/wshcmd-setbg.go b/cmd/wsh/cmd/wshcmd-setbg.go index 5c05feeaf..9cc35a528 100644 --- a/cmd/wsh/cmd/wshcmd-setbg.go +++ b/cmd/wsh/cmd/wshcmd-setbg.go @@ -12,7 +12,7 @@ import ( "strings" "github.com/spf13/cobra" - "github.com/wavetermdev/waveterm/pkg/util/utilfn" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" @@ -131,7 +131,7 @@ func setBgRun(cmd *cobra.Command, args []string) (rtnErr error) { return fmt.Errorf("path is a directory, not an image file") } - mimeType := utilfn.DetectMimeType(absPath, fileInfo, true) + mimeType := fileutil.DetectMimeType(absPath, fileInfo, true) switch mimeType { case "image/jpeg", "image/png", "image/gif", "image/webp", "image/svg+xml": // Valid image type diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 5d42826d5..6298e2120 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -172,6 +172,11 @@ class RpcApiType { return client.wshRpcCall("filelist", data, opts); } + // command "fileliststream" [responsestream] + FileListStreamCommand(client: WshClient, data: FileListData, opts?: RpcOpts): AsyncGenerator { + return client.wshRpcStream("fileliststream", data, opts); + } + // command "fileread" [call] FileReadCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("fileread", data, opts); diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index 56a604959..f51fa3d4d 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -4,8 +4,9 @@ package connparse import ( - "os" "strings" + + "github.com/wavetermdev/waveterm/pkg/wshutil" ) const ( @@ -13,7 +14,7 @@ const ( ConnectionTypeS3 = "s3" ConnectionTypeWave = "wavefile" - ConnHostLocal = "local" + ConnHostCurrent = "current" ConnHostWaveSrv = "wavesrv" ) @@ -60,42 +61,42 @@ func ParseURI(uri string) (*Connection, error) { } else { rest = split[0] } - if scheme == "" { - scheme = ConnectionTypeWsh - } var host string - var path string - if strings.HasPrefix(rest, "//") { - rest = strings.TrimPrefix(rest, "//") - split = strings.SplitN(rest, "/", 2) - if len(split) > 1 { - host = split[0] - path = "/" + split[1] + var remotePath string + if scheme == "" { + scheme = ConnectionTypeWsh + if strings.HasPrefix(rest, "//") { + rest = strings.TrimPrefix(rest, "//") + split = strings.SplitN(rest, "/", 2) + if len(split) > 1 { + host = split[0] + remotePath = "/" + split[1] + } else { + host = split[0] + remotePath = "/" + } + } else if strings.HasPrefix(rest, "/~") { + host = wshutil.DefaultRoute + remotePath = strings.TrimPrefix(rest, "/") } else { - host = split[0] - path = "/" + host = ConnHostCurrent + remotePath = rest } - } else if strings.HasPrefix(rest, "/~") { - host = ConnHostWaveSrv - path = strings.TrimPrefix(rest, "/") - } else if stat, _ := os.Stat(rest); stat != nil { - host = ConnHostLocal - path = rest } else { split = strings.SplitN(rest, "/", 2) if len(split) > 1 { host = split[0] - path = "/" + split[1] + remotePath = "/" + split[1] } else { host = split[0] - path = "/" + remotePath = "/" } } return &Connection{ Scheme: scheme, Host: host, - Path: path, + Path: remotePath, }, nil } diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index ca697f823..8d91970be 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -6,9 +6,11 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/connparse" ) -func TestParseURI_BasicWSH(t *testing.T) { +func TestParseURI_WSHWithScheme(t *testing.T) { t.Parallel() - cstr := "wsh://localhost:8080/path/to/file" + + // Test with localhost + cstr := "wsh://user@localhost:8080/path/to/file" c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) @@ -17,11 +19,11 @@ func TestParseURI_BasicWSH(t *testing.T) { if c.Path != expected { t.Fatalf("expected path to be %q, got %q", expected, c.Path) } - expected = "localhost:8080" + expected = "user@localhost:8080" if c.Host != expected { t.Fatalf("expected host to be %q, got %q", expected, c.Host) } - expected = "localhost:8080/path/to/file" + expected = "user@localhost:8080/path/to/file" pathWithHost := c.GetPathWithHost() if pathWithHost != expected { t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) @@ -33,16 +35,14 @@ func TestParseURI_BasicWSH(t *testing.T) { if len(c.GetSchemeParts()) != 1 { t.Fatalf("expected scheme parts to be 1, got %d", len(c.GetSchemeParts())) } -} -func TestParseURI_FullConnectionWSH(t *testing.T) { - t.Parallel() - cstr := "wsh://user@192.168.0.1:22/path/to/file" - c, err := connparse.ParseURI(cstr) + // Test with an IP address + cstr = "wsh://user@192.168.0.1:22/path/to/file" + c, err = connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } - expected := "/path/to/file" + expected = "/path/to/file" if c.Path != expected { t.Fatalf("expected path to be %q, got %q", expected, c.Path) } @@ -51,7 +51,7 @@ func TestParseURI_FullConnectionWSH(t *testing.T) { t.Fatalf("expected host to be %q, got %q", expected, c.Host) } expected = "user@192.168.0.1:22/path/to/file" - pathWithHost := c.GetPathWithHost() + pathWithHost = c.GetPathWithHost() if pathWithHost != expected { t.Fatalf("expected path with host to be %q, got %q", expected, pathWithHost) } @@ -68,9 +68,11 @@ func TestParseURI_FullConnectionWSH(t *testing.T) { } } -func TestParseURI_MissingScheme(t *testing.T) { +func TestParseURI_WSHRemoteShorthand(t *testing.T) { t.Parallel() - cstr := "localhost:8080/path/to/file" + + // Test with a simple remote path + cstr := "//conn/path/to/file" c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) @@ -79,7 +81,29 @@ func TestParseURI_MissingScheme(t *testing.T) { if c.Path != expected { t.Fatalf("expected path to be %q, got %q", expected, c.Path) } - expected = "localhost:8080" + if c.Host != "conn" { + t.Fatalf("expected host to be empty, got %q", c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } + expected = "wsh://conn/path/to/file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } + + // Test with a complex remote path + cstr = "//user@localhost:8080/path/to/file" + c, err = connparse.ParseURI(cstr) + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected = "/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "user@localhost:8080" if c.Host != expected { t.Fatalf("expected host to be %q, got %q", expected, c.Host) } @@ -87,31 +111,40 @@ func TestParseURI_MissingScheme(t *testing.T) { if c.Scheme != expected { t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) } -} + expected = "wsh://user@localhost:8080/path/to/file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } -func TestParseURI_WSHShorthand(t *testing.T) { - t.Parallel() - cstr := "//conn/path/to/file" - c, err := connparse.ParseURI(cstr) + // Test with an IP address + cstr = "//user@192.168.0.1:8080/path/to/file" + c, err = connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } - expected := "/path/to/file" + expected = "/path/to/file" if c.Path != expected { t.Fatalf("expected path to be %q, got %q", expected, c.Path) } - if c.Host != "conn" { - t.Fatalf("expected host to be empty, got %q", c.Host) + expected = "user@192.168.0.1:8080" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) } expected = "wsh" if c.Scheme != expected { t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) } + expected = "wsh://user@192.168.0.1:8080/path/to/file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } } -func TestParseURI_WSHLocalHomeShorthand(t *testing.T) { +func TestParseURI_WSHCurrentPathShorthand(t *testing.T) { t.Parallel() - cstr := "/~/path/to/file" + + // Test with a relative path to home + cstr := "~/path/to/file" c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) @@ -120,29 +153,56 @@ func TestParseURI_WSHLocalHomeShorthand(t *testing.T) { if c.Path != expected { t.Fatalf("expected path to be %q, got %q", expected, c.Path) } - if c.Host != "local" { - t.Fatalf("expected host to be empty, got %q", c.Host) + expected = "current" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) } expected = "wsh" if c.Scheme != expected { t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) } + expected = "wsh://current/~/path/to/file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } + + // Test with a absolute path + cstr = "/path/to/file" + c, err = connparse.ParseURI(cstr) + if err != nil { + t.Fatalf("expected nil, got %v", err) + } + expected = "/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "current" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } + expected = "wsh://current/path/to/file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } } -func TestParseURI_WSHCurrentAbsolutePath(t *testing.T) { +func TestParseURI_WSHLocalShorthand(t *testing.T) { t.Parallel() - cstr := t.TempDir() + cstr := "/~/path/to/file" c, err := connparse.ParseURI(cstr) if err != nil { t.Fatalf("failed to parse URI: %v", err) } - expected := cstr + expected := "~/path/to/file" if c.Path != expected { t.Fatalf("expected path to be %q, got %q", expected, c.Path) } - expected = "current" - if c.Host != expected { - t.Fatalf("expected host to be %q, got %q", expected, c.Host) + if c.Host != "wavesrv" { + t.Fatalf("expected host to be empty, got %q", c.Host) } expected = "wsh" if c.Scheme != expected { diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 9a000076b..99f3f5315 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -23,7 +23,7 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS log.Printf("error parsing connection: %v", err) return nil, nil } - if conn.Host == connparse.ConnHostLocal { + if conn.Host == connparse.ConnHostCurrent { handler := wshutil.GetRpcResponseHandlerFromContext(ctx) if handler == nil { conn.Host = connparse.ConnHostWaveSrv @@ -61,12 +61,15 @@ func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([ return client.ListEntries(ctx, conn, opts) } -func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) (<-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], error) { +func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) + defer close(rtn) + rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Error: fmt.Errorf("error creating fileshare client, could not parse connection %s", path)} + return rtn } - return client.ListEntriesStream(ctx, conn, opts), nil + return client.ListEntriesStream(ctx, conn, opts) } func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 8b150067a..1159d99c8 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -29,7 +29,7 @@ func NewWshClient() *WshClient { func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { client := wshclient.GetBareRpcClient() streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path} - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) fullFile := &wshrpc.FileData{} firstPk := true isDir := false @@ -88,7 +88,7 @@ func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opts *wshrpc.FileListOpts, data *wshrpc.FileData) ([]*wshrpc.FileInfo, error) { var entries []*wshrpc.FileInfo if opts.All || data == nil { - rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) log.Printf("listEntriesInternal: remote list entries") for respUnion := range rtnCh { if respUnion.Error != nil { @@ -118,28 +118,28 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { client := wshclient.GetBareRpcClient() - return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) } func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) } func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { client := wshclient.GetBareRpcClient() writeData := wshrpc.CommandRemoteWriteFileData{Path: conn.Path, Data64: data.Data64} - return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) } func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteMkdirCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteMkdirCommand(client, conn.Path, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) } func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) + return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: fixRouteId(srcConn.Host)}) } func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { @@ -148,9 +148,19 @@ func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connec func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileDeleteCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteFileDeleteCommand(client, conn.Path, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) } func (c WshClient) GetConnectionType() string { return connparse.ConnectionTypeWsh } + +func fixRouteId(host string) string { + if host == "" { + return wshutil.DefaultRoute + } + if wshutil.IsRouteId(host) { + return host + } + return wshutil.MakeConnectionRouteId(host) +} diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go new file mode 100644 index 000000000..2a4a717bb --- /dev/null +++ b/pkg/util/fileutil/fileutil.go @@ -0,0 +1,87 @@ +package fileutil + +import ( + "io" + "io/fs" + "mime" + "net/http" + "os" + "os/user" + "path/filepath" + "strings" +) + +// on error just returns "" +// does not return "application/octet-stream" as this is considered a detection failure +// can pass an existing fileInfo to avoid re-statting the file +// falls back to text/plain for 0 byte files +func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { + if fileInfo == nil { + statRtn, err := os.Stat(path) + if err != nil { + return "" + } + fileInfo = statRtn + } + if fileInfo.IsDir() { + return "directory" + } + if fileInfo.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { + return "pipe" + } + charDevice := os.ModeDevice | os.ModeCharDevice + if fileInfo.Mode()&charDevice == charDevice { + return "character-special" + } + if fileInfo.Mode()&os.ModeDevice == os.ModeDevice { + return "block-special" + } + ext := filepath.Ext(path) + if mimeType, ok := StaticMimeTypeMap[ext]; ok { + return mimeType + } + if mimeType := mime.TypeByExtension(ext); mimeType != "" { + return mimeType + } + if fileInfo.Size() == 0 { + return "text/plain" + } + if !extended { + return "" + } + fd, err := os.Open(path) + if err != nil { + return "" + } + defer fd.Close() + buf := make([]byte, 512) + // ignore the error (EOF / UnexpectedEOF is fine, just process how much we got back) + n, _ := io.ReadAtLeast(fd, buf, 512) + if n == 0 { + return "" + } + buf = buf[:n] + rtn := http.DetectContentType(buf) + if rtn == "application/octet-stream" { + return "" + } + return rtn +} + +func FixPath(path string) (string, error) { + if strings.HasPrefix(path, "~") { + curUser, err := user.Current() + if err != nil { + return "", err + } + return filepath.Join(curUser.HomeDir, path[1:]), nil + } else if !filepath.IsAbs(path) { + path, err := filepath.Abs(path) + if err != nil { + return "", err + } + return path, nil + } else { + return path, nil + } +} diff --git a/pkg/util/utilfn/mimetypes.go b/pkg/util/fileutil/mimetypes.go similarity index 99% rename from pkg/util/utilfn/mimetypes.go rename to pkg/util/fileutil/mimetypes.go index 688e934ef..cf1b9d069 100644 --- a/pkg/util/utilfn/mimetypes.go +++ b/pkg/util/fileutil/mimetypes.go @@ -1,7 +1,7 @@ // Copyright 2025, Command Line Inc. // SPDX-License-Identifier: Apache-2.0 -package utilfn +package fileutil var StaticMimeTypeMap = map[string]string{ ".a2l": "application/A2L", diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index f70b5b46a..6c7042d86 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -13,14 +13,10 @@ import ( "errors" "fmt" "io" - "io/fs" "math" mathrand "math/rand" - "mime" - "net/http" "os" "os/exec" - "path/filepath" "reflect" "regexp" "sort" @@ -617,63 +613,6 @@ func CopyToChannel(outputCh chan<- []byte, reader io.Reader) error { } } -// on error just returns "" -// does not return "application/octet-stream" as this is considered a detection failure -// can pass an existing fileInfo to avoid re-statting the file -// falls back to text/plain for 0 byte files -func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { - if fileInfo == nil { - statRtn, err := os.Stat(path) - if err != nil { - return "" - } - fileInfo = statRtn - } - if fileInfo.IsDir() { - return "directory" - } - if fileInfo.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { - return "pipe" - } - charDevice := os.ModeDevice | os.ModeCharDevice - if fileInfo.Mode()&charDevice == charDevice { - return "character-special" - } - if fileInfo.Mode()&os.ModeDevice == os.ModeDevice { - return "block-special" - } - ext := filepath.Ext(path) - if mimeType, ok := StaticMimeTypeMap[ext]; ok { - return mimeType - } - if mimeType := mime.TypeByExtension(ext); mimeType != "" { - return mimeType - } - if fileInfo.Size() == 0 { - return "text/plain" - } - if !extended { - return "" - } - fd, err := os.Open(path) - if err != nil { - return "" - } - defer fd.Close() - buf := make([]byte, 512) - // ignore the error (EOF / UnexpectedEOF is fine, just process how much we got back) - n, _ := io.ReadAtLeast(fd, buf, 512) - if n == 0 { - return "" - } - buf = buf[:n] - rtn := http.DetectContentType(buf) - if rtn == "application/octet-stream" { - return "" - } - return rtn -} - func GetCmdExitCode(cmd *exec.Cmd, err error) int { if cmd == nil || cmd.ProcessState == nil { return GetExitCode(err) diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index f023213c9..02616096b 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -212,6 +212,11 @@ func FileListCommand(w *wshutil.WshRpc, data wshrpc.FileListData, opts *wshrpc.R return resp, err } +// command "fileliststream", wshserver.FileListStreamCommand +func FileListStreamCommand(w *wshutil.WshRpc, data wshrpc.FileListData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + return sendRpcRequestResponseStreamHelper[wshrpc.CommandRemoteListEntriesRtnData](w, "fileliststream", data, opts) +} + // command "fileread", wshserver.FileReadCommand func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) (string, error) { resp, err := sendRpcRequestCallHelper[string](w, "fileread", data, opts) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 5c5d7def3..18977bdd4 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -15,6 +15,7 @@ import ( "path/filepath" "strings" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -276,7 +277,7 @@ func listFilesInternal(path string, fileInfoArr []*fs.DirEntry, seen *int, opts } func statToFileInfo(fullPath string, finfo fs.FileInfo, extended bool) *wshrpc.FileInfo { - mimeType := utilfn.DetectMimeType(fullPath, finfo, extended) + mimeType := fileutil.DetectMimeType(fullPath, finfo, extended) rtn := &wshrpc.FileInfo{ Path: wavebase.ReplaceHomeDir(fullPath), Dir: computeDirPart(fullPath, finfo.IsDir()), diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 677b4e0ae..6f003ce50 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -137,6 +137,7 @@ type WshRpcInterface interface { FileReadCommand(ctx context.Context, data FileData) (string, error) FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) FileListCommand(ctx context.Context, data FileListData) ([]*FileInfo, error) + FileListStreamCommand(ctx context.Context, data FileListData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] EventPublishCommand(ctx context.Context, data wps.WaveEvent) error EventSubCommand(ctx context.Context, data wps.SubscriptionRequest) error EventUnsubCommand(ctx context.Context, data string) error @@ -372,6 +373,8 @@ type FileOptsType struct { type FileMeta = map[string]any +type FileListStreamResponse <-chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] + type FileListData struct { Path string `json:"path"` Opts *FileListOpts `json:"opts,omitempty"` diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index fca5419ae..59cbc0233 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -305,7 +305,7 @@ func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.FileListDa return fileshare.ListEntries(ctx, data.Path, data.Opts) } -func (ws *WshServer) FileListStreamCommand(ctx context.Context, data wshrpc.FileListData) (<-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], error) { +func (ws *WshServer) FileListStreamCommand(ctx context.Context, data wshrpc.FileListData) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { return fileshare.ListEntriesStream(ctx, data.Path, data.Opts) } diff --git a/pkg/wshutil/wshrouter.go b/pkg/wshutil/wshrouter.go index fdf1d49df..bc1fe761a 100644 --- a/pkg/wshutil/wshrouter.go +++ b/pkg/wshutil/wshrouter.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "log" + "strings" "sync" "time" @@ -19,10 +20,18 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshrpc" ) -const DefaultRoute = "wavesrv" -const UpstreamRoute = "upstream" -const SysRoute = "sys" // this route doesn't exist, just a placeholder for system messages -const ElectronRoute = "electron" +const ( + DefaultRoute = "wavesrv" + UpstreamRoute = "upstream" + SysRoute = "sys" // this route doesn't exist, just a placeholder for system messages + ElectronRoute = "electron" + + RoutePrefix_Conn = "conn:" + RoutePrefix_Controller = "controller:" + RoutePrefix_Proc = "proc:" + RoutePrefix_Tab = "tab:" + RoutePrefix_FeBlock = "feblock:" +) // this works like a network switch @@ -49,6 +58,27 @@ type WshRouter struct { InputCh chan msgAndRoute } +func IsRouteId(routeId string) bool { + if routeId == "" { + return false + } + if len(routeId) < 5 { + return false + } + if routeId == DefaultRoute || + routeId == UpstreamRoute || + routeId == SysRoute || + routeId == ElectronRoute || + strings.HasPrefix(routeId, RoutePrefix_Conn) || + strings.HasPrefix(routeId, RoutePrefix_Controller) || + strings.HasPrefix(routeId, RoutePrefix_Proc) || + strings.HasPrefix(routeId, RoutePrefix_Tab) || + strings.HasPrefix(routeId, RoutePrefix_FeBlock) { + return true + } + return true +} + func MakeConnectionRouteId(connId string) string { return "conn:" + connId } From 8f863db466729ab2c0efcd1b53559dcd10e0401f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 18:11:38 -0800 Subject: [PATCH 026/126] fixed conn issue --- pkg/remote/connparse/connparse.go | 4 ++-- pkg/remote/connparse/connparse_test.go | 2 +- pkg/remote/fileshare/fileshare.go | 9 +++++++-- pkg/remote/fileshare/wshfs/wshfs.go | 26 ++++++++------------------ 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index f51fa3d4d..b48197ead 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -6,7 +6,7 @@ package connparse import ( "strings" - "github.com/wavetermdev/waveterm/pkg/wshutil" + "github.com/wavetermdev/waveterm/pkg/wshrpc" ) const ( @@ -77,7 +77,7 @@ func ParseURI(uri string) (*Connection, error) { remotePath = "/" } } else if strings.HasPrefix(rest, "/~") { - host = wshutil.DefaultRoute + host = wshrpc.LocalConnName remotePath = strings.TrimPrefix(rest, "/") } else { host = ConnHostCurrent diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index 8d91970be..27ca79422 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -201,7 +201,7 @@ func TestParseURI_WSHLocalShorthand(t *testing.T) { if c.Path != expected { t.Fatalf("expected path to be %q, got %q", expected, c.Path) } - if c.Host != "wavesrv" { + if c.Host != "local" { t.Fatalf("expected host to be empty, got %q", c.Host) } expected = "wsh" diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 99f3f5315..939bc2c92 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -26,9 +26,14 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS if conn.Host == connparse.ConnHostCurrent { handler := wshutil.GetRpcResponseHandlerFromContext(ctx) if handler == nil { - conn.Host = connparse.ConnHostWaveSrv + log.Printf("error getting rpc response handler from context") + return nil, nil + } + source := handler.GetRpcContext().Conn + if source == "" { + source = wshrpc.LocalConnName } - conn.Host = handler.GetSource() + conn.Host = source } conntype := conn.GetType() if conntype == connparse.ConnectionTypeS3 { diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 1159d99c8..8b150067a 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -29,7 +29,7 @@ func NewWshClient() *WshClient { func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { client := wshclient.GetBareRpcClient() streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path} - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) fullFile := &wshrpc.FileData{} firstPk := true isDir := false @@ -88,7 +88,7 @@ func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opts *wshrpc.FileListOpts, data *wshrpc.FileData) ([]*wshrpc.FileInfo, error) { var entries []*wshrpc.FileInfo if opts.All || data == nil { - rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) + rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) log.Printf("listEntriesInternal: remote list entries") for respUnion := range rtnCh { if respUnion.Error != nil { @@ -118,28 +118,28 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { client := wshclient.GetBareRpcClient() - return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) + return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) + return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { client := wshclient.GetBareRpcClient() writeData := wshrpc.CommandRemoteWriteFileData{Path: conn.Path, Data64: data.Data64} - return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) + return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteMkdirCommand(client, conn.Path, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) + return wshclient.RemoteMkdirCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: fixRouteId(srcConn.Host)}) + return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) } func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { @@ -148,19 +148,9 @@ func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connec func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileDeleteCommand(client, conn.Path, &wshrpc.RpcOpts{Route: fixRouteId(conn.Host)}) + return wshclient.RemoteFileDeleteCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) GetConnectionType() string { return connparse.ConnectionTypeWsh } - -func fixRouteId(host string) string { - if host == "" { - return wshutil.DefaultRoute - } - if wshutil.IsRouteId(host) { - return host - } - return wshutil.MakeConnectionRouteId(host) -} From 9de524fa31cb9ab160ac014e87d23d523b36047e Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 18:40:23 -0800 Subject: [PATCH 027/126] cat is working! --- cmd/wsh/cmd/wshcmd-file-util.go | 4 ++-- cmd/wsh/cmd/wshcmd-file.go | 1 + cmd/wsh/cmd/wshcmd-getvar.go | 4 ++-- cmd/wsh/cmd/wshcmd-readfile.go | 4 ++-- frontend/app/store/wshclientapi.ts | 2 +- pkg/remote/fileshare/fileshare.go | 19 +++++++++++++++---- pkg/remote/fileshare/wshfs/wshfs.go | 12 ++++++++++-- pkg/wshrpc/wshclient/wshclient.go | 4 ++-- pkg/wshrpc/wshrpctypes.go | 2 +- pkg/wshrpc/wshserver/wshserver.go | 5 ++--- pkg/wshutil/wshrouter.go | 22 ---------------------- 11 files changed, 38 insertions(+), 41 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index d805f1e6b..41dfbfd3d 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -103,13 +103,13 @@ func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) } // Read the chunk - content64, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) + data, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) if err != nil { return fmt.Errorf("reading chunk at offset %d: %w", offset, err) } // Decode and write the chunk - chunk, err := base64.StdEncoding.DecodeString(content64) + chunk, err := base64.StdEncoding.DecodeString(data.Data64) if err != nil { return fmt.Errorf("decoding chunk at offset %d: %w", offset, err) } diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index ace03c2d8..e70f330fd 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -445,6 +445,7 @@ func filePrintColumns(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRem strs := make([]string, len(respUnion.Response.FileInfo)) for i, f := range respUnion.Response.FileInfo { strs[i] = f.Name + log.Printf("file: %s", f.Name) } return strs, nil }, diff --git a/cmd/wsh/cmd/wshcmd-getvar.go b/cmd/wsh/cmd/wshcmd-getvar.go index 693e1b797..24701c588 100644 --- a/cmd/wsh/cmd/wshcmd-getvar.go +++ b/cmd/wsh/cmd/wshcmd-getvar.go @@ -117,7 +117,7 @@ func getAllVariables(zoneId string) error { Info: &wshrpc.FileInfo{ Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, zoneId, getVarFileName)}} - envStr64, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) + data, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { return nil @@ -125,7 +125,7 @@ func getAllVariables(zoneId string) error { if err != nil { return fmt.Errorf("reading variables: %w", err) } - envBytes, err := base64.StdEncoding.DecodeString(envStr64) + envBytes, err := base64.StdEncoding.DecodeString(data.Data64) if err != nil { return fmt.Errorf("decoding variables: %w", err) } diff --git a/cmd/wsh/cmd/wshcmd-readfile.go b/cmd/wsh/cmd/wshcmd-readfile.go index 652e7d49e..cb7dee442 100644 --- a/cmd/wsh/cmd/wshcmd-readfile.go +++ b/cmd/wsh/cmd/wshcmd-readfile.go @@ -31,12 +31,12 @@ func runReadFile(cmd *cobra.Command, args []string) { WriteStderr("[error] %v\n", err) return } - resp64, err := wshclient.FileReadCommand(RpcClient, wshrpc.FileData{Info: &wshrpc.FileInfo{Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, fullORef.OID, args[0])}}, &wshrpc.RpcOpts{Timeout: 5000}) + data, err := wshclient.FileReadCommand(RpcClient, wshrpc.FileData{Info: &wshrpc.FileInfo{Path: fmt.Sprintf(wavefileutil.WaveFilePathPattern, fullORef.OID, args[0])}}, &wshrpc.RpcOpts{Timeout: 5000}) if err != nil { WriteStderr("[error] reading file: %v\n", err) return } - resp, err := base64.StdEncoding.DecodeString(resp64) + resp, err := base64.StdEncoding.DecodeString(data.Data64) if err != nil { WriteStderr("[error] decoding file: %v\n", err) return diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 6298e2120..4e5ccf6fd 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -178,7 +178,7 @@ class RpcApiType { } // command "fileread" [call] - FileReadCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { + FileReadCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("fileread", data, opts); } diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 939bc2c92..c1c9e86b8 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -18,6 +18,7 @@ import ( // CreateFileShareClient creates a fileshare client based on the connection string // Returns the client and the parsed connection func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileShareClient, *connparse.Connection) { + log.Printf("CreateFileShareClient: connection=%s", connection) conn, err := connparse.ParseURI(connection) if err != nil { log.Printf("error parsing connection: %v", err) @@ -30,6 +31,8 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS return nil, nil } source := handler.GetRpcContext().Conn + + // RPC context connection is empty for local connections if source == "" { source = wshrpc.LocalConnName } @@ -50,15 +53,16 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } } -func Read(ctx context.Context, path string) (*wshrpc.FileData, error) { - client, conn := CreateFileShareClient(ctx, path) +func Read(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { + client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { - return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path) } - return client.Read(ctx, conn, wshrpc.FileData{}) + return client.Read(ctx, conn, data) } func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { + log.Printf("ListEntries: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) @@ -67,6 +71,7 @@ func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([ } func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + log.Printf("ListEntriesStream: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) @@ -78,6 +83,7 @@ func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOp } func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { + log.Printf("Stat: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) @@ -86,6 +92,7 @@ func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { } func PutFile(ctx context.Context, data wshrpc.FileData) error { + log.Printf("PutFile: path=%s", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { return fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path) @@ -98,6 +105,7 @@ func PutFile(ctx context.Context, data wshrpc.FileData) error { } func Mkdir(ctx context.Context, path string) error { + log.Printf("Mkdir: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { return fmt.Errorf("error creating fileshare client, could not parse connection %s", path) @@ -107,6 +115,7 @@ func Mkdir(ctx context.Context, path string) error { // TODO: Implement move across different fileshare types func Move(ctx context.Context, srcPath, destPath string, recursive bool) error { + log.Printf("Move: src=%s, dest=%s", srcPath, destPath) srcClient, srcConn := CreateFileShareClient(ctx, srcPath) if srcConn == nil || srcClient == nil { return fmt.Errorf("error creating fileshare client, could not parse connection %s or %s", srcPath, destPath) @@ -120,10 +129,12 @@ func Move(ctx context.Context, srcPath, destPath string, recursive bool) error { // TODO: Implement copy across different fileshare types func Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { + log.Printf("Copy: src=%s, dest=%s", srcPath, destPath) return nil } func Delete(ctx context.Context, path string) error { + log.Printf("Delete: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { return fmt.Errorf("error creating fileshare client, could not parse connection %s", path) diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 8b150067a..38a17b8fa 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -27,8 +27,13 @@ func NewWshClient() *WshClient { } func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { + log.Printf("WshClient:Read: path=%s; conn=%v; data=%v", conn.Path, conn, data) client := wshclient.GetBareRpcClient() - streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path} + byteRange := "" + if data.At.Size > 0 { + byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+data.At.Size) + } + streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path, ByteRange: byteRange} rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) fullFile := &wshrpc.FileData{} firstPk := true @@ -117,13 +122,16 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt } func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + log.Printf("WshClient:ListEntriesStream: path=%s", conn.Path) client := wshclient.GetBareRpcClient() return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + resp, err := wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + log.Printf("WshClient:Stat: path=%s; resp: %v, err: %v", conn.Path, resp, err) + return resp, err } func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 02616096b..2b11aec65 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -218,8 +218,8 @@ func FileListStreamCommand(w *wshutil.WshRpc, data wshrpc.FileListData, opts *ws } // command "fileread", wshserver.FileReadCommand -func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) (string, error) { - resp, err := sendRpcRequestCallHelper[string](w, "fileread", data, opts) +func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) (*wshrpc.FileData, error) { + resp, err := sendRpcRequestCallHelper[*wshrpc.FileData](w, "fileread", data, opts) return resp, err } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 6f003ce50..872f0653e 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -134,7 +134,7 @@ type WshRpcInterface interface { FileAppendCommand(ctx context.Context, data FileData) error FileAppendIJsonCommand(ctx context.Context, data CommandAppendIJsonData) error FileWriteCommand(ctx context.Context, data FileData) error - FileReadCommand(ctx context.Context, data FileData) (string, error) + FileReadCommand(ctx context.Context, data FileData) (*FileData, error) FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) FileListCommand(ctx context.Context, data FileListData) ([]*FileInfo, error) FileListStreamCommand(ctx context.Context, data FileListData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 59cbc0233..74af1130b 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -313,9 +313,8 @@ func (ws *WshServer) FileWriteCommand(ctx context.Context, data wshrpc.FileData) return fileshare.PutFile(ctx, data) } -func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) (string, error) { - // TODO: implement this - return "", nil +func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { + return fileshare.Read(ctx, data) } func (ws *WshServer) FileAppendIJsonCommand(ctx context.Context, data wshrpc.CommandAppendIJsonData) error { diff --git a/pkg/wshutil/wshrouter.go b/pkg/wshutil/wshrouter.go index bc1fe761a..2a99b19ce 100644 --- a/pkg/wshutil/wshrouter.go +++ b/pkg/wshutil/wshrouter.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "log" - "strings" "sync" "time" @@ -58,27 +57,6 @@ type WshRouter struct { InputCh chan msgAndRoute } -func IsRouteId(routeId string) bool { - if routeId == "" { - return false - } - if len(routeId) < 5 { - return false - } - if routeId == DefaultRoute || - routeId == UpstreamRoute || - routeId == SysRoute || - routeId == ElectronRoute || - strings.HasPrefix(routeId, RoutePrefix_Conn) || - strings.HasPrefix(routeId, RoutePrefix_Controller) || - strings.HasPrefix(routeId, RoutePrefix_Proc) || - strings.HasPrefix(routeId, RoutePrefix_Tab) || - strings.HasPrefix(routeId, RoutePrefix_FeBlock) { - return true - } - return true -} - func MakeConnectionRouteId(connId string) string { return "conn:" + connId } From 96e6725ac0c0917b4f2361d7295ee77f3f95e3c8 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 19:02:41 -0800 Subject: [PATCH 028/126] formatting info --- cmd/wsh/cmd/wshcmd-file.go | 15 ++++++++++----- pkg/remote/fileshare/wshfs/wshfs.go | 5 +---- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index e70f330fd..572cb1cef 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -182,13 +182,18 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("getting file info: %w", err) } - WriteStdout("filename: %s\n", info.Name) - WriteStdout("size: %d\n", info.Size) - WriteStdout("mtime: %s\n", time.Unix(info.ModTime/1000, 0).Format(time.DateTime)) - if len(*info.Meta) > 0 { + WriteStdout("name:\t%s\n", info.Name) + if info.Mode != 0 { + WriteStdout("mode:\t%s\n", info.Mode.String()) + } + WriteStdout("mtime:\t%s\n", time.Unix(info.ModTime/1000, 0).Format(time.DateTime)) + if !info.IsDir { + WriteStdout("size:\t%d\n", info.Size) + } + if info.Meta != nil && len(*info.Meta) > 0 { WriteStdout("metadata:\n") for k, v := range *info.Meta { - WriteStdout(" %s: %v\n", k, v) + WriteStdout("\t\t\t%s: %v\n", k, v) } } return nil diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 38a17b8fa..11ed14ae2 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -122,16 +122,13 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt } func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { - log.Printf("WshClient:ListEntriesStream: path=%s", conn.Path) client := wshclient.GetBareRpcClient() return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { client := wshclient.GetBareRpcClient() - resp, err := wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) - log.Printf("WshClient:Stat: path=%s; resp: %v, err: %v", conn.Path, resp, err) - return resp, err + return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { From 57deb43a9a0679c8d990d920f133603a011cfd29 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 19:55:50 -0800 Subject: [PATCH 029/126] add updated help text --- cmd/wsh/cmd/wshcmd-file.go | 76 ++++++++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 572cb1cef..f20ce936b 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -34,13 +34,40 @@ const ( WaveFilePrefix = "wavefile://" DefaultFileTimeout = 5000 + + UriHelpText = ` + +URI format: [profile]:[uri-scheme]://[connection]/[path] + +Supported URI schemes: + wavefile: + Used to retrieve blockfiles from the internal Wave filesystem. + + Format: wsh://[zoneid]/[path] + wsh: + Used to access files on remotes via the WSH helper. Allows for file streaming to Wave and other remotes. + Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. + + Format: wsh://[remote]/[path] + + Shorthands can be used for the current remote and your local machine: + - [path] is a relative or absolute path on the current remote + - //[remote]/[path] is a path on a remote + - /~/[path] is a path relative to your home directory on your local machine + + s3: + Used to access files on S3. Requires S3 credentials to be set up, either in "profiles.json" or in the AWS CLI. + If no profile is provided, the default from your AWS CLI configuration will be used. Profiles from the AWS CLI must be prefixed with "aws:". + + Format: s3://[bucket]/[path] + aws:[profile]:s3://[bucket]/[path] + [profile]:s3://[bucket]/[path]` ) var fileCmd = &cobra.Command{ Use: "file", Short: "manage Wave Terminal files", - Long: "Commands to manage Wave Terminal files stored in blocks", -} + Long: "Commands to manage Wave Terminal files stored in blocks." + UriHelpText} var fileTimeout int @@ -73,43 +100,47 @@ func resolveWaveFile(ref *waveFileRef) (*waveobj.ORef, error) { } var fileListCmd = &cobra.Command{ - Use: "ls [wavefile://zone[/path]]", - Short: "list wave files", - Example: " wsh file ls wavefile://block/\n wsh file ls wavefile://client/configs/", + Use: "ls [uri]", + Short: "list files", + Long: "List files in a directory. By default, lists files in the current directory." + UriHelpText, + Example: " wsh file ls wsh://user@ec2/home/user/\n wsh file ls wavefile://client/configs/", RunE: activityWrap("file", fileListRun), PreRunE: preRunSetupRpcClient, } var fileCatCmd = &cobra.Command{ - Use: "cat wavefile://zone/file", - Short: "display contents of a wave file", - Example: " wsh file cat wavefile://block/config.txt\n wsh file cat wavefile://client/settings.json", + Use: "cat [uri]", + Short: "display contents of a file", + Long: "Display the contents of a file." + UriHelpText, + Example: " wsh file cat wsh://user@ec2/home/user/config.txt\n wsh file cat wavefile://client/settings.json", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileCatRun), PreRunE: preRunSetupRpcClient, } var fileInfoCmd = &cobra.Command{ - Use: "info wavefile://zone/file", + Use: "info [uri]", Short: "show wave file information", - Example: " wsh file info wavefile://block/config.txt", + Long: "Show information about a file." + UriHelpText, + Example: " wsh file info wsh://user@ec2/home/user/config.txt\n wsh file info wavefile://client/settings.json", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileInfoRun), PreRunE: preRunSetupRpcClient, } var fileRmCmd = &cobra.Command{ - Use: "rm wavefile://zone/file", - Short: "remove a wave file", - Example: " wsh file rm wavefile://block/config.txt", + Use: "rm [uri]", + Short: "remove a file", + Example: " wsh file rm wsh://user@ec2/home/user/config.txt\n wsh file rm wavefile://client/settings.json", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileRmRun), PreRunE: preRunSetupRpcClient, } var fileWriteCmd = &cobra.Command{ - Use: "write wavefile://zone/file", - Short: "write stdin into a wave file (up to 10MB)", + Use: "write [uri]", + Short: "write stdin into a file (up to 10MB)", + Long: "Write stdin into a file, buffering input and respecting 10MB total file size limit." + UriHelpText, Example: " echo 'hello' | wsh file write wavefile://block/greeting.txt", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileWriteRun), @@ -117,9 +148,9 @@ var fileWriteCmd = &cobra.Command{ } var fileAppendCmd = &cobra.Command{ - Use: "append wavefile://zone/file", - Short: "append stdin to a wave file", - Long: "append stdin to a wave file, buffering input and respecting 10MB total file size limit", + Use: "append [uri]", + Short: "append stdin to a file", + Long: "Append stdin to a file, buffering input and respecting 10MB total file size limit" + UriHelpText, Example: " tail -f log.txt | wsh file append wavefile://block/app.log", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileAppendRun), @@ -127,11 +158,10 @@ var fileAppendCmd = &cobra.Command{ } var fileCpCmd = &cobra.Command{ - Use: "cp source destination", - Short: "copy between wave files and local files", - Long: `Copy files between wave storage and local filesystem. -Exactly one of source or destination must be a wavefile:// URL.`, - Example: " wsh file cp wavefile://block/config.txt ./local-config.txt\n wsh file cp ./local-config.txt wavefile://block/config.txt", + Use: "cp [source-uri] [destination-uri]" + UriHelpText, + Short: "copy between files on different storage systems", + Long: "Copy between files on different storage systems." + UriHelpText, + Example: " wsh file cp wavefile://block/config.txt ./local-config.txt\n wsh file cp ./local-config.txt wavefile://block/config.txt\n wsh file cp wsh://user@ec2/home/user/config.txt wavefile://client/config.txt", Args: cobra.ExactArgs(2), RunE: activityWrap("file", fileCpRun), PreRunE: preRunSetupRpcClient, From 041720a8b22f8fb084046017f3dfca5f233d6911 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 19:58:37 -0800 Subject: [PATCH 030/126] update text --- cmd/wsh/cmd/wshcmd-file.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index f20ce936b..d41fce341 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -67,7 +67,7 @@ Supported URI schemes: var fileCmd = &cobra.Command{ Use: "file", Short: "manage Wave Terminal files", - Long: "Commands to manage Wave Terminal files stored in blocks." + UriHelpText} + Long: "Commands to manage files across different storage systems." + UriHelpText} var fileTimeout int @@ -159,8 +159,8 @@ var fileAppendCmd = &cobra.Command{ var fileCpCmd = &cobra.Command{ Use: "cp [source-uri] [destination-uri]" + UriHelpText, - Short: "copy between files on different storage systems", - Long: "Copy between files on different storage systems." + UriHelpText, + Short: "copy files between storage systems", + Long: "Copy files between different storage systems." + UriHelpText, Example: " wsh file cp wavefile://block/config.txt ./local-config.txt\n wsh file cp ./local-config.txt wavefile://block/config.txt\n wsh file cp wsh://user@ec2/home/user/config.txt wavefile://client/config.txt", Args: cobra.ExactArgs(2), RunE: activityWrap("file", fileCpRun), From 1ad237291b08507a7c57d5d8f70a9d9f10cfa704 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:00:44 -0800 Subject: [PATCH 031/126] update text --- cmd/wsh/cmd/wshcmd-file.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index d41fce341..80d00687a 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -46,7 +46,7 @@ Supported URI schemes: Format: wsh://[zoneid]/[path] wsh: Used to access files on remotes via the WSH helper. Allows for file streaming to Wave and other remotes. - Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. + Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. If a profile is provided, it must be defined in "profiles.json" in the Wave configuration directory. Format: wsh://[remote]/[path] @@ -56,7 +56,7 @@ Supported URI schemes: - /~/[path] is a path relative to your home directory on your local machine s3: - Used to access files on S3. Requires S3 credentials to be set up, either in "profiles.json" or in the AWS CLI. + Used to access files on S3-compatible systems. Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. If no profile is provided, the default from your AWS CLI configuration will be used. Profiles from the AWS CLI must be prefixed with "aws:". Format: s3://[bucket]/[path] From 294ae81d4449c05171c8cd45a13a6020c049ef9d Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:01:36 -0800 Subject: [PATCH 032/126] update text --- cmd/wsh/cmd/wshcmd-file.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 80d00687a..92902ddbc 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -46,7 +46,8 @@ Supported URI schemes: Format: wsh://[zoneid]/[path] wsh: Used to access files on remotes via the WSH helper. Allows for file streaming to Wave and other remotes. - Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. If a profile is provided, it must be defined in "profiles.json" in the Wave configuration directory. + Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. + If a profile is provided, it must be defined in "profiles.json" in the Wave configuration directory. Format: wsh://[remote]/[path] @@ -56,7 +57,8 @@ Supported URI schemes: - /~/[path] is a path relative to your home directory on your local machine s3: - Used to access files on S3-compatible systems. Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. + Used to access files on S3-compatible systems. + Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. If no profile is provided, the default from your AWS CLI configuration will be used. Profiles from the AWS CLI must be prefixed with "aws:". Format: s3://[bucket]/[path] From e265ccaf6485933cf1d77315ef6f5ac085528eae Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:02:34 -0800 Subject: [PATCH 033/126] fix indents --- cmd/wsh/cmd/wshcmd-file.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 92902ddbc..941424cf9 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -47,8 +47,8 @@ Supported URI schemes: wsh: Used to access files on remotes via the WSH helper. Allows for file streaming to Wave and other remotes. Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. - If a profile is provided, it must be defined in "profiles.json" in the Wave configuration directory. - + If a profile is provided, it must be defined in "profiles.json" in the Wave configuration directory. + Format: wsh://[remote]/[path] Shorthands can be used for the current remote and your local machine: @@ -58,9 +58,9 @@ Supported URI schemes: s3: Used to access files on S3-compatible systems. - Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. + Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. If no profile is provided, the default from your AWS CLI configuration will be used. Profiles from the AWS CLI must be prefixed with "aws:". - + Format: s3://[bucket]/[path] aws:[profile]:s3://[bucket]/[path] [profile]:s3://[bucket]/[path]` From 75b5bf74d8255f17f1067c81e0787f5b4313a37f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:03:29 -0800 Subject: [PATCH 034/126] remove newline --- cmd/wsh/cmd/wshcmd-file.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 941424cf9..c473131cf 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -55,7 +55,6 @@ Supported URI schemes: - [path] is a relative or absolute path on the current remote - //[remote]/[path] is a path on a remote - /~/[path] is a path relative to your home directory on your local machine - s3: Used to access files on S3-compatible systems. Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. From aca09f2da0dccd5e9a85660bec562515600577de Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:05:51 -0800 Subject: [PATCH 035/126] more formatting --- cmd/wsh/cmd/wshcmd-file.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index c473131cf..08fe2a1c4 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -52,9 +52,9 @@ Supported URI schemes: Format: wsh://[remote]/[path] Shorthands can be used for the current remote and your local machine: - - [path] is a relative or absolute path on the current remote - - //[remote]/[path] is a path on a remote - - /~/[path] is a path relative to your home directory on your local machine + [path] a relative or absolute path on the current remote + //[remote]/[path] a path on a remote + /~/[path] a path relative to your home directory on your local machine s3: Used to access files on S3-compatible systems. Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. From ca281d6ed65962ec1bc43da79b328cd686b8d213 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:09:03 -0800 Subject: [PATCH 036/126] more improvements --- cmd/wsh/cmd/wshcmd-file.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 08fe2a1c4..c5fb70923 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -102,6 +102,7 @@ func resolveWaveFile(ref *waveFileRef) (*waveobj.ORef, error) { var fileListCmd = &cobra.Command{ Use: "ls [uri]", + Aliases: []string{"list"}, Short: "list files", Long: "List files in a directory. By default, lists files in the current directory." + UriHelpText, Example: " wsh file ls wsh://user@ec2/home/user/\n wsh file ls wavefile://client/configs/", @@ -123,7 +124,7 @@ var fileInfoCmd = &cobra.Command{ Use: "info [uri]", Short: "show wave file information", Long: "Show information about a file." + UriHelpText, - Example: " wsh file info wsh://user@ec2/home/user/config.txt\n wsh file info wavefile://client/settings.json", + Example: " wsh file info wsh://user@ec2/home/user/config.txt\n wsh file info wavefile://client/settings.json", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileInfoRun), PreRunE: preRunSetupRpcClient, @@ -162,7 +163,7 @@ var fileCpCmd = &cobra.Command{ Use: "cp [source-uri] [destination-uri]" + UriHelpText, Short: "copy files between storage systems", Long: "Copy files between different storage systems." + UriHelpText, - Example: " wsh file cp wavefile://block/config.txt ./local-config.txt\n wsh file cp ./local-config.txt wavefile://block/config.txt\n wsh file cp wsh://user@ec2/home/user/config.txt wavefile://client/config.txt", + Example: " wsh file cp wavefile://block/config.txt ./local-config.txt\n wsh file cp ./local-config.txt wavefile://block/config.txt\n wsh file cp wsh://user@ec2/home/user/config.txt wavefile://client/config.txt", Args: cobra.ExactArgs(2), RunE: activityWrap("file", fileCpRun), PreRunE: preRunSetupRpcClient, From 579a6e8bef238bce7c5d4a08b2c9efea10c9181b Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:10:26 -0800 Subject: [PATCH 037/126] fix wavefile scheme --- cmd/wsh/cmd/wshcmd-file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index c5fb70923..4d18d07bc 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -43,7 +43,7 @@ Supported URI schemes: wavefile: Used to retrieve blockfiles from the internal Wave filesystem. - Format: wsh://[zoneid]/[path] + Format: wavefile://[zoneid]/[path] wsh: Used to access files on remotes via the WSH helper. Allows for file streaming to Wave and other remotes. Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. From fba2550e8213c395248c2477d87d3eb7d9952a91 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:11:26 -0800 Subject: [PATCH 038/126] move wavefile below others --- cmd/wsh/cmd/wshcmd-file.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 4d18d07bc..af433d931 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -40,10 +40,6 @@ const ( URI format: [profile]:[uri-scheme]://[connection]/[path] Supported URI schemes: - wavefile: - Used to retrieve blockfiles from the internal Wave filesystem. - - Format: wavefile://[zoneid]/[path] wsh: Used to access files on remotes via the WSH helper. Allows for file streaming to Wave and other remotes. Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. @@ -62,7 +58,11 @@ Supported URI schemes: Format: s3://[bucket]/[path] aws:[profile]:s3://[bucket]/[path] - [profile]:s3://[bucket]/[path]` + [profile]:s3://[bucket]/[path] + wavefile: + Used to retrieve blockfiles from the internal Wave filesystem. + + Format: wavefile://[zoneid]/[path]` ) var fileCmd = &cobra.Command{ From 3bb25cbac6a54eaa0fc29d3c84c10e11aec1fa7a Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 20:20:20 -0800 Subject: [PATCH 039/126] more doc updates --- cmd/wsh/cmd/wshcmd-file.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index af433d931..141434f95 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -41,16 +41,16 @@ URI format: [profile]:[uri-scheme]://[connection]/[path] Supported URI schemes: wsh: - Used to access files on remotes via the WSH helper. Allows for file streaming to Wave and other remotes. + Used to access files on remote hosts over SSH via the WSH helper. Allows for file streaming to Wave and other remotes. Profiles are optional for WSH URIs, provided that you have configured the remote host in your "connections.json" or "~/.ssh/config" file. If a profile is provided, it must be defined in "profiles.json" in the Wave configuration directory. Format: wsh://[remote]/[path] - Shorthands can be used for the current remote and your local machine: + Shorthands can be used for the current remote and your local computer: [path] a relative or absolute path on the current remote //[remote]/[path] a path on a remote - /~/[path] a path relative to your home directory on your local machine + /~/[path] a path relative to your home directory on your local computer s3: Used to access files on S3-compatible systems. Requires S3 credentials to be set up, either in the AWS CLI configuration files, or in "profiles.json" in the Wave configuration directory. @@ -67,8 +67,8 @@ Supported URI schemes: var fileCmd = &cobra.Command{ Use: "file", - Short: "manage Wave Terminal files", - Long: "Commands to manage files across different storage systems." + UriHelpText} + Short: "manage files across different storage systems", + Long: "Manage files across different storage systems.\n\nWave Terminal is capable of managing files from remote SSH hosts, S3-compatible systems, and the internal Wave filesystem.\nFiles are addressed via URIs, which vary depending on the storage system." + UriHelpText} var fileTimeout int From 58785c862748c496efcbbb31804ec7be53ee2682 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 21:26:08 -0800 Subject: [PATCH 040/126] got ls working --- cmd/wsh/cmd/wshcmd-file.go | 19 ++++---- pkg/remote/fileshare/fileshare.go | 6 +-- pkg/remote/fileshare/fstype/fstype.go | 2 +- pkg/remote/fileshare/s3fs/s3fs.go | 2 +- pkg/remote/fileshare/wavefs/wavefs.go | 4 +- pkg/remote/fileshare/wshfs/wshfs.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 63 ++++++++++++++------------- pkg/wshrpc/wshserver/wshserver.go | 2 +- 8 files changed, 51 insertions(+), 49 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 141434f95..a069ad584 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -15,6 +15,7 @@ import ( "path" "path/filepath" "strings" + "text/tabwriter" "time" "github.com/spf13/cobra" @@ -133,6 +134,7 @@ var fileInfoCmd = &cobra.Command{ var fileRmCmd = &cobra.Command{ Use: "rm [uri]", Short: "remove a file", + Long: "Remove a file." + UriHelpText, Example: " wsh file rm wsh://user@ec2/home/user/config.txt\n wsh file rm wavefile://client/settings.json", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileRmRun), @@ -152,7 +154,7 @@ var fileWriteCmd = &cobra.Command{ var fileAppendCmd = &cobra.Command{ Use: "append [uri]", Short: "append stdin to a file", - Long: "Append stdin to a file, buffering input and respecting 10MB total file size limit" + UriHelpText, + Long: "Append stdin to a file, buffering input and respecting 10MB total file size limit." + UriHelpText, Example: " tail -f log.txt | wsh file append wavefile://block/app.log", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileAppendRun), @@ -482,7 +484,6 @@ func filePrintColumns(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRem strs := make([]string, len(respUnion.Response.FileInfo)) for i, f := range respUnion.Response.FileInfo { strs[i] = f.Name - log.Printf("file: %s", f.Name) } return strs, nil }, @@ -509,15 +510,17 @@ func filePrintLong(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemote nameWidth = 60 } + writer := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0) + // Print samples for _, f := range samples { name := f.Name t := time.Unix(f.ModTime/1000, 0) timestamp := utilfn.FormatLsTime(t) if f.Size == 0 && strings.HasSuffix(name, "/") { - fmt.Fprintf(os.Stdout, "%-*s %8s %s\n", nameWidth, name, "-", timestamp) + fmt.Fprintf(writer, "%-*s\t%8s\t%s\n", nameWidth, name, "-", timestamp) } else { - fmt.Fprintf(os.Stdout, "%-*s %8d %s\n", nameWidth, name, f.Size, timestamp) + fmt.Fprintf(writer, "%-*s\t%8d\t%s\n", nameWidth, name, f.Size, timestamp) } } @@ -531,12 +534,13 @@ func filePrintLong(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemote t := time.Unix(f.ModTime/1000, 0) timestamp := utilfn.FormatLsTime(t) if f.Size == 0 && strings.HasSuffix(name, "/") { - fmt.Fprintf(os.Stdout, "%-*s %8s %s\n", nameWidth, name, "-", timestamp) + fmt.Fprintf(writer, "%-*s\t%8s\t%s\n", nameWidth, name, "-", timestamp) } else { - fmt.Fprintf(os.Stdout, "%-*s %8d %s\n", nameWidth, name, f.Size, timestamp) + fmt.Fprintf(writer, "%-*s\t%8d\t%s\n", nameWidth, name, f.Size, timestamp) } } } + writer.Flush() return nil } @@ -559,21 +563,18 @@ func fileListRun(cmd *cobra.Command, args []string) error { } filesChan := wshclient.FileListStreamCommand(RpcClient, wshrpc.FileListData{Path: path, Opts: &wshrpc.FileListOpts{All: recursive}}, &wshrpc.RpcOpts{Timeout: 2000}) - log.Printf("list entries stream") if longForm { return filePrintLong(filesChan) } if onePerLine { - log.Printf("onePerLine") for respUnion := range filesChan { if respUnion.Error != nil { log.Printf("error: %v", respUnion.Error) return respUnion.Error } for _, f := range respUnion.Response.FileInfo { - log.Printf("file: %s", f.Name) fmt.Fprintln(os.Stdout, f.Name) } return nil diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index c1c9e86b8..a8978350e 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -70,13 +70,13 @@ func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([ return client.ListEntries(ctx, conn, opts) } -func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { log.Printf("ListEntriesStream: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) - defer close(rtn) + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 1) rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Error: fmt.Errorf("error creating fileshare client, could not parse connection %s", path)} + close(rtn) return rtn } return client.ListEntriesStream(ctx, conn, opts) diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 5376d6955..34155b318 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -15,7 +15,7 @@ type FileShareClient interface { // ListEntries returns the list of entries at the given path, or nothing if the path is a file ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) // ListEntriesStream returns a stream of entries at the given path - ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] + ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] // PutFile writes the given data to the file at the given path PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error // Mkdir creates a directory at the given path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index f98275492..483b459f1 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -33,7 +33,7 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o return nil, nil } -func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { return nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 8974958c2..44419e499 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -59,8 +59,8 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w return &wshrpc.FileData{Info: data.Info, Entries: list}, nil } -func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { - ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) +func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) go func() { defer close(ch) list, err := c.ListEntries(ctx, conn, opts) diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 11ed14ae2..9ebef53f8 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -121,7 +121,7 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt return entries, nil } -func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { client := wshclient.GetBareRpcClient() return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 18977bdd4..9635655e3 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -210,29 +210,54 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrpc.CommandRemoteListEntriesData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) go func() { + log.Printf("RemoteListEntriesCommand: path=%s\n", data.Path) defer close(ch) path, err := wavebase.ExpandHomeDir(data.Path) if err != nil { + log.Printf("cannot expand path %q: %v\n", data.Path, err) ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](err) return } - innerFilesEntries := []*fs.DirEntry{} + innerFilesEntries := []os.DirEntry{} seen := 0 if data.Opts.Limit == 0 { data.Opts.Limit = MaxDirSize } - err = listFilesInternal(path, innerFilesEntries, &seen, data.Opts) - if err != nil { - ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("cannot list entries in dir %q: %w", path, err)) - return + if data.Opts.All { + fs.WalkDir(os.DirFS(path), ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + innerFilesEntries = append(innerFilesEntries, d) + seen++ + if seen >= data.Opts.Limit { + return io.EOF + } + return nil + }) + } else { + innerFilesEntries, err = os.ReadDir(path) + if err != nil { + log.Printf("cannot open dir %q: %v\n", path, err) + ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("cannot open dir %q: %w", path, err)) + return + } } + log.Printf("innerFilesEntries: %v\n", innerFilesEntries) var fileInfoArr []*wshrpc.FileInfo for _, innerFileEntry := range innerFilesEntries { + log.Printf("innerFileEntry: %v\n", innerFileEntry) if ctx.Err() != nil { + log.Printf("context error\n") + ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](ctx.Err()) return } - innerFileInfoInt, err := (*innerFileEntry).Info() + innerFileInfoInt, err := innerFileEntry.Info() if err != nil { + log.Printf("cannot stat file %q: %v\n", innerFileEntry.Name(), err) continue } innerFileInfo := statToFileInfo(filepath.Join(path, innerFileInfoInt.Name()), innerFileInfoInt, false) @@ -244,6 +269,7 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp } } if len(fileInfoArr) > 0 { + log.Printf("sending final chunk: %v\n", fileInfoArr) resp := wshrpc.CommandRemoteListEntriesRtnData{FileInfo: fileInfoArr} ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: resp} } @@ -251,31 +277,6 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp return ch } -func listFilesInternal(path string, fileInfoArr []*fs.DirEntry, seen *int, opts *wshrpc.FileListOpts) error { - innerFilesEntries, err := os.ReadDir(path) - if err != nil { - return fmt.Errorf("cannot open dir %q: %w", path, err) - } - for _, innerFileEntry := range innerFilesEntries { - if err != nil { - continue - } - if innerFileEntry.IsDir() && opts.All { - err = listFilesInternal(filepath.Join(path, innerFileEntry.Name()), fileInfoArr, seen, opts) - if err != nil { - return err - } - } else { - fileInfoArr = append(fileInfoArr, &innerFileEntry) - } - *seen++ - if *seen >= opts.Offset || *seen >= opts.Offset+opts.Limit { - break - } - } - return nil -} - func statToFileInfo(fullPath string, finfo fs.FileInfo, extended bool) *wshrpc.FileInfo { mimeType := fileutil.DetectMimeType(fullPath, finfo, extended) rtn := &wshrpc.FileInfo{ diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 74af1130b..c8ff75345 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -305,7 +305,7 @@ func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.FileListDa return fileshare.ListEntries(ctx, data.Path, data.Opts) } -func (ws *WshServer) FileListStreamCommand(ctx context.Context, data wshrpc.FileListData) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func (ws *WshServer) FileListStreamCommand(ctx context.Context, data wshrpc.FileListData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { return fileshare.ListEntriesStream(ctx, data.Path, data.Opts) } From 489c81e880a12680ea94f2c732e3bb82e95dbd72 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 21:39:07 -0800 Subject: [PATCH 041/126] add default . to ls --- cmd/wsh/cmd/wshcmd-file.go | 4 ++++ pkg/remote/fileshare/wshfs/wshfs.go | 4 ---- pkg/wshrpc/wshremote/wshremote.go | 20 +++++++++----------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index a069ad584..cfe2790cc 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -557,6 +557,10 @@ func fileListRun(cmd *cobra.Command, args []string) error { onePerLine = true } + if len(args) == 0 { + args = []string{"."} + } + path, err := fixRelativePaths(args[0]) if err != nil { return err diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 9ebef53f8..07212fe02 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "fmt" "io" - "log" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -27,7 +26,6 @@ func NewWshClient() *WshClient { } func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { - log.Printf("WshClient:Read: path=%s; conn=%v; data=%v", conn.Path, conn, data) client := wshclient.GetBareRpcClient() byteRange := "" if data.At.Size > 0 { @@ -94,7 +92,6 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt var entries []*wshrpc.FileInfo if opts.All || data == nil { rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) - log.Printf("listEntriesInternal: remote list entries") for respUnion := range rtnCh { if respUnion.Error != nil { return nil, respUnion.Error @@ -102,7 +99,6 @@ func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opt resp := respUnion.Response entries = append(entries, resp.FileInfo...) } - log.Printf("listEntriesInternal: remote list entries done") } else { entries = append(entries, data.Entries...) if opts.Offset > 0 { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 9635655e3..2377479ae 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -210,11 +210,9 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrpc.CommandRemoteListEntriesData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) go func() { - log.Printf("RemoteListEntriesCommand: path=%s\n", data.Path) defer close(ch) path, err := wavebase.ExpandHomeDir(data.Path) if err != nil { - log.Printf("cannot expand path %q: %v\n", data.Path, err) ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](err) return } @@ -225,6 +223,15 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp } if data.Opts.All { fs.WalkDir(os.DirFS(path), ".", func(path string, d fs.DirEntry, err error) error { + defer func() { + seen++ + }() + if seen < data.Opts.Offset { + return nil + } + if seen >= data.Opts.Offset+data.Opts.Limit { + return io.EOF + } if err != nil { return err } @@ -232,26 +239,18 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp return nil } innerFilesEntries = append(innerFilesEntries, d) - seen++ - if seen >= data.Opts.Limit { - return io.EOF - } return nil }) } else { innerFilesEntries, err = os.ReadDir(path) if err != nil { - log.Printf("cannot open dir %q: %v\n", path, err) ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("cannot open dir %q: %w", path, err)) return } } - log.Printf("innerFilesEntries: %v\n", innerFilesEntries) var fileInfoArr []*wshrpc.FileInfo for _, innerFileEntry := range innerFilesEntries { - log.Printf("innerFileEntry: %v\n", innerFileEntry) if ctx.Err() != nil { - log.Printf("context error\n") ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](ctx.Err()) return } @@ -269,7 +268,6 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp } } if len(fileInfoArr) > 0 { - log.Printf("sending final chunk: %v\n", fileInfoArr) resp := wshrpc.CommandRemoteListEntriesRtnData{FileInfo: fileInfoArr} ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: resp} } From 106d9c77f7a9a8bb2f7d26f5637b0925d55e9df7 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 21:42:54 -0800 Subject: [PATCH 042/126] move fixRelativePaths to wshcmd-file-util --- cmd/wsh/cmd/wshcmd-file-util.go | 13 +++++++++++++ cmd/wsh/cmd/wshcmd-file.go | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 41dfbfd3d..b4aa53184 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -10,6 +10,8 @@ import ( "io/fs" "strings" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" @@ -206,3 +208,14 @@ func streamFileList(zoneId string, path string, recursive bool, filesOnly bool) return resultChan, nil } + +func fixRelativePaths(path string) (string, error) { + conn, err := connparse.ParseURI(path) + if err != nil { + return "", err + } + if conn.Scheme == connparse.ConnectionTypeWsh && conn.Host == connparse.ConnHostCurrent { + return fileutil.FixPath(conn.Path) + } + return path, nil +} diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index cfe2790cc..30b30e07e 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -19,9 +19,7 @@ import ( "time" "github.com/spf13/cobra" - "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/util/colprint" - "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -587,14 +585,3 @@ func fileListRun(cmd *cobra.Command, args []string) error { return filePrintColumns(filesChan) } - -func fixRelativePaths(path string) (string, error) { - conn, err := connparse.ParseURI(path) - if err != nil { - return "", err - } - if conn.Scheme == connparse.ConnectionTypeWsh && conn.Host == connparse.ConnHostCurrent { - return fileutil.FixPath(conn.Path) - } - return path, nil -} From 66e2461b7f5bd6ff0492f1f6aac0f0e297341590 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 14 Jan 2025 22:48:59 -0800 Subject: [PATCH 043/126] it's working!!!! --- .../app/view/preview/directorypreview.tsx | 38 ++++++++++++++----- frontend/app/view/preview/preview.tsx | 23 ++++++++--- frontend/types/gotypes.d.ts | 12 +++--- pkg/remote/connparse/connparse.go | 5 +++ pkg/remote/connparse/connparse_test.go | 22 +++++++++++ pkg/remote/fileshare/wshfs/wshfs.go | 4 +- pkg/wshrpc/wshrpctypes.go | 14 +++---- 7 files changed, 90 insertions(+), 28 deletions(-) diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index 51fc1f90d..8ef899e65 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -5,9 +5,11 @@ import { Button } from "@/app/element/button"; import { Input } from "@/app/element/input"; import { ContextMenuModel } from "@/app/store/contextmenu"; import { PLATFORM, atoms, createBlock, getApi, globalStore } from "@/app/store/global"; +import { RpcApi } from "@/app/store/wshclientapi"; +import { TabRpcClient } from "@/app/store/wshrpcutil"; import type { PreviewModel } from "@/app/view/preview/preview"; import { checkKeyPressed, isCharacterKeyEvent } from "@/util/keyutil"; -import { base64ToString, fireAndForget, isBlank } from "@/util/util"; +import { fireAndForget, isBlank } from "@/util/util"; import { offset, useDismiss, useFloating, useInteractions } from "@floating-ui/react"; import { Column, @@ -293,7 +295,7 @@ function DirectoryTable({ console.log(`replacing ${fileName} with ${newName}: ${path}`); fireAndForget(async () => { const connection = await globalStore.get(model.connection); - await FileService.Rename(connection, path, newPath); + // await FileService.Rename(connection, path, newPath); model.refreshCallback(); }); } @@ -617,7 +619,11 @@ function TableBody({ label: "Delete", click: () => { fireAndForget(async () => { - await FileService.DeleteFile(conn, finfo.path).catch((e) => console.log(e)); + await RpcApi.FileDeleteCommand(TabRpcClient, { + info: { + path: finfo.path, + }, + }).catch((e) => console.log(e)); setRefreshVersion((current) => current + 1); }); }, @@ -710,10 +716,16 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { useEffect(() => { const getContent = async () => { - const file = await FileService.ReadFile(conn, dirPath); - const serializedContent = base64ToString(file?.data64); - const content: FileInfo[] = JSON.parse(serializedContent); - setUnfilteredData(content); + const file = await RpcApi.FileReadCommand( + TabRpcClient, + { + info: { + path: dirPath, + }, + }, + null + ); + setUnfilteredData(file.entries); }; getContent(); }, [conn, dirPath, refreshVersion]); @@ -805,7 +817,15 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { console.log(`newFile: ${newName}`); fireAndForget(async () => { const connection = await globalStore.get(model.connection); - await FileService.TouchFile(connection, `${dirPath}/${newName}`); + await RpcApi.FileCreateCommand( + TabRpcClient, + { + info: { + path: `wsh://${connection}/${dirPath}/${newName}`, + }, + }, + null + ); model.refreshCallback(); }); setEntryManagerProps(undefined); @@ -819,7 +839,7 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { console.log(`newDirectory: ${newName}`); fireAndForget(async () => { const connection = await globalStore.get(model.connection); - await FileService.Mkdir(connection, `${dirPath}/${newName}`); + // await FileService.Mkdir(connection, `${dirPath}/${newName}`); model.refreshCallback(); }); setEntryManagerProps(undefined); diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index 7add7e3de..f70f3acb6 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -139,7 +139,7 @@ export class PreviewModel implements ViewModel { loadableFileInfo: Atom>; connection: Atom>; statFile: Atom>; - fullFile: Atom>; + fullFile: Atom>; fileMimeType: Atom>; fileMimeTypeLoadable: Atom>; fileContentSaved: PrimitiveAtom; @@ -369,7 +369,11 @@ export class PreviewModel implements ViewModel { return null; } const conn = (await get(this.connection)) ?? ""; - const statFile = await services.FileService.StatFile(conn, fileName); + const statFile = await RpcApi.FileInfoCommand(TabRpcClient, { + info: { + path: `wsh://${conn}/${fileName}`, + }, + }); return statFile; }); this.fileMimeType = atom>(async (get) => { @@ -380,13 +384,17 @@ export class PreviewModel implements ViewModel { this.newFileContent = atom(null) as PrimitiveAtom; this.goParentDirectory = this.goParentDirectory.bind(this); - const fullFileAtom = atom>(async (get) => { + const fullFileAtom = atom>(async (get) => { const fileName = get(this.metaFilePath); if (fileName == null) { return null; } const conn = (await get(this.connection)) ?? ""; - const file = await services.FileService.ReadFile(conn, fileName); + const file = await RpcApi.FileReadCommand(TabRpcClient, { + info: { + path: `wsh://${conn}/${fileName}`, + }, + }); return file; }); @@ -593,7 +601,12 @@ export class PreviewModel implements ViewModel { } const conn = (await globalStore.get(this.connection)) ?? ""; try { - await services.FileService.SaveFile(conn, filePath, stringToBase64(newFileContent)); + await RpcApi.FileWriteCommand(TabRpcClient, { + info: { + path: `wsh://${conn}/${filePath}`, + }, + data64: stringToBase64(newFileContent), + }); globalStore.set(this.fileContent, newFileContent); globalStore.set(this.newFileContent, null); console.log("saved file", filePath); diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 2139192e8..af425a9fb 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -368,15 +368,15 @@ declare global { // wshrpc.FileInfo type FileInfo = { path: string; - dir: string; - name: string; + dir?: string; + name?: string; notfound?: boolean; opts?: FileOptsType; - size: number; + size?: number; meta?: {[key: string]: any}; - mode: number; - modestr: string; - modtime: number; + mode?: number; + modestr?: string; + modtime?: number; isdir?: boolean; supportsmkdir?: boolean; mimetype?: string; diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index b48197ead..aad924a40 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -94,6 +94,11 @@ func ParseURI(uri string) (*Connection, error) { } } + if host == "" && scheme == ConnectionTypeWsh { + host = wshrpc.LocalConnName + remotePath = strings.TrimPrefix(rest, "/") + } + return &Connection{ Scheme: scheme, Host: host, diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index 27ca79422..eda847a96 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -208,6 +208,28 @@ func TestParseURI_WSHLocalShorthand(t *testing.T) { if c.Scheme != expected { t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) } + + cstr = "wsh:///~/path/to/file" + c, err = connparse.ParseURI(cstr) + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected = "~/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + if c.Host != "local" { + t.Fatalf("expected host to be empty, got %q", c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } + expected = "wsh://local/~/path/to/file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } + } func TestParseURI_BasicS3(t *testing.T) { diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 07212fe02..d73612c65 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "fmt" "io" + "log" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -28,7 +29,7 @@ func NewWshClient() *WshClient { func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { client := wshclient.GetBareRpcClient() byteRange := "" - if data.At.Size > 0 { + if data.At != nil && data.At.Size > 0 { byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+data.At.Size) } streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path, ByteRange: byteRange} @@ -36,6 +37,7 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws fullFile := &wshrpc.FileData{} firstPk := true isDir := false + log.Printf("stream file: %s", conn.Path) var fileBuf bytes.Buffer var fileInfoArr []*wshrpc.FileInfo for respUnion := range rtnCh { diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index a2aef92fc..c8b215a14 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -348,16 +348,16 @@ type FileData struct { } type FileInfo struct { - Path string `json:"path"` // cleaned path (may have "~") - Dir string `json:"dir"` // returns the directory part of the path (if this is a a directory, it will be equal to Path). "~" will be expanded, and separators will be normalized to "/" - Name string `json:"name"` + Path string `json:"path"` // cleaned path (may have "~") + Dir string `json:"dir,omitempty"` // returns the directory part of the path (if this is a a directory, it will be equal to Path). "~" will be expanded, and separators will be normalized to "/" + Name string `json:"name,omitempty"` NotFound bool `json:"notfound,omitempty"` Opts *FileOptsType `json:"opts,omitempty"` - Size int64 `json:"size"` + Size int64 `json:"size,omitempty"` Meta *FileMeta `json:"meta,omitempty"` - Mode os.FileMode `json:"mode"` - ModeStr string `json:"modestr"` - ModTime int64 `json:"modtime"` + Mode os.FileMode `json:"mode,omitempty"` + ModeStr string `json:"modestr,omitempty"` + ModTime int64 `json:"modtime,omitempty"` IsDir bool `json:"isdir,omitempty"` SupportsMkdir bool `json:"supportsmkdir,omitempty"` MimeType string `json:"mimetype,omitempty"` From 1410f315157530ba8f2a4ccd92f15e89497e3c88 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 15 Jan 2025 16:05:24 -0800 Subject: [PATCH 044/126] add readstream --- frontend/app/store/wshclientapi.ts | 2 +- frontend/types/gotypes.d.ts | 6 -- pkg/remote/fileshare/fileshare.go | 11 ++++ pkg/remote/fileshare/fstype/fstype.go | 4 +- pkg/remote/fileshare/s3fs/s3fs.go | 4 ++ pkg/remote/fileshare/wavefs/wavefs.go | 35 ++++++++-- pkg/remote/fileshare/wshfs/wshfs.go | 95 +++++++++------------------ pkg/web/web.go | 6 +- pkg/wshrpc/wshclient/wshclient.go | 4 +- pkg/wshrpc/wshremote/wshremote.go | 54 +++++++-------- pkg/wshrpc/wshrpctypes.go | 18 +++-- 11 files changed, 125 insertions(+), 114 deletions(-) diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 4e5ccf6fd..fa7fddf03 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -278,7 +278,7 @@ class RpcApiType { } // command "remotestreamfile" [responsestream] - RemoteStreamFileCommand(client: WshClient, data: CommandRemoteStreamFileData, opts?: RpcOpts): AsyncGenerator { + RemoteStreamFileCommand(client: WshClient, data: CommandRemoteStreamFileData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotestreamfile", data, opts); } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index af425a9fb..192c32e62 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -199,12 +199,6 @@ declare global { byterange?: string; }; - // wshrpc.CommandRemoteStreamFileRtnData - type CommandRemoteStreamFileRtnData = { - fileinfo?: FileInfo[]; - data64?: string; - }; - // wshrpc.CommandRemoteWriteFileData type CommandRemoteWriteFileData = { path: string; diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index a8978350e..89e66bf16 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -61,6 +61,17 @@ func Read(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { return client.Read(ctx, conn, data) } +func ReadStream(ctx context.Context, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + client, conn := CreateFileShareClient(ctx, data.Info.Path) + if conn == nil || client == nil { + rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) + rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Error: fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path)} + close(rtn) + return rtn + } + return client.ReadStream(ctx, conn, data) +} + func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { log.Printf("ListEntries: path=%s", path) client, conn := CreateFileShareClient(ctx, path) diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 34155b318..baf8f2bfc 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -10,8 +10,10 @@ import ( type FileShareClient interface { // Stat returns the file info at the given parsed connection path Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) - // Read returns the file info at the given path, if it's a dir, then the list of entries + // Read returns the file info at the given path, if it's a directory, then the list of entries Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) + // ReadStream returns a stream of file data at the given path. If it's a directory, then the list of entries + ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] // ListEntries returns the list of entries at the given path, or nothing if the path is a file ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) // ListEntriesStream returns a stream of entries at the given path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 483b459f1..77fc0a8e5 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -29,6 +29,10 @@ func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wsh return nil, nil } +func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + return nil +} + func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { return nil, nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 44419e499..ca80918dd 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -19,8 +19,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshrpc" ) -const DirChunkSize = 128 - type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -29,6 +27,35 @@ func NewWaveClient() *WaveClient { return &WaveClient{} } +func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) + go func() { + defer close(ch) + rtnData, err := c.Read(ctx, conn, data) + if err != nil { + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Error: err} + return + } + for { + if ctx.Err() != nil { + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Error: ctx.Err()} + } + dataLen := len(rtnData.Data64) + if !rtnData.Info.IsDir { + for i := 0; i < dataLen; i += wshrpc.FileChunkSize { + dataEnd := min(i+wshrpc.FileChunkSize, dataLen) + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: rtnData.Data64[i:dataEnd], Info: rtnData.Info, At: &wshrpc.FileDataAt{Offset: int64(i), Size: int64(dataEnd - i)}}} + } + } else { + for i := 0; i < len(rtnData.Entries); i += wshrpc.DirChunkSize { + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: rtnData.Entries[i:min(i+wshrpc.DirChunkSize, len(rtnData.Entries))], Info: rtnData.Info}} + } + } + } + }() + return ch +} + func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { zoneId := conn.Host if zoneId == "" { @@ -68,8 +95,8 @@ func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Conne ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Error: err} return } - for i := 0; i < len(list); i += DirChunkSize { - ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: list[i:min(i+DirChunkSize, len(list))]}} + for i := 0; i < len(list); i += wshrpc.DirChunkSize { + ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: list[i:min(i+wshrpc.DirChunkSize, len(list))]}} } }() return ch diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index d73612c65..b814facfc 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "fmt" "io" - "log" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -27,19 +26,11 @@ func NewWshClient() *WshClient { } func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { - client := wshclient.GetBareRpcClient() - byteRange := "" - if data.At != nil && data.At.Size > 0 { - byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+data.At.Size) - } - streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path, ByteRange: byteRange} - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) - fullFile := &wshrpc.FileData{} + rtnCh := c.ReadStream(ctx, conn, data) + var fileData *wshrpc.FileData firstPk := true isDir := false - log.Printf("stream file: %s", conn.Path) var fileBuf bytes.Buffer - var fileInfoArr []*wshrpc.FileInfo for respUnion := range rtnCh { if respUnion.Error != nil { return nil, respUnion.Error @@ -48,20 +39,20 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws if firstPk { firstPk = false // first packet has the fileinfo - if len(resp.FileInfo) != 1 { - return nil, fmt.Errorf("stream file protocol error, first pk fileinfo len=%d", len(resp.FileInfo)) + if resp.Info == nil { + return nil, fmt.Errorf("stream file protocol error, first pk fileinfo is empty") } - fullFile.Info = resp.FileInfo[0] - if fullFile.Info.IsDir { + fileData = &resp + if fileData.Info.IsDir { isDir = true } continue } if isDir { - if len(resp.FileInfo) == 0 { + if len(resp.Entries) == 0 { continue } - fileInfoArr = append(fileInfoArr, resp.FileInfo...) + fileData.Entries = append(fileData.Entries, resp.Entries...) } else { if resp.Data64 == "" { continue @@ -73,76 +64,53 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws } } } - if isDir { - entries, err := listEntriesInternal(client, conn, &wshrpc.FileListOpts{}, &wshrpc.FileData{Entries: fileInfoArr}) - if err != nil { - return nil, fmt.Errorf("stream file, failed to list entries: %w", err) - } - fullFile.Entries = entries - } else { - // we can avoid this re-encoding if we ensure the remote side always encodes chunks of 3 bytes so we don't get padding chars - fullFile.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) + if !isDir { + fileData.Data64 = base64.StdEncoding.EncodeToString(fileBuf.Bytes()) } - return fullFile, nil + return fileData, nil } -func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { - return listEntriesInternal(wshclient.GetBareRpcClient(), conn, opts, nil) +func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + byteRange := "" + if data.At != nil && data.At.Size > 0 { + byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+data.At.Size) + } + streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path, ByteRange: byteRange} + return wshclient.RemoteStreamFileCommand(wshclient.GetBareRpcClient(), streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } -func listEntriesInternal(client *wshutil.WshRpc, conn *connparse.Connection, opts *wshrpc.FileListOpts, data *wshrpc.FileData) ([]*wshrpc.FileInfo, error) { +func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { var entries []*wshrpc.FileInfo - if opts.All || data == nil { - rtnCh := wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) - for respUnion := range rtnCh { - if respUnion.Error != nil { - return nil, respUnion.Error - } - resp := respUnion.Response - entries = append(entries, resp.FileInfo...) - } - } else { - entries = append(entries, data.Entries...) - if opts.Offset > 0 { - if opts.Offset >= len(entries) { - entries = nil - } else { - entries = entries[opts.Offset:] - } - } - if opts.Limit > 0 { - if opts.Limit < len(entries) { - entries = entries[:opts.Limit] - } + rtnCh := c.ListEntriesStream(ctx, conn, opts) + for respUnion := range rtnCh { + if respUnion.Error != nil { + return nil, respUnion.Error } + resp := respUnion.Response + entries = append(entries, resp.FileInfo...) } return entries, nil } func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { - client := wshclient.GetBareRpcClient() - return wshclient.RemoteListEntriesCommand(client, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteListEntriesCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { - client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileInfoCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteFileInfoCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { - client := wshclient.GetBareRpcClient() writeData := wshrpc.CommandRemoteWriteFileData{Path: conn.Path, Data64: data.Data64} - return wshclient.RemoteWriteFileCommand(client, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteWriteFileCommand(wshclient.GetBareRpcClient(), writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { - client := wshclient.GetBareRpcClient() - return wshclient.RemoteMkdirCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteMkdirCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { - client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileRenameCommand(client, [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) + return wshclient.RemoteFileRenameCommand(wshclient.GetBareRpcClient(), [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) } func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { @@ -150,8 +118,7 @@ func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connec } func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { - client := wshclient.GetBareRpcClient() - return wshclient.RemoteFileDeleteCommand(client, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteFileDeleteCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) GetConnectionType() string { diff --git a/pkg/web/web.go b/pkg/web/web.go index 78d1a67e1..35d9445c6 100644 --- a/pkg/web/web.go +++ b/pkg/web/web.go @@ -267,10 +267,10 @@ func handleRemoteStreamFile(w http.ResponseWriter, _ *http.Request, conn string, } if firstPk { firstPk = false - if len(respUnion.Response.FileInfo) != 1 { - return fmt.Errorf("stream file protocol error, first pk fileinfo len=%d", len(respUnion.Response.FileInfo)) + if respUnion.Response.Info == nil { + return fmt.Errorf("stream file protocol error, fileinfo is empty") } - fileInfo = respUnion.Response.FileInfo[0] + fileInfo = respUnion.Response.Info if fileInfo.NotFound { if no404 { serveTransparentGIF(w) diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 2b11aec65..38c61eea0 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -336,8 +336,8 @@ func RemoteStreamCpuDataCommand(w *wshutil.WshRpc, opts *wshrpc.RpcOpts) chan ws } // command "remotestreamfile", wshserver.RemoteStreamFileCommand -func RemoteStreamFileCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamFileData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData] { - return sendRpcRequestResponseStreamHelper[wshrpc.CommandRemoteStreamFileRtnData](w, "remotestreamfile", data, opts) +func RemoteStreamFileCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamFileData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + return sendRpcRequestResponseStreamHelper[wshrpc.FileData](w, "remotestreamfile", data, opts) } // command "remotewritefile", wshserver.RemoteWriteFileCommand diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 2377479ae..61b3194f7 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -22,11 +22,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) -const MaxFileSize = 50 * 1024 * 1024 // 10M -const MaxDirSize = 1024 -const FileChunkSize = 16 * 1024 -const DirChunkSize = 128 - type ServerImpl struct { LogWriter io.Writer } @@ -71,14 +66,14 @@ func parseByteRange(rangeStr string) (ByteRangeType, error) { return ByteRangeType{Start: start, End: end}, nil } -func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error { +func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType)) error { innerFilesEntries, err := os.ReadDir(path) if err != nil { return fmt.Errorf("cannot open dir %q: %w", path, err) } if byteRange.All { - if len(innerFilesEntries) > MaxDirSize { - innerFilesEntries = innerFilesEntries[:MaxDirSize] + if len(innerFilesEntries) > wshrpc.MaxDirSize { + innerFilesEntries = innerFilesEntries[:wshrpc.MaxDirSize] } } else { if byteRange.Start >= int64(len(innerFilesEntries)) { @@ -108,19 +103,18 @@ func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, by } innerFileInfo := statToFileInfo(filepath.Join(path, innerFileInfoInt.Name()), innerFileInfoInt, false) fileInfoArr = append(fileInfoArr, innerFileInfo) - if len(fileInfoArr) >= DirChunkSize { - dataCallback(fileInfoArr, nil) + if len(fileInfoArr) >= wshrpc.DirChunkSize { + dataCallback(fileInfoArr, nil, byteRange) fileInfoArr = nil } } if len(fileInfoArr) > 0 { - dataCallback(fileInfoArr, nil) + dataCallback(fileInfoArr, nil, byteRange) } return nil } -// TODO make sure the read is in chunks of 3 bytes (so 4 bytes of base64) in order to make decoding more efficient -func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error { +func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType)) error { fd, err := os.Open(path) if err != nil { return fmt.Errorf("cannot open file %q: %w", path, err) @@ -134,7 +128,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string } filePos = byteRange.Start } - buf := make([]byte, FileChunkSize) + buf := make([]byte, wshrpc.FileChunkSize) for { if ctx.Err() != nil { return ctx.Err() @@ -145,7 +139,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string n = int(byteRange.End - filePos) } filePos += int64(n) - dataCallback(nil, buf[:n]) + dataCallback(nil, buf[:n], byteRange) } if !byteRange.All && filePos >= byteRange.End { break @@ -160,7 +154,7 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string return nil } -func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte)) error { +func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType)) error { byteRange, err := parseByteRange(data.ByteRange) if err != nil { return err @@ -173,11 +167,11 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp if err != nil { return fmt.Errorf("cannot stat file %q: %w", path, err) } - dataCallback([]*wshrpc.FileInfo{finfo}, nil) + dataCallback([]*wshrpc.FileInfo{finfo}, nil, byteRange) if finfo.NotFound { return nil } - if finfo.Size > MaxFileSize { + if finfo.Size > wshrpc.MaxFileSize { return fmt.Errorf("file %q is too large to read, use /wave/stream-file", path) } if finfo.IsDir { @@ -187,21 +181,27 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp } } -func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc.CommandRemoteStreamFileData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData] { - ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData], 16) +func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc.CommandRemoteStreamFileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) go func() { defer close(ch) - err := impl.remoteStreamFileInternal(ctx, data, func(fileInfo []*wshrpc.FileInfo, data []byte) { - resp := wshrpc.CommandRemoteStreamFileRtnData{} - resp.FileInfo = fileInfo + err := impl.remoteStreamFileInternal(ctx, data, func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType) { + resp := wshrpc.FileData{} + fileInfoLen := len(fileInfo) + if fileInfoLen > 1 { + resp.Entries = fileInfo + } else if fileInfoLen == 1 { + resp.Info = fileInfo[0] + } if len(data) > 0 { resp.Data64 = base64.StdEncoding.EncodeToString(data) + resp.At = &wshrpc.FileDataAt{Offset: byteRange.Start, Size: int64(len(data))} } log.Printf("callback -- sending response %d\n", len(resp.Data64)) - ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteStreamFileRtnData]{Response: resp} + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: resp} }) if err != nil { - ch <- respErr[wshrpc.CommandRemoteStreamFileRtnData](err) + ch <- respErr[wshrpc.FileData](err) } }() return ch @@ -219,7 +219,7 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp innerFilesEntries := []os.DirEntry{} seen := 0 if data.Opts.Limit == 0 { - data.Opts.Limit = MaxDirSize + data.Opts.Limit = wshrpc.MaxDirSize } if data.Opts.All { fs.WalkDir(os.DirFS(path), ".", func(path string, d fs.DirEntry, err error) error { @@ -261,7 +261,7 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp } innerFileInfo := statToFileInfo(filepath.Join(path, innerFileInfoInt.Name()), innerFileInfoInt, false) fileInfoArr = append(fileInfoArr, innerFileInfo) - if len(fileInfoArr) >= DirChunkSize { + if len(fileInfoArr) >= wshrpc.DirChunkSize { resp := wshrpc.CommandRemoteListEntriesRtnData{FileInfo: fileInfoArr} ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: resp} fileInfoArr = nil diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index c8b215a14..953240779 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -19,6 +19,17 @@ import ( "github.com/wavetermdev/waveterm/pkg/wps" ) +const ( + // MaxFileSize is the maximum file size that can be read + MaxFileSize = 50 * 1024 * 1024 // 10M + // MaxDirSize is the maximum number of entries that can be read in a directory + MaxDirSize = 1024 + // FileChunkSize is the size of the file chunk to read + FileChunkSize = 16 * 1024 + // DirChunkSize is the size of the directory chunk to read + DirChunkSize = 128 +) + const LocalConnName = "local" const ( @@ -175,7 +186,7 @@ type WshRpcInterface interface { EventRecvCommand(ctx context.Context, data wps.WaveEvent) error // remotes - RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[CommandRemoteStreamFileRtnData] + RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[FileData] RemoteListEntriesCommand(ctx context.Context, data CommandRemoteListEntriesData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) RemoteFileTouchCommand(ctx context.Context, path string) error @@ -469,11 +480,6 @@ type CommandRemoteStreamFileData struct { ByteRange string `json:"byterange,omitempty"` } -type CommandRemoteStreamFileRtnData struct { - FileInfo []*FileInfo `json:"fileinfo,omitempty"` - Data64 string `json:"data64,omitempty"` -} - type CommandRemoteListEntriesData struct { Path string `json:"path"` Opts *FileListOpts `json:"opts,omitempty"` From f71a04fc46aea580173e0da76c445ad248c52e31 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 15 Jan 2025 17:34:10 -0800 Subject: [PATCH 045/126] save --- pkg/util/fileutil/fileutil.go | 9 +++------ pkg/wshrpc/wshremote/wshremote.go | 1 + pkg/wshrpc/wshrpctypes.go | 5 +++++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 2a4a717bb..d62bf1ef9 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -6,9 +6,10 @@ import ( "mime" "net/http" "os" - "os/user" "path/filepath" "strings" + + "github.com/wavetermdev/waveterm/pkg/wavebase" ) // on error just returns "" @@ -70,11 +71,7 @@ func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { func FixPath(path string) (string, error) { if strings.HasPrefix(path, "~") { - curUser, err := user.Current() - if err != nil { - return "", err - } - return filepath.Join(curUser.HomeDir, path[1:]), nil + return filepath.Join(wavebase.GetHomeDir(), path[1:]), nil } else if !filepath.IsAbs(path) { path, err := filepath.Abs(path) if err != nil { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 61b3194f7..da35bf224 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -189,6 +189,7 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc resp := wshrpc.FileData{} fileInfoLen := len(fileInfo) if fileInfoLen > 1 { + resp.Info = fileInfo[0] resp.Entries = fileInfo } else if fileInfoLen == 1 { resp.Info = fileInfo[0] diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 953240779..747d48669 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -475,6 +475,11 @@ type CpuDataType struct { Value float64 `json:"value"` } +type FileRemoteStreamOpts struct { + UseTar bool `json:"usetar,omitempty"` + DeleteAfter bool `json:"deleteafter,omitempty"` +} + type CommandRemoteStreamFileData struct { Path string `json:"path"` ByteRange string `json:"byterange,omitempty"` From 1a2c6b0d7a61b147ac5a7c42566c53a7a44aff04 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 15 Jan 2025 19:49:47 -0800 Subject: [PATCH 046/126] got ui working --- cmd/wsh/cmd/wshcmd-conn.go | 7 ++- cmd/wsh/cmd/wshcmd-file-util.go | 9 ++- .../app/view/preview/directorypreview.tsx | 11 ++-- frontend/app/view/preview/preview.tsx | 18 +++--- pkg/remote/connparse/connparse.go | 13 +++-- pkg/remote/fileshare/fileshare.go | 11 ++++ pkg/remote/fileshare/fstype/fstype.go | 2 + pkg/remote/fileshare/s3fs/s3fs.go | 4 ++ pkg/remote/fileshare/wavefs/wavefs.go | 58 ++++++++++++++++--- pkg/remote/fileshare/wshfs/wshfs.go | 11 ++++ 10 files changed, 118 insertions(+), 26 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-conn.go b/cmd/wsh/cmd/wshcmd-conn.go index 2b2d3a98f..b0a6cd352 100644 --- a/cmd/wsh/cmd/wshcmd-conn.go +++ b/cmd/wsh/cmd/wshcmd-conn.go @@ -30,7 +30,6 @@ var connStatusCmd = &cobra.Command{ var connReinstallCmd = &cobra.Command{ Use: "reinstall CONNECTION", Short: "reinstall wsh on a connection", - Args: cobra.ExactArgs(1), RunE: connReinstallRun, PreRunE: preRunSetupRpcClient, } @@ -124,6 +123,12 @@ func connStatusRun(cmd *cobra.Command, args []string) error { } func connReinstallRun(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + if RpcContext.Conn == "" { + return fmt.Errorf("no connection specified") + } + args = []string{RpcContext.Conn} + } connName := args[0] if err := validateConnectionName(connName); err != nil { return err diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index b4aa53184..da5ffbe80 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "io/fs" + "log" "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" @@ -215,7 +216,13 @@ func fixRelativePaths(path string) (string, error) { return "", err } if conn.Scheme == connparse.ConnectionTypeWsh && conn.Host == connparse.ConnHostCurrent { - return fileutil.FixPath(conn.Path) + conn.Host = RpcContext.Conn + log.Printf("Fixing relative path to %s", conn.Host) + fixedPath, err := fileutil.FixPath(conn.Path) + if err != nil { + return "", err + } + return fmt.Sprintf("wsh://%s/%s", RpcContext.Conn, fixedPath), nil } return path, nil } diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index 8ef899e65..a2b64d8be 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -68,7 +68,7 @@ const displaySuffixes = { }; function getBestUnit(bytes: number, si: boolean = false, sigfig: number = 3): string { - if (bytes < 0) { + if (bytes === undefined || bytes < 0) { return "-"; } const units = si ? ["kB", "MB", "GB", "TB"] : ["KiB", "MiB", "GiB", "TiB"]; @@ -720,7 +720,7 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { TabRpcClient, { info: { - path: dirPath, + path: await model.formatRemoteUri(dirPath), }, }, null @@ -731,13 +731,13 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { }, [conn, dirPath, refreshVersion]); useEffect(() => { - const filtered = unfilteredData.filter((fileInfo) => { + const filtered = unfilteredData?.filter((fileInfo) => { if (!showHiddenFiles && fileInfo.name.startsWith(".") && fileInfo.name != "..") { return false; } return fileInfo.name.toLowerCase().includes(searchText); }); - setFilteredData(filtered); + setFilteredData(filtered ?? []); }, [unfilteredData, showHiddenFiles, searchText]); useEffect(() => { @@ -816,12 +816,11 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { onSave: (newName: string) => { console.log(`newFile: ${newName}`); fireAndForget(async () => { - const connection = await globalStore.get(model.connection); await RpcApi.FileCreateCommand( TabRpcClient, { info: { - path: `wsh://${connection}/${dirPath}/${newName}`, + path: await model.formatRemoteUri(`${dirPath}/${newName}`), }, }, null diff --git a/frontend/app/view/preview/preview.tsx b/frontend/app/view/preview/preview.tsx index f70f3acb6..a59d87ed7 100644 --- a/frontend/app/view/preview/preview.tsx +++ b/frontend/app/view/preview/preview.tsx @@ -368,12 +368,12 @@ export class PreviewModel implements ViewModel { if (fileName == null) { return null; } - const conn = (await get(this.connection)) ?? ""; const statFile = await RpcApi.FileInfoCommand(TabRpcClient, { info: { - path: `wsh://${conn}/${fileName}`, + path: await this.formatRemoteUri(fileName), }, }); + console.log("stat file", statFile); return statFile; }); this.fileMimeType = atom>(async (get) => { @@ -389,12 +389,12 @@ export class PreviewModel implements ViewModel { if (fileName == null) { return null; } - const conn = (await get(this.connection)) ?? ""; const file = await RpcApi.FileReadCommand(TabRpcClient, { info: { - path: `wsh://${conn}/${fileName}`, + path: await this.formatRemoteUri(fileName), }, }); + console.log("full file", file); return file; }); @@ -413,7 +413,7 @@ export class PreviewModel implements ViewModel { const fullFile = await get(fullFileAtom); return base64ToString(fullFile?.data64); }, - (get, set, update: string) => { + (_, set, update: string) => { set(this.fileContentSaved, update); } ); @@ -599,11 +599,10 @@ export class PreviewModel implements ViewModel { console.log("not saving file, newFileContent is null"); return; } - const conn = (await globalStore.get(this.connection)) ?? ""; try { await RpcApi.FileWriteCommand(TabRpcClient, { info: { - path: `wsh://${conn}/${filePath}`, + path: await this.formatRemoteUri(filePath), }, data64: stringToBase64(newFileContent), }); @@ -778,6 +777,11 @@ export class PreviewModel implements ViewModel { } return false; } + + async formatRemoteUri(path: string): Promise { + const conn = (await globalStore.get(this.connection)) ?? "local"; + return `wsh://${conn}/${path}`; + } } function makePreviewModel(blockId: string, nodeModel: BlockNodeModel): PreviewModel { diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index aad924a40..3d4a875e9 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -4,6 +4,7 @@ package connparse import ( + "log" "strings" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -78,7 +79,6 @@ func ParseURI(uri string) (*Connection, error) { } } else if strings.HasPrefix(rest, "/~") { host = wshrpc.LocalConnName - remotePath = strings.TrimPrefix(rest, "/") } else { host = ConnHostCurrent remotePath = rest @@ -94,9 +94,14 @@ func ParseURI(uri string) (*Connection, error) { } } - if host == "" && scheme == ConnectionTypeWsh { - host = wshrpc.LocalConnName - remotePath = strings.TrimPrefix(rest, "/") + if scheme == ConnectionTypeWsh { + if host == "" { + host = wshrpc.LocalConnName + } + if strings.HasPrefix(remotePath, "/~") { + log.Printf("Fixing relative path to %s; %s", host, remotePath) + remotePath = strings.TrimPrefix(remotePath, "/") + } } return &Connection{ diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 89e66bf16..372de0da2 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -54,6 +54,7 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } func Read(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { + log.Printf("Read: path=%s", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path) @@ -62,6 +63,7 @@ func Read(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { } func ReadStream(ctx context.Context, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + log.Printf("ReadStream: path=%s", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) @@ -152,3 +154,12 @@ func Delete(ctx context.Context, path string) error { } return client.Delete(ctx, conn) } + +func Join(ctx context.Context, path string, parts ...string) (string, error) { + log.Printf("Join: path=%s, parts=%v", path, parts) + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return "", fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + } + return client.Join(ctx, conn, parts...) +} diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index baf8f2bfc..933aaf66c 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -28,6 +28,8 @@ type FileShareClient interface { Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error // Delete deletes the entry at the given path Delete(ctx context.Context, conn *connparse.Connection) error + // Join joins the given parts to the connection path + Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) // GetConnectionType returns the type of connection for the fileshare GetConnectionType() string } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 77fc0a8e5..1c7e7e173 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -65,6 +65,10 @@ func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection) error return nil } +func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { + return "", nil +} + func (c S3Client) GetConnectionType() string { return connparse.ConnectionTypeS3 } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index ca80918dd..96a4cd2b5 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -8,6 +8,7 @@ import ( "encoding/base64" "fmt" "io/fs" + "path" "strings" "github.com/wavetermdev/waveterm/pkg/filestore" @@ -61,7 +62,10 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w if zoneId == "" { return nil, fmt.Errorf("zoneid not found in connection") } - fileName := conn.Path + fileName, err := cleanPath(conn.Path) + if err != nil { + return nil, fmt.Errorf("error cleaning path: %w", err) + } if data.At != nil { _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, data.At.Size) if err == nil { @@ -107,8 +111,10 @@ func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, if zoneId == "" { return nil, fmt.Errorf("zoneid not found in connection") } - fileName := conn.Path - prefix := fileName + prefix, err := cleanPath(conn.Path) + if err != nil { + return nil, fmt.Errorf("error cleaning path: %w", err) + } fileListOrig, err := filestore.WFS.ListFiles(ctx, zoneId) if err != nil { return nil, fmt.Errorf("error listing blockfiles: %w", err) @@ -174,7 +180,10 @@ func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshr if zoneId == "" { return nil, fmt.Errorf("zoneid not found in connection") } - fileName := conn.Path + fileName, err := cleanPath(conn.Path) + if err != nil { + return nil, fmt.Errorf("error cleaning path: %w", err) + } fileInfo, err := filestore.WFS.Stat(ctx, zoneId, fileName) if err != nil { if err == fs.ErrNotExist { @@ -194,7 +203,10 @@ func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, dat if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - fileName := conn.Path + fileName, err := cleanPath(conn.Path) + if err != nil { + return fmt.Errorf("error cleaning path: %w", err) + } if data.At != nil { err = filestore.WFS.WriteAt(ctx, zoneId, fileName, data.At.Offset, dataBuf) if err == fs.ErrNotExist { @@ -242,8 +254,11 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) erro if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - fileName := conn.Path - err := filestore.WFS.DeleteFile(ctx, zoneId, fileName) + fileName, err := cleanPath(conn.Path) + if err != nil { + return fmt.Errorf("error cleaning path: %w", err) + } + err = filestore.WFS.DeleteFile(ctx, zoneId, fileName) if err != nil { return fmt.Errorf("error deleting blockfile: %w", err) } @@ -259,6 +274,35 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) erro return nil } +func (c WaveClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { + newPath := path.Join(append([]string{conn.Path}, parts...)...) + newPath, err := cleanPath(newPath) + if err != nil { + return "", fmt.Errorf("error cleaning path: %w", err) + } + return newPath, nil +} + +func cleanPath(path string) (string, error) { + if path == "" { + return "", fmt.Errorf("path is empty") + } + if strings.HasPrefix(path, "/") || strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { + return "", fmt.Errorf("path cannot start with /, ~, ., or ..") + } + var newParts []string + for _, part := range strings.Split(path, "/") { + if part == ".." { + if len(newParts) > 0 { + newParts = newParts[:len(newParts)-1] + } + } else if part != "." { + newParts = append(newParts, part) + } + } + return strings.Join(newParts, "/"), nil +} + func (c WaveClient) GetConnectionType() string { return connparse.ConnectionTypeWave } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index b814facfc..2c941d50a 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -9,6 +9,7 @@ import ( "encoding/base64" "fmt" "io" + "log" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -38,6 +39,7 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws resp := respUnion.Response if firstPk { firstPk = false + log.Printf("stream file, first pk: %v", resp) // first packet has the fileinfo if resp.Info == nil { return nil, fmt.Errorf("stream file protocol error, first pk fileinfo is empty") @@ -50,6 +52,7 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws } if isDir { if len(resp.Entries) == 0 { + log.Printf("stream dir, no entries") continue } fileData.Entries = append(fileData.Entries, resp.Entries...) @@ -121,6 +124,14 @@ func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error return wshclient.RemoteFileDeleteCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } +func (c WshClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { + finfo, err := wshclient.RemoteFileJoinCommand(wshclient.GetBareRpcClient(), append([]string{conn.Path}, parts...), &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + if err != nil { + return "", err + } + return finfo.Path, nil +} + func (c WshClient) GetConnectionType() string { return connparse.ConnectionTypeWsh } From 714c60d83f1099954ed93a7fdb0d14455a1aa7e1 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 15 Jan 2025 20:50:19 -0800 Subject: [PATCH 047/126] fix merge error --- frontend/types/gotypes.d.ts | 2 +- pkg/wconfig/settingsconfig.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 4906b6791..bde17d9e8 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -289,7 +289,6 @@ declare global { "ssh:hostname"?: string; "ssh:port"?: string; "ssh:identityfile"?: string[]; - "ssh:identitiesonly"?: boolean; "ssh:batchmode"?: boolean; "ssh:pubkeyauthentication"?: boolean; "ssh:passwordauthentication"?: boolean; @@ -297,6 +296,7 @@ declare global { "ssh:preferredauthentications"?: string[]; "ssh:addkeystoagent"?: boolean; "ssh:identityagent"?: string; + "ssh:identitiesonly"?: boolean; "ssh:proxyjump"?: string[]; "ssh:userknownhostsfile"?: string[]; "ssh:globalknownhostsfile"?: string[]; diff --git a/pkg/wconfig/settingsconfig.go b/pkg/wconfig/settingsconfig.go index 1bc30a1bf..6743a6952 100644 --- a/pkg/wconfig/settingsconfig.go +++ b/pkg/wconfig/settingsconfig.go @@ -161,6 +161,7 @@ type ConnKeywords struct { SshPreferredAuthentications []string `json:"ssh:preferredauthentications,omitempty"` SshAddKeysToAgent *bool `json:"ssh:addkeystoagent,omitempty"` SshIdentityAgent *string `json:"ssh:identityagent,omitempty"` + SshIdentitiesOnly *bool `json:"ssh:identitiesonly,omitempty"` SshProxyJump []string `json:"ssh:proxyjump,omitempty"` SshUserKnownHostsFile []string `json:"ssh:userknownhostsfile,omitempty"` SshGlobalKnownHostsFile []string `json:"ssh:globalknownhostsfile,omitempty"` From 149126a2d0670b88f222c3367fa5c391bb1619d2 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 15 Jan 2025 20:55:09 -0800 Subject: [PATCH 048/126] update comment --- pkg/remote/sshclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/remote/sshclient.go b/pkg/remote/sshclient.go index 6aa81d58b..bb9c3f420 100644 --- a/pkg/remote/sshclient.go +++ b/pkg/remote/sshclient.go @@ -579,7 +579,7 @@ func createClientConfig(connCtx context.Context, sshKeywords *wconfig.ConnKeywor var authSockSigners []ssh.Signer var agentClient agent.ExtendedAgent - // IdentitiesOnly indicates that only the keys listed in IdentityFile should be used, even if there are matches in the SSH Agent, PKCS11Provider, or SecurityKeyProvider. See https://man.openbsd.org/ssh_config#IdentitiesOnly + // IdentitiesOnly indicates that only the keys listed in the identity and certificate files or passed as arguments should be used, even if there are matches in the SSH Agent, PKCS11Provider, or SecurityKeyProvider. See https://man.openbsd.org/ssh_config#IdentitiesOnly // TODO: Update if we decide to support PKCS11Provider and SecurityKeyProvider if !utilfn.SafeDeref(sshKeywords.SshIdentitiesOnly) { conn, err := net.Dial("unix", utilfn.SafeDeref(sshKeywords.SshIdentityAgent)) From 47df22fedb8ac26fb6d61fdd5282f140e057e822 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 16 Jan 2025 14:10:32 -0800 Subject: [PATCH 049/126] add tar stream command --- frontend/app/store/wshclientapi.ts | 10 +++++ go.mod | 3 +- pkg/remote/fileshare/fileshare.go | 23 ++++++----- pkg/remote/fileshare/fstype/fstype.go | 9 +++- pkg/remote/fileshare/s3fs/s3fs.go | 8 +++- pkg/remote/fileshare/wavefs/wavefs.go | 8 +++- pkg/remote/fileshare/wshfs/wshfs.go | 8 +++- pkg/util/iochan/iochan.go | 52 +++++++++++++++++++++++ pkg/wshrpc/wshclient/wshclient.go | 10 +++++ pkg/wshrpc/wshremote/wshremote.go | 59 +++++++++++++++++++++++---- pkg/wshrpc/wshrpctypes.go | 5 ++- pkg/wshrpc/wshserver/wshserver.go | 6 ++- pkg/wshutil/wshutil.go | 11 +++++ 13 files changed, 182 insertions(+), 30 deletions(-) create mode 100644 pkg/util/iochan/iochan.go diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 68c113ff4..6e458badb 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -187,6 +187,11 @@ class RpcApiType { return client.wshRpcCall("fileread", data, opts); } + // command "filestreamtar" [responsestream] + FileStreamTarCommand(client: WshClient, data: FileData, opts?: RpcOpts): AsyncGenerator { + return client.wshRpcStream("filestreamtar", data, opts); + } + // command "filewrite" [call] FileWriteCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("filewrite", data, opts); @@ -287,6 +292,11 @@ class RpcApiType { return client.wshRpcStream("remotestreamfile", data, opts); } + // command "remotetarstream" [responsestream] + RemoteTarStreamCommand(client: WshClient, data: string, opts?: RpcOpts): AsyncGenerator { + return client.wshRpcStream("remotetarstream", data, opts); + } + // command "remotewritefile" [call] RemoteWriteFileCommand(client: WshClient, data: CommandRemoteWriteFileData, opts?: RpcOpts): Promise { return client.wshRpcCall("remotewritefile", data, opts); diff --git a/go.mod b/go.mod index 3cdfdbf6e..377eb1396 100644 --- a/go.mod +++ b/go.mod @@ -32,9 +32,8 @@ require ( golang.org/x/mod v0.22.0 golang.org/x/sys v0.29.0 golang.org/x/term v0.28.0 - google.golang.org/api v0.216.0 - gopkg.in/ini.v1 v1.67.0 google.golang.org/api v0.217.0 + gopkg.in/ini.v1 v1.67.0 ) require ( diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 372de0da2..ae1bd3e88 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -62,18 +62,24 @@ func Read(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { return client.Read(ctx, conn, data) } -func ReadStream(ctx context.Context, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { +func ReadStream(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { log.Printf("ReadStream: path=%s", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { - rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 1) - rtn <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Error: fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path)} - close(rtn) - return rtn + return wshutil.SendErrCh[wshrpc.FileData](fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path)) } return client.ReadStream(ctx, conn, data) } +func ReadTarStream(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { + log.Printf("ReadTarStream: path=%s", data.Info.Path) + client, conn := CreateFileShareClient(ctx, data.Info.Path) + if conn == nil || client == nil { + return wshutil.SendErrCh[[]byte](fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path)) + } + return client.ReadTarStream(ctx, conn, data) +} + func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { log.Printf("ListEntries: path=%s", path) client, conn := CreateFileShareClient(ctx, path) @@ -83,14 +89,11 @@ func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([ return client.ListEntries(ctx, conn, opts) } -func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { log.Printf("ListEntriesStream: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - rtn := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 1) - rtn <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Error: fmt.Errorf("error creating fileshare client, could not parse connection %s", path)} - close(rtn) - return rtn + return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("error creating fileshare client, could not parse connection %s", path)) } return client.ListEntriesStream(ctx, conn, opts) } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 933aaf66c..b2f81513d 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -1,3 +1,6 @@ +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + package fstype import ( @@ -13,11 +16,13 @@ type FileShareClient interface { // Read returns the file info at the given path, if it's a directory, then the list of entries Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) // ReadStream returns a stream of file data at the given path. If it's a directory, then the list of entries - ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] + ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] + // ReadTarStream returns a stream of tar data at the given path + ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] // ListEntries returns the list of entries at the given path, or nothing if the path is a file ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) // ListEntriesStream returns a stream of entries at the given path - ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] + ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] // PutFile writes the given data to the file at the given path PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error // Mkdir creates a directory at the given path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 1c7e7e173..b5af85e6b 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -29,7 +29,11 @@ func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wsh return nil, nil } -func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { +func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { + return nil +} + +func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { return nil } @@ -37,7 +41,7 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o return nil, nil } -func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { return nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 96a4cd2b5..1477b5a18 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -28,7 +28,7 @@ func NewWaveClient() *WaveClient { return &WaveClient{} } -func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { +func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) go func() { defer close(ch) @@ -90,7 +90,11 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w return &wshrpc.FileData{Info: data.Info, Entries: list}, nil } -func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { + return nil +} + +func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) go func() { defer close(ch) diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 2c941d50a..6b69dbb07 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -73,7 +73,7 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws return fileData, nil } -func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { +func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { byteRange := "" if data.At != nil && data.At.Size > 0 { byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+data.At.Size) @@ -82,6 +82,10 @@ func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, d return wshclient.RemoteStreamFileCommand(wshclient.GetBareRpcClient(), streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } +func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { + return wshclient.RemoteTarStreamCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) +} + func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { var entries []*wshrpc.FileInfo rtnCh := c.ListEntriesStream(ctx, conn, opts) @@ -95,7 +99,7 @@ func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, return entries, nil } -func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { return wshclient.RemoteListEntriesCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go new file mode 100644 index 000000000..7c4d7c2ac --- /dev/null +++ b/pkg/util/iochan/iochan.go @@ -0,0 +1,52 @@ +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +// allows for streaming an io.Reader to a channel and an io.Writer from a channel +package iochan + +import ( + "context" + "io" + "log" + + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) + +// ReaderChan reads from an io.Reader and sends the data to a channel +func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) <-chan wshrpc.RespOrErrorUnion[[]byte] { + ch := make(chan wshrpc.RespOrErrorUnion[[]byte], 16) + go func() { + defer close(ch) + buf := make([]byte, chunkSize) + for { + if ctx.Err() != nil { + log.Printf("ReaderChan: context error: %v", ctx.Err()) + callback() + return + } + n, err := r.Read(buf) + if err != nil && err != io.EOF { + ch <- wshrpc.RespOrErrorUnion[[]byte]{Error: err} + callback() + return + } + if n > 0 { + ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} + } + } + }() + return ch +} + +// WriterChan reads from a channel and writes the data to an io.Writer +func WriterChan(w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { + go func() { + for resp := range ch { + if resp.Error != nil { + log.Printf("WriterChan: error: %v", resp.Error) + return + } + w.Write(resp.Response) + } + }() +} diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 6b932a241..cab5a6015 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -229,6 +229,11 @@ func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOp return resp, err } +// command "filestreamtar", wshserver.FileStreamTarCommand +func FileStreamTarCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { + return sendRpcRequestResponseStreamHelper[[]uint8](w, "filestreamtar", data, opts) +} + // command "filewrite", wshserver.FileWriteCommand func FileWriteCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "filewrite", data, opts) @@ -346,6 +351,11 @@ func RemoteStreamFileCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamF return sendRpcRequestResponseStreamHelper[wshrpc.FileData](w, "remotestreamfile", data, opts) } +// command "remotetarstream", wshserver.RemoteTarStreamCommand +func RemoteTarStreamCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { + return sendRpcRequestResponseStreamHelper[[]uint8](w, "remotetarstream", data, opts) +} + // command "remotewritefile", wshserver.RemoteWriteFileCommand func RemoteWriteFileCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteWriteFileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "remotewritefile", data, opts) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index da35bf224..fcbb12572 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -4,6 +4,7 @@ package wshremote import ( + "archive/tar" "context" "encoding/base64" "errors" @@ -16,6 +17,7 @@ import ( "strings" "github.com/wavetermdev/waveterm/pkg/util/fileutil" + "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -41,10 +43,6 @@ func (impl *ServerImpl) MessageCommand(ctx context.Context, data wshrpc.CommandM return nil } -func respErr[T any](err error) wshrpc.RespOrErrorUnion[T] { - return wshrpc.RespOrErrorUnion[T]{Error: err} -} - type ByteRangeType struct { All bool Start int64 @@ -202,19 +200,64 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: resp} }) if err != nil { - ch <- respErr[wshrpc.FileData](err) + ch <- wshutil.RespErr[wshrpc.FileData](err) } }() return ch } +func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, path string) <-chan wshrpc.RespOrErrorUnion[[]byte] { + path, err := wavebase.ExpandHomeDir(path) + if err != nil { + return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot expand path %q: %w", path, err)) + } + cleanedPath := filepath.Clean(wavebase.ExpandHomeDirSafe(path)) + finfo, err := os.Stat(cleanedPath) + if err != nil { + return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot stat file %q: %w", path, err)) + } + pipeReader, pipeWriter := io.Pipe() + tarWriter := tar.NewWriter(pipeWriter) + if finfo.IsDir() { + err := tarWriter.AddFS(os.DirFS(path)) + if err != nil { + return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) + } + rtn := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { + pipeWriter.Close() + tarWriter.Close() + }) + return rtn + } else { + header, err := tar.FileInfoHeader(finfo, "") + if err != nil { + return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot create tar header for %q: %w", path, err)) + } + err = tarWriter.WriteHeader(header) + if err != nil { + return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot write tar header for %q: %w", path, err)) + } + file, err := os.Open(cleanedPath) + if err != nil { + return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot open file %q: %w", path, err)) + } + file.WriteTo(tarWriter) + rtn := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { + file.Close() + pipeWriter.Close() + tarWriter.Close() + }) + return rtn + } +} + func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrpc.CommandRemoteListEntriesData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) go func() { defer close(ch) path, err := wavebase.ExpandHomeDir(data.Path) if err != nil { - ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](err) + ch <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) return } innerFilesEntries := []os.DirEntry{} @@ -245,14 +288,14 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp } else { innerFilesEntries, err = os.ReadDir(path) if err != nil { - ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("cannot open dir %q: %w", path, err)) + ch <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("cannot open dir %q: %w", path, err)) return } } var fileInfoArr []*wshrpc.FileInfo for _, innerFileEntry := range innerFilesEntries { if ctx.Err() != nil { - ch <- respErr[wshrpc.CommandRemoteListEntriesRtnData](ctx.Err()) + ch <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](ctx.Err()) return } innerFileInfoInt, err := innerFileEntry.Info() diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 023020d19..36435b78c 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -76,6 +76,7 @@ const ( Command_SetConnectionsConfig = "connectionsconfig" Command_GetFullConfig = "getfullconfig" Command_RemoteStreamFile = "remotestreamfile" + Command_RemoteTarStream = "remotetarstream" Command_RemoteFileInfo = "remotefileinfo" Command_RemoteFileTouch = "remotefiletouch" Command_RemoteWriteFile = "remotewritefile" @@ -149,9 +150,10 @@ type WshRpcInterface interface { FileAppendIJsonCommand(ctx context.Context, data CommandAppendIJsonData) error FileWriteCommand(ctx context.Context, data FileData) error FileReadCommand(ctx context.Context, data FileData) (*FileData, error) + FileStreamTarCommand(ctx context.Context, data FileData) <-chan RespOrErrorUnion[[]byte] FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) FileListCommand(ctx context.Context, data FileListData) ([]*FileInfo, error) - FileListStreamCommand(ctx context.Context, data FileListData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] + FileListStreamCommand(ctx context.Context, data FileListData) <-chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] EventPublishCommand(ctx context.Context, data wps.WaveEvent) error EventSubCommand(ctx context.Context, data wps.SubscriptionRequest) error EventUnsubCommand(ctx context.Context, data string) error @@ -190,6 +192,7 @@ type WshRpcInterface interface { // remotes RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[FileData] + RemoteTarStreamCommand(ctx context.Context, path string) <-chan RespOrErrorUnion[[]byte] RemoteListEntriesCommand(ctx context.Context, data CommandRemoteListEntriesData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) RemoteFileTouchCommand(ctx context.Context, path string) error diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index bb3046ee1..f5f3ad2c3 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -307,7 +307,7 @@ func (ws *WshServer) FileListCommand(ctx context.Context, data wshrpc.FileListDa return fileshare.ListEntries(ctx, data.Path, data.Opts) } -func (ws *WshServer) FileListStreamCommand(ctx context.Context, data wshrpc.FileListData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { +func (ws *WshServer) FileListStreamCommand(ctx context.Context, data wshrpc.FileListData) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { return fileshare.ListEntriesStream(ctx, data.Path, data.Opts) } @@ -319,6 +319,10 @@ func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) return fileshare.Read(ctx, data) } +func (ws *WshServer) FileStreamTarCommand(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { + return fileshare.ReadTarStream(ctx, data) +} + func (ws *WshServer) FileAppendIJsonCommand(ctx context.Context, data wshrpc.CommandAppendIJsonData) error { tryCreate := true if data.FileName == blockcontroller.BlockFile_VDom && tryCreate { diff --git a/pkg/wshutil/wshutil.go b/pkg/wshutil/wshutil.go index 3bdb17e6c..4f83ead2a 100644 --- a/pkg/wshutil/wshutil.go +++ b/pkg/wshutil/wshutil.go @@ -569,3 +569,14 @@ func InstallRcFiles() error { wshBinDir := filepath.Join(waveDir, wavebase.RemoteWshBinDirName) return shellutil.InitRcFiles(waveDir, wshBinDir) } + +func SendErrCh[T any](err error) <-chan wshrpc.RespOrErrorUnion[T] { + ch := make(chan wshrpc.RespOrErrorUnion[T], 1) + ch <- RespErr[T](err) + close(ch) + return ch +} + +func RespErr[T any](err error) wshrpc.RespOrErrorUnion[T] { + return wshrpc.RespOrErrorUnion[T]{Error: err} +} From 81c7b4752785e954f2c565634b5ad6ee08db6ad6 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 16 Jan 2025 14:19:20 -0800 Subject: [PATCH 050/126] add iochan test --- pkg/util/iochan/iochan_test.go | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 pkg/util/iochan/iochan_test.go diff --git a/pkg/util/iochan/iochan_test.go b/pkg/util/iochan/iochan_test.go new file mode 100644 index 000000000..d54d2704c --- /dev/null +++ b/pkg/util/iochan/iochan_test.go @@ -0,0 +1,40 @@ +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +package iochan_test + +import ( + "context" + "io" + "testing" + + "github.com/wavetermdev/waveterm/pkg/util/iochan" +) + +func TestIochan_Basic(t *testing.T) { + pipeReader, pipeWriter := io.Pipe() + packet := []byte("hello world") + go func() { + pipeWriter.Write(packet) + pipeWriter.Close() + }() + destPipeReader, destPipeWriter := io.Pipe() + defer destPipeReader.Close() + defer destPipeWriter.Close() + ioch := iochan.ReaderChan(context.TODO(), pipeReader, 1024, func() { + pipeReader.Close() + pipeWriter.Close() + }) + iochan.WriterChan(destPipeWriter, ioch) + buf := make([]byte, 1024) + n, err := destPipeReader.Read(buf) + if err != nil { + t.Fatalf("Read failed: %v", err) + } + if n != len(packet) { + t.Fatalf("Read length mismatch: %d != %d", n, len(packet)) + } + if string(buf[:n]) != string(packet) { + t.Fatalf("Read data mismatch: %s != %s", buf[:n], packet) + } +} From 231157769c62a9476df5a1bd48353b7b830db79b Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 16 Jan 2025 16:34:20 -0800 Subject: [PATCH 051/126] tar seems to be working, can't write files to os --- cmd/wsh/cmd/wshcmd-file-util.go | 17 +-- cmd/wsh/cmd/wshcmd-file.go | 36 +++-- frontend/app/store/wshclientapi.ts | 14 +- frontend/types/gotypes.d.ts | 27 ++++ pkg/remote/connparse/connparse.go | 1 + pkg/remote/fileshare/fileshare.go | 62 ++++---- pkg/remote/fileshare/fstype/fstype.go | 8 +- pkg/remote/fileshare/s3fs/s3fs.go | 4 +- pkg/remote/fileshare/wavefs/wavefs.go | 4 +- pkg/remote/fileshare/wshfs/wshfs.go | 9 +- pkg/util/iochan/iochan.go | 4 +- pkg/util/iochan/iochan_test.go | 14 +- pkg/wshrpc/wshclient/wshclient.go | 16 ++- pkg/wshrpc/wshremote/wshremote.go | 194 ++++++++++++++++++++++---- pkg/wshrpc/wshrpctypes.go | 30 +++- pkg/wshrpc/wshserver/wshserver.go | 6 +- 16 files changed, 345 insertions(+), 101 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index da5ffbe80..547b8d584 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "io/fs" - "log" "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" @@ -215,14 +214,16 @@ func fixRelativePaths(path string) (string, error) { if err != nil { return "", err } - if conn.Scheme == connparse.ConnectionTypeWsh && conn.Host == connparse.ConnHostCurrent { - conn.Host = RpcContext.Conn - log.Printf("Fixing relative path to %s", conn.Host) - fixedPath, err := fileutil.FixPath(conn.Path) - if err != nil { - return "", err + if conn.Scheme == connparse.ConnectionTypeWsh { + fixedPath := conn.Path + if conn.Host == connparse.ConnHostCurrent { + conn.Host = RpcContext.Conn + fixedPath, err = fileutil.FixPath(conn.Path) + if err != nil { + return "", err + } } - return fmt.Sprintf("wsh://%s/%s", RpcContext.Conn, fixedPath), nil + return fmt.Sprintf("wsh://%s/%s", conn.Host, fixedPath), nil } return path, nil } diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 30b30e07e..5c6a9c35d 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -87,6 +87,9 @@ func init() { fileCmd.AddCommand(fileRmCmd) fileCmd.AddCommand(fileInfoCmd) fileCmd.AddCommand(fileAppendCmd) + fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") + fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") + fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") fileCmd.AddCommand(fileCpCmd) } @@ -368,23 +371,34 @@ func getTargetPath(src, dst string) (string, error) { } func fileCpRun(cmd *cobra.Command, args []string) error { - src, origDst := args[0], args[1] - dst, err := getTargetPath(src, origDst) + src, dst := args[0], args[1] + recursive, err := cmd.Flags().GetBool("recursive") if err != nil { return err } - srcIsWave := strings.HasPrefix(src, WaveFilePrefix) - dstIsWave := strings.HasPrefix(dst, WaveFilePrefix) - - if srcIsWave == dstIsWave { - return fmt.Errorf("exactly one file must be a wavefile:// URL") + merge, err := cmd.Flags().GetBool("merge") + if err != nil { + return err + } + force, err := cmd.Flags().GetBool("force") + if err != nil { + return err } - if srcIsWave { - return copyFromWaveToLocal(src, dst) - } else { - return copyFromLocalToWave(src, dst) + srcPath, err := fixRelativePaths(src) + if err != nil { + return fmt.Errorf("unable to parse src path: %w", err) + } + destPath, err := fixRelativePaths(dst) + if err != nil { + return fmt.Errorf("unable to parse dest path: %w", err) } + log.Printf("Copying %s to %s; recursive: %v, merge: %v, force: %v", srcPath, destPath, recursive, merge, force) + err = wshclient.FileCopyCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcPath, DestUri: destPath, Opts: &wshrpc.FileCopyOpts{Recursive: recursive, Merge: merge, Overwrite: force}}, &wshrpc.RpcOpts{Timeout: fileTimeout}) + if err != nil { + return fmt.Errorf("copying file: %w", err) + } + return nil } func copyFromWaveToLocal(src, dst string) error { diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 6e458badb..1dcee0799 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -157,6 +157,11 @@ class RpcApiType { return client.wshRpcCall("fileappendijson", data, opts); } + // command "filecopy" [call] + FileCopyCommand(client: WshClient, data: CommandFileCopyData, opts?: RpcOpts): Promise { + return client.wshRpcCall("filecopy", data, opts); + } + // command "filecreate" [call] FileCreateCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("filecreate", data, opts); @@ -188,7 +193,7 @@ class RpcApiType { } // command "filestreamtar" [responsestream] - FileStreamTarCommand(client: WshClient, data: FileData, opts?: RpcOpts): AsyncGenerator { + FileStreamTarCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("filestreamtar", data, opts); } @@ -237,6 +242,11 @@ class RpcApiType { return client.wshRpcCall("path", data, opts); } + // command "remotefilecopy" [call] + RemoteFileCopyCommand(client: WshClient, data: CommandRemoteFileCopyData, opts?: RpcOpts): Promise { + return client.wshRpcCall("remotefilecopy", data, opts); + } + // command "remotefiledelete" [call] RemoteFileDeleteCommand(client: WshClient, data: string, opts?: RpcOpts): Promise { return client.wshRpcCall("remotefiledelete", data, opts); @@ -293,7 +303,7 @@ class RpcApiType { } // command "remotetarstream" [responsestream] - RemoteTarStreamCommand(client: WshClient, data: string, opts?: RpcOpts): AsyncGenerator { + RemoteTarStreamCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotetarstream", data, opts); } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 2be1373bd..cc9c6cb37 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -178,6 +178,13 @@ declare global { maxitems: number; }; + // wshrpc.CommandFileCopyData + type CommandFileCopyData = { + srcuri: string; + desturi: string; + opts?: FileCopyOpts; + }; + // wshrpc.CommandGetMetaData type CommandGetMetaData = { oref: ORef; @@ -189,6 +196,13 @@ declare global { message: string; }; + // wshrpc.CommandRemoteFileCopyData + type CommandRemoteFileCopyData = { + srcuri: string; + destpath: string; + opts?: FileCopyOpts; + }; + // wshrpc.CommandRemoteListEntriesData type CommandRemoteListEntriesData = { path: string; @@ -206,6 +220,12 @@ declare global { byterange?: string; }; + // wshrpc.CommandRemoteStreamTarData + type CommandRemoteStreamTarData = { + path: string; + opts?: FileCopyOpts; + }; + // wshrpc.CommandRemoteWriteFileData type CommandRemoteWriteFileData = { path: string; @@ -346,6 +366,13 @@ declare global { height: number; }; + // wshrpc.FileCopyOpts + type FileCopyOpts = { + overwrite?: boolean; + recursive?: boolean; + merge?: boolean; + }; + // wshrpc.FileData type FileData = { info?: FileInfo; diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index 3d4a875e9..a323adc02 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -79,6 +79,7 @@ func ParseURI(uri string) (*Connection, error) { } } else if strings.HasPrefix(rest, "/~") { host = wshrpc.LocalConnName + remotePath = rest } else { host = ConnHostCurrent remotePath = rest diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index ae1bd3e88..8afe012b6 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -19,25 +19,11 @@ import ( // Returns the client and the parsed connection func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileShareClient, *connparse.Connection) { log.Printf("CreateFileShareClient: connection=%s", connection) - conn, err := connparse.ParseURI(connection) + conn, err := parseConnUriAndReplaceCurrentHost(connection) if err != nil { log.Printf("error parsing connection: %v", err) return nil, nil } - if conn.Host == connparse.ConnHostCurrent { - handler := wshutil.GetRpcResponseHandlerFromContext(ctx) - if handler == nil { - log.Printf("error getting rpc response handler from context") - return nil, nil - } - source := handler.GetRpcContext().Conn - - // RPC context connection is empty for local connections - if source == "" { - source = wshrpc.LocalConnName - } - conn.Host = source - } conntype := conn.GetType() if conntype == connparse.ConnectionTypeS3 { config, err := awsconn.GetConfig(ctx, connection) @@ -71,13 +57,13 @@ func ReadStream(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrE return client.ReadStream(ctx, conn, data) } -func ReadTarStream(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { - log.Printf("ReadTarStream: path=%s", data.Info.Path) - client, conn := CreateFileShareClient(ctx, data.Info.Path) +func ReadTarStream(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[[]byte] { + log.Printf("ReadTarStream: path=%s", data.Path) + client, conn := CreateFileShareClient(ctx, data.Path) if conn == nil || client == nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path)) + return wshutil.SendErrCh[[]byte](fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Path)) } - return client.ReadTarStream(ctx, conn, data) + return client.ReadTarStream(ctx, conn, data.Opts) } func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { @@ -143,10 +129,17 @@ func Move(ctx context.Context, srcPath, destPath string, recursive bool) error { return srcClient.Move(ctx, srcConn, destConn, recursive) } -// TODO: Implement copy across different fileshare types -func Copy(ctx context.Context, srcPath, destPath string, recursive bool) error { - log.Printf("Copy: src=%s, dest=%s", srcPath, destPath) - return nil +func Copy(ctx context.Context, data wshrpc.CommandFileCopyData) error { + log.Printf("Copy: src=%s, dest=%s", data.SrcUri, data.DestUri) + srcConn, err := parseConnUriAndReplaceCurrentHost(data.SrcUri) + if err != nil { + return fmt.Errorf("error parsing source connection %s: %v", data.SrcUri, err) + } + destClient, destConn := CreateFileShareClient(ctx, data.DestUri) + if destConn == nil || destClient == nil { + return fmt.Errorf("error creating fileshare client, could not parse connection %s or %s", data.SrcUri, data.DestUri) + } + return destClient.Copy(ctx, srcConn, destConn, data.Opts) } func Delete(ctx context.Context, path string) error { @@ -166,3 +159,24 @@ func Join(ctx context.Context, path string, parts ...string) (string, error) { } return client.Join(ctx, conn, parts...) } + +func parseConnUriAndReplaceCurrentHost(uri string) (*connparse.Connection, error) { + conn, err := connparse.ParseURI(uri) + if err != nil { + return nil, fmt.Errorf("error parsing connection: %v", err) + } + if conn.Host == connparse.ConnHostCurrent { + handler := wshutil.GetRpcResponseHandlerFromContext(context.Background()) + if handler == nil { + return nil, fmt.Errorf("error getting rpc response handler from context") + } + source := handler.GetRpcContext().Conn + + // RPC context connection is empty for local connections + if source == "" { + source = wshrpc.LocalConnName + } + conn.Host = source + } + return conn, nil +} diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index b2f81513d..4e6bdffca 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -18,7 +18,7 @@ type FileShareClient interface { // ReadStream returns a stream of file data at the given path. If it's a directory, then the list of entries ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] // ReadTarStream returns a stream of tar data at the given path - ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] + ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] // ListEntries returns the list of entries at the given path, or nothing if the path is a file ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) // ListEntriesStream returns a stream of entries at the given path @@ -27,10 +27,10 @@ type FileShareClient interface { PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error // Mkdir creates a directory at the given path Mkdir(ctx context.Context, conn *connparse.Connection) error - // Move moves the file from srcPath to destPath + // Copy copies the file from srcConn to destConn Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error - // Copy copies the file from srcPath to destPath - Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error + // Copy copies the file from srcConn to destConn + Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error // Delete deletes the entry at the given path Delete(ctx context.Context, conn *connparse.Connection) error // Join joins the given parts to the connection path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index b5af85e6b..ae3552029 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -33,7 +33,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da return nil } -func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { return nil } @@ -61,7 +61,7 @@ func (c S3Client) Move(ctx context.Context, srcConn, destConn *connparse.Connect return nil } -func (c S3Client) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { +func (c S3Client) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { return nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 1477b5a18..84c23f26b 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -90,7 +90,7 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w return &wshrpc.FileData{Info: data.Info, Entries: list}, nil } -func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { return nil } @@ -249,7 +249,7 @@ func (c WaveClient) Move(ctx context.Context, srcConn, destConn *connparse.Conne return nil } -func (c WaveClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { +func (c WaveClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { return nil } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 6b69dbb07..1c6cb1e10 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -82,8 +82,9 @@ func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, d return wshclient.RemoteStreamFileCommand(wshclient.GetBareRpcClient(), streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } -func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { - return wshclient.RemoteTarStreamCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) +func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { + log.Print("ReadTarStream wshfs") + return wshclient.RemoteTarStreamCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { @@ -120,8 +121,8 @@ func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connec return wshclient.RemoteFileRenameCommand(wshclient.GetBareRpcClient(), [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) } -func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { - return nil +func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { + return wshclient.RemoteFileCopyCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestPath: destConn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) } func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 7c4d7c2ac..b6db354e4 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -13,7 +13,7 @@ import ( ) // ReaderChan reads from an io.Reader and sends the data to a channel -func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) chan wshrpc.RespOrErrorUnion[[]byte] { ch := make(chan wshrpc.RespOrErrorUnion[[]byte], 16) go func() { defer close(ch) @@ -39,7 +39,7 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } // WriterChan reads from a channel and writes the data to an io.Writer -func WriterChan(w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { +func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { go func() { for resp := range ch { if resp.Error != nil { diff --git a/pkg/util/iochan/iochan_test.go b/pkg/util/iochan/iochan_test.go index d54d2704c..b1a7361b3 100644 --- a/pkg/util/iochan/iochan_test.go +++ b/pkg/util/iochan/iochan_test.go @@ -12,20 +12,20 @@ import ( ) func TestIochan_Basic(t *testing.T) { - pipeReader, pipeWriter := io.Pipe() + srcPipeReader, srcPipeWriter := io.Pipe() packet := []byte("hello world") go func() { - pipeWriter.Write(packet) - pipeWriter.Close() + srcPipeWriter.Write(packet) + srcPipeWriter.Close() }() destPipeReader, destPipeWriter := io.Pipe() defer destPipeReader.Close() defer destPipeWriter.Close() - ioch := iochan.ReaderChan(context.TODO(), pipeReader, 1024, func() { - pipeReader.Close() - pipeWriter.Close() + ioch := iochan.ReaderChan(context.TODO(), srcPipeReader, 1024, func() { + srcPipeReader.Close() + srcPipeWriter.Close() }) - iochan.WriterChan(destPipeWriter, ioch) + iochan.WriterChan(context.TODO(), destPipeWriter, ioch) buf := make([]byte, 1024) n, err := destPipeReader.Read(buf) if err != nil { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index cab5a6015..348c7ffd6 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -194,6 +194,12 @@ func FileAppendIJsonCommand(w *wshutil.WshRpc, data wshrpc.CommandAppendIJsonDat return err } +// command "filecopy", wshserver.FileCopyCommand +func FileCopyCommand(w *wshutil.WshRpc, data wshrpc.CommandFileCopyData, opts *wshrpc.RpcOpts) error { + _, err := sendRpcRequestCallHelper[any](w, "filecopy", data, opts) + return err +} + // command "filecreate", wshserver.FileCreateCommand func FileCreateCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "filecreate", data, opts) @@ -230,7 +236,7 @@ func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOp } // command "filestreamtar", wshserver.FileStreamTarCommand -func FileStreamTarCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { +func FileStreamTarCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTarData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { return sendRpcRequestResponseStreamHelper[[]uint8](w, "filestreamtar", data, opts) } @@ -288,6 +294,12 @@ func PathCommand(w *wshutil.WshRpc, data wshrpc.PathCommandData, opts *wshrpc.Rp return resp, err } +// command "remotefilecopy", wshserver.RemoteFileCopyCommand +func RemoteFileCopyCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteFileCopyData, opts *wshrpc.RpcOpts) error { + _, err := sendRpcRequestCallHelper[any](w, "remotefilecopy", data, opts) + return err +} + // command "remotefiledelete", wshserver.RemoteFileDeleteCommand func RemoteFileDeleteCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "remotefiledelete", data, opts) @@ -352,7 +364,7 @@ func RemoteStreamFileCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamF } // command "remotetarstream", wshserver.RemoteTarStreamCommand -func RemoteTarStreamCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { +func RemoteTarStreamCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTarData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { return sendRpcRequestResponseStreamHelper[[]uint8](w, "remotetarstream", data, opts) } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index fcbb12572..e5e0b2de0 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -21,6 +21,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshutil" ) @@ -206,7 +207,10 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc return ch } -func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, path string) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[[]byte] { + path := data.Path + opts := data.Opts + log.Printf("RemoteTarStreamCommand: path=%s\n", path) path, err := wavebase.ExpandHomeDir(path) if err != nil { return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot expand path %q: %w", path, err)) @@ -218,37 +222,173 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, path string) } pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) - if finfo.IsDir() { - err := tarWriter.AddFS(os.DirFS(path)) - if err != nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) + iochanCtx, cancel := context.WithCancel(ctx) + rtn := iochan.ReaderChan(iochanCtx, pipeReader, wshrpc.FileChunkSize, func() { + pipeReader.Close() + pipeWriter.Close() + tarWriter.Close() + }) + go func() { + defer cancel() + if finfo.IsDir() { + log.Printf("creating tar stream for directory %q\n", path) + if opts != nil && opts.Recursive { + log.Printf("creating tar stream for directory %q recursively\n", path) + err := tarWriter.AddFS(os.DirFS(path)) + if err != nil { + rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) + return + } + log.Printf("added directory %q to tar stream\n", path) + log.Printf("returning tar stream\n") + } else { + rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) + } + } else { + log.Printf("creating tar stream for file %q\n", path) + header, err := tar.FileInfoHeader(finfo, "") + if err != nil { + rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar header for %q: %w", path, err)) + return + } + log.Printf("created tar header for file %q\n", path) + err = tarWriter.WriteHeader(header) + if err != nil { + rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot write tar header for %q: %w", path, err)) + return + } + log.Printf("wrote tar header for file %q\n", path) + file, err := os.Open(cleanedPath) + if err != nil { + rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot open file %q: %w", path, err)) + return + } + log.Printf("opened file %q\n", path) + n, err := file.WriteTo(tarWriter) + if err != nil { + rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot write file %q to tar stream: %w", path, err)) + return + } + log.Printf("wrote %d bytes to tar stream\n", n) } - rtn := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { - pipeWriter.Close() - tarWriter.Close() - }) - return rtn - } else { - header, err := tar.FileInfoHeader(finfo, "") - if err != nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot create tar header for %q: %w", path, err)) + }() + log.Printf("returning channel\n") + return rtn +} + +func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.CommandRemoteFileCopyData) error { + opts := data.Opts + destPath := data.DestPath + srcUri := data.SrcUri + merge := opts != nil && opts.Merge + overwrite := opts != nil && opts.Overwrite + recursive := opts != nil && opts.Recursive + destPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(destPath)) + destinfo, err := os.Stat(destPathCleaned) + if err == nil { + if destinfo.IsDir() { + if !recursive { + return fmt.Errorf("destination %q is a directory, use recursive option", destPath) + } + if !merge { + if overwrite { + err := os.RemoveAll(destPathCleaned) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", destPath, err) + } + } else { + return fmt.Errorf("destination %q is a directory, use overwrite option", destPath) + } + } + } else { + if !overwrite { + return fmt.Errorf("destination %q already exists, use overwrite option", destPath) + } else { + err := os.Remove(destPathCleaned) + if err != nil { + return fmt.Errorf("cannot remove file %q: %w", destPath, err) + } + } } - err = tarWriter.WriteHeader(header) - if err != nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot write tar header for %q: %w", path, err)) + } + ioch := wshclient.FileStreamTarCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}, &wshrpc.RpcOpts{}) + pipeReader, pipeWriter := io.Pipe() + tarReader := tar.NewReader(pipeReader) + ctx, cancel := context.WithCancel(ctx) + iochan.WriterChan(ctx, pipeWriter, ioch) + defer pipeWriter.Close() + defer pipeReader.Close() + defer cancel() + for next, err := tarReader.Next(); err == nil; { + finfo := next.FileInfo() + nextPath := filepath.Clean(filepath.Join(destPathCleaned, next.Name)) + destinfo, err = os.Stat(nextPath) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("cannot stat file %q: %w", nextPath, err) } - file, err := os.Open(cleanedPath) - if err != nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot open file %q: %w", path, err)) + + if destinfo != nil { + if destinfo.IsDir() { + if !finfo.IsDir() { + if !overwrite { + return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) + } else { + err := os.Remove(nextPath) + if err != nil { + return fmt.Errorf("cannot remove file %q: %w", nextPath, err) + } + } + } else if !merge && !overwrite { + return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) + } else if overwrite { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + } + } + } else { + if finfo.IsDir() { + if !overwrite { + return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) + } else { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + } + } + } else if !overwrite { + return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) + } else { + err := os.Remove(nextPath) + if err != nil { + return fmt.Errorf("cannot remove file %q: %w", nextPath, err) + } + } + } + } else { + if finfo.IsDir() { + err := os.MkdirAll(nextPath, finfo.Mode()) + if err != nil { + return fmt.Errorf("cannot create directory %q: %w", nextPath, err) + } + } else { + file, err := os.Create(nextPath) + if err != nil { + return fmt.Errorf("cannot create new file %q: %w", nextPath, err) + } + _, err = io.Copy(file, tarReader) + if err != nil { + return fmt.Errorf("cannot write file %q: %w", nextPath, err) + } + file.Chmod(finfo.Mode()) + file.Close() + } } - file.WriteTo(tarWriter) - rtn := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { - file.Close() - pipeWriter.Close() - tarWriter.Close() - }) - return rtn } + if err != nil && err != io.EOF { + return fmt.Errorf("cannot read tar stream: %w", err) + } + return nil } func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrpc.CommandRemoteListEntriesData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 36435b78c..aa8aab23f 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -150,7 +150,8 @@ type WshRpcInterface interface { FileAppendIJsonCommand(ctx context.Context, data CommandAppendIJsonData) error FileWriteCommand(ctx context.Context, data FileData) error FileReadCommand(ctx context.Context, data FileData) (*FileData, error) - FileStreamTarCommand(ctx context.Context, data FileData) <-chan RespOrErrorUnion[[]byte] + FileStreamTarCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[[]byte] + FileCopyCommand(ctx context.Context, data CommandFileCopyData) error FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) FileListCommand(ctx context.Context, data FileListData) ([]*FileInfo, error) FileListStreamCommand(ctx context.Context, data FileListData) <-chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] @@ -192,7 +193,8 @@ type WshRpcInterface interface { // remotes RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[FileData] - RemoteTarStreamCommand(ctx context.Context, path string) <-chan RespOrErrorUnion[[]byte] + RemoteTarStreamCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[[]byte] + RemoteFileCopyCommand(ctx context.Context, data CommandRemoteFileCopyData) error RemoteListEntriesCommand(ctx context.Context, data CommandRemoteListEntriesData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) RemoteFileTouchCommand(ctx context.Context, path string) error @@ -489,9 +491,27 @@ type CpuDataType struct { Value float64 `json:"value"` } -type FileRemoteStreamOpts struct { - UseTar bool `json:"usetar,omitempty"` - DeleteAfter bool `json:"deleteafter,omitempty"` +type CommandFileCopyData struct { + SrcUri string `json:"srcuri"` + DestUri string `json:"desturi"` + Opts *FileCopyOpts `json:"opts,omitempty"` +} + +type CommandRemoteFileCopyData struct { + SrcUri string `json:"srcuri"` + DestPath string `json:"destpath"` + Opts *FileCopyOpts `json:"opts,omitempty"` +} + +type CommandRemoteStreamTarData struct { + Path string `json:"path"` + Opts *FileCopyOpts `json:"opts,omitempty"` +} + +type FileCopyOpts struct { + Overwrite bool `json:"overwrite,omitempty"` + Recursive bool `json:"recursive,omitempty"` + Merge bool `json:"merge,omitempty"` } type CommandRemoteStreamFileData struct { diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index f5f3ad2c3..cce72bc91 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -319,7 +319,11 @@ func (ws *WshServer) FileReadCommand(ctx context.Context, data wshrpc.FileData) return fileshare.Read(ctx, data) } -func (ws *WshServer) FileStreamTarCommand(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (ws *WshServer) FileCopyCommand(ctx context.Context, data wshrpc.CommandFileCopyData) error { + return fileshare.Copy(ctx, data) +} + +func (ws *WshServer) FileStreamTarCommand(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[[]byte] { return fileshare.ReadTarStream(ctx, data) } From d992edcef18ee9675f95b57b2aa849eef7c299c9 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Thu, 16 Jan 2025 17:47:40 -0800 Subject: [PATCH 052/126] save work --- cmd/wsh/cmd/wshcmd-file-util.go | 10 +++++++-- pkg/remote/connparse/connparse.go | 33 ++++++++++++++++++++++++++--- pkg/remote/fileshare/fileshare.go | 25 ++-------------------- pkg/remote/fileshare/wshfs/wshfs.go | 2 +- pkg/util/fileutil/fileutil.go | 3 +++ pkg/util/iochan/iochan.go | 3 ++- pkg/wshrpc/wshremote/wshremote.go | 33 +++++++++++++++++++++++------ 7 files changed, 72 insertions(+), 37 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 547b8d584..499772bf2 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "io/fs" + "log" "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" @@ -210,11 +211,13 @@ func streamFileList(zoneId string, path string, recursive bool, filesOnly bool) } func fixRelativePaths(path string) (string, error) { + log.Printf("Fixing relative paths: %s", path) conn, err := connparse.ParseURI(path) if err != nil { return "", err } if conn.Scheme == connparse.ConnectionTypeWsh { + log.Printf("wsh conn type %s; %s", conn.Host, conn.Path) fixedPath := conn.Path if conn.Host == connparse.ConnHostCurrent { conn.Host = RpcContext.Conn @@ -222,8 +225,11 @@ func fixRelativePaths(path string) (string, error) { if err != nil { return "", err } + conn.Path = fixedPath + } + if conn.Host == "" { + conn.Host = wshrpc.LocalConnName } - return fmt.Sprintf("wsh://%s/%s", conn.Host, fixedPath), nil } - return path, nil + return conn.GetFullURI(), nil } diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index a323adc02..e2fa6f9fd 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -4,10 +4,13 @@ package connparse import ( + "context" + "fmt" "log" "strings" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshutil" ) const ( @@ -51,6 +54,27 @@ func (c *Connection) GetFullURI() string { return c.Scheme + "://" + c.GetPathWithHost() } +func ParseURIAndReplaceCurrentHost(ctx context.Context, uri string) (*Connection, error) { + conn, err := ParseURI(uri) + if err != nil { + return nil, fmt.Errorf("error parsing connection: %v", err) + } + if conn.Host == ConnHostCurrent { + handler := wshutil.GetRpcResponseHandlerFromContext(context.Background()) + if handler == nil { + return nil, fmt.Errorf("error getting rpc response handler from context") + } + source := handler.GetRpcContext().Conn + + // RPC context connection is empty for local connections + if source == "" { + source = wshrpc.LocalConnName + } + conn.Host = source + } + return conn, nil +} + // ParseURI parses a connection URI and returns the connection type, host/path, and parameters. func ParseURI(uri string) (*Connection, error) { split := strings.SplitN(uri, "://", 2) @@ -81,6 +105,7 @@ func ParseURI(uri string) (*Connection, error) { host = wshrpc.LocalConnName remotePath = rest } else { + log.Printf("Fixing relative path to %s; %s", ConnHostCurrent, rest) host = ConnHostCurrent remotePath = rest } @@ -100,14 +125,16 @@ func ParseURI(uri string) (*Connection, error) { host = wshrpc.LocalConnName } if strings.HasPrefix(remotePath, "/~") { - log.Printf("Fixing relative path to %s; %s", host, remotePath) remotePath = strings.TrimPrefix(remotePath, "/") + log.Printf("Fixing relative path to %s; %s", host, remotePath) } } - return &Connection{ + conn := &Connection{ Scheme: scheme, Host: host, Path: remotePath, - }, nil + } + log.Printf("Parsed connection: %v", conn) + return conn, nil } diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 8afe012b6..a701e247d 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -19,7 +19,7 @@ import ( // Returns the client and the parsed connection func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileShareClient, *connparse.Connection) { log.Printf("CreateFileShareClient: connection=%s", connection) - conn, err := parseConnUriAndReplaceCurrentHost(connection) + conn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, connection) if err != nil { log.Printf("error parsing connection: %v", err) return nil, nil @@ -131,7 +131,7 @@ func Move(ctx context.Context, srcPath, destPath string, recursive bool) error { func Copy(ctx context.Context, data wshrpc.CommandFileCopyData) error { log.Printf("Copy: src=%s, dest=%s", data.SrcUri, data.DestUri) - srcConn, err := parseConnUriAndReplaceCurrentHost(data.SrcUri) + srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, data.SrcUri) if err != nil { return fmt.Errorf("error parsing source connection %s: %v", data.SrcUri, err) } @@ -159,24 +159,3 @@ func Join(ctx context.Context, path string, parts ...string) (string, error) { } return client.Join(ctx, conn, parts...) } - -func parseConnUriAndReplaceCurrentHost(uri string) (*connparse.Connection, error) { - conn, err := connparse.ParseURI(uri) - if err != nil { - return nil, fmt.Errorf("error parsing connection: %v", err) - } - if conn.Host == connparse.ConnHostCurrent { - handler := wshutil.GetRpcResponseHandlerFromContext(context.Background()) - if handler == nil { - return nil, fmt.Errorf("error getting rpc response handler from context") - } - source := handler.GetRpcContext().Conn - - // RPC context connection is empty for local connections - if source == "" { - source = wshrpc.LocalConnName - } - conn.Host = source - } - return conn, nil -} diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 1c6cb1e10..0c7440c97 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -122,7 +122,7 @@ func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connec } func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return wshclient.RemoteFileCopyCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestPath: destConn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) + return wshclient.RemoteFileCopyCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestPath: destConn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host)}) } func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index d62bf1ef9..a30b485e6 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -3,6 +3,7 @@ package fileutil import ( "io" "io/fs" + "log" "mime" "net/http" "os" @@ -73,10 +74,12 @@ func FixPath(path string) (string, error) { if strings.HasPrefix(path, "~") { return filepath.Join(wavebase.GetHomeDir(), path[1:]), nil } else if !filepath.IsAbs(path) { + log.Printf("FixPath: path is not absolute: %s", path) path, err := filepath.Abs(path) if err != nil { return "", err } + log.Printf("FixPath: fixed path: %s", path) return path, nil } else { return path, nil diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index b6db354e4..4bb9f26c8 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -6,6 +6,7 @@ package iochan import ( "context" + "fmt" "io" "log" @@ -26,7 +27,7 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } n, err := r.Read(buf) if err != nil && err != io.EOF { - ch <- wshrpc.RespOrErrorUnion[[]byte]{Error: err} + ch <- wshrpc.RespOrErrorUnion[[]byte]{Error: fmt.Errorf("ReaderChan: read error: %v", err)} callback() return } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index e5e0b2de0..b85f9908f 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -15,16 +15,21 @@ import ( "os" "path/filepath" "strings" + "time" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" - "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshutil" ) +const ( + OneYear = int(time.Hour * 24 * 365) +) + type ServerImpl struct { LogWriter io.Writer } @@ -277,6 +282,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.CommandRemoteFileCopyData) error { + log.Printf("RemoteFileCopyCommand: src=%s, dest=%s\n", data.SrcUri, data.DestPath) opts := data.Opts destPath := data.DestPath srcUri := data.SrcUri @@ -310,22 +316,35 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } } - } - ioch := wshclient.FileStreamTarCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}, &wshrpc.RpcOpts{}) + } else if !os.IsNotExist(err) { + return fmt.Errorf("cannot stat destination %q: %w", destPath, err) + } + log.Printf("copying %q to %q\n", srcUri, destPath) + // conn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) + // if err != nil { + // return fmt.Errorf("cannot parse source URI %q: %w", srcUri, err) + // } + // rpcOpts := &wshrpc.RpcOpts{Timeout: OneYear, Route: wshutil.MakeConnectionRouteId(conn.Host)} + // ioch := wshclient.FileStreamTarCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, rpcOpts) + ioch := fileshare.ReadTarStream(ctx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() tarReader := tar.NewReader(pipeReader) ctx, cancel := context.WithCancel(ctx) iochan.WriterChan(ctx, pipeWriter, ioch) - defer pipeWriter.Close() - defer pipeReader.Close() - defer cancel() + defer func() { + pipeWriter.Close() + pipeReader.Close() + cancel() + // rpcOpts.StreamCancelFn() + }() for next, err := tarReader.Next(); err == nil; { finfo := next.FileInfo() - nextPath := filepath.Clean(filepath.Join(destPathCleaned, next.Name)) + nextPath := filepath.Join(destPathCleaned, next.Name) destinfo, err = os.Stat(nextPath) if err != nil && !os.IsNotExist(err) { return fmt.Errorf("cannot stat file %q: %w", nextPath, err) } + log.Printf("copying %q to %q\n", next.Name, nextPath) if destinfo != nil { if destinfo.IsDir() { From 09f4239940d733418377a924fe2dbfe22ad3a90c Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 17 Jan 2025 11:55:55 -0800 Subject: [PATCH 053/126] fix merge error --- pkg/wshrpc/wshrpctypes.go | 31 ------------------------------- pkg/wslconn/wslconn.go | 4 ++-- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index aa8aab23f..d1ba0c845 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -534,37 +534,6 @@ type CommandRemoteWriteFileData struct { CreateMode os.FileMode `json:"createmode,omitempty"` } -type ConnKeywords struct { - ConnWshEnabled *bool `json:"conn:wshenabled,omitempty"` - ConnAskBeforeWshInstall *bool `json:"conn:askbeforewshinstall,omitempty"` - ConnOverrideConfig bool `json:"conn:overrideconfig,omitempty"` - ConnWshPath string `json:"conn:wshpath,omitempty"` - - DisplayHidden *bool `json:"display:hidden,omitempty"` - DisplayOrder float32 `json:"display:order,omitempty"` - - TermClear bool `json:"term:*,omitempty"` - TermFontSize float64 `json:"term:fontsize,omitempty"` - TermFontFamily string `json:"term:fontfamily,omitempty"` - TermTheme string `json:"term:theme,omitempty"` - - SshUser *string `json:"ssh:user,omitempty"` - SshHostName *string `json:"ssh:hostname,omitempty"` - SshPort *string `json:"ssh:port,omitempty"` - SshIdentityFile []string `json:"ssh:identityfile,omitempty"` - SshIdentitiesOnly *bool `json:"ssh:identitiesonly,omitempty"` - SshBatchMode *bool `json:"ssh:batchmode,omitempty"` - SshPubkeyAuthentication *bool `json:"ssh:pubkeyauthentication,omitempty"` - SshPasswordAuthentication *bool `json:"ssh:passwordauthentication,omitempty"` - SshKbdInteractiveAuthentication *bool `json:"ssh:kbdinteractiveauthentication,omitempty"` - SshPreferredAuthentications []string `json:"ssh:preferredauthentications,omitempty"` - SshAddKeysToAgent *bool `json:"ssh:addkeystoagent,omitempty"` - SshIdentityAgent *string `json:"ssh:identityagent,omitempty"` - SshProxyJump []string `json:"ssh:proxyjump,omitempty"` - SshUserKnownHostsFile []string `json:"ssh:userknownhostsfile,omitempty"` - SshGlobalKnownHostsFile []string `json:"ssh:globalknownhostsfile,omitempty"` -} - type ConnRequest struct { Host string `json:"host"` Keywords wconfig.ConnKeywords `json:"keywords,omitempty"` diff --git a/pkg/wslconn/wslconn.go b/pkg/wslconn/wslconn.go index 1f561c826..22d2fb368 100644 --- a/pkg/wslconn/wslconn.go +++ b/pkg/wslconn/wslconn.go @@ -638,11 +638,11 @@ func (conn *WslConn) tryEnableWsh(ctx context.Context, clientDisplayName string) } } -func (conn *WslConn) getConnectionConfig() (wshrpc.ConnKeywords, bool) { +func (conn *WslConn) getConnectionConfig() (wconfig.ConnKeywords, bool) { config := wconfig.GetWatcher().GetFullConfig() connSettings, ok := config.Connections[conn.GetName()] if !ok { - return wshrpc.ConnKeywords{}, false + return wconfig.ConnKeywords{}, false } return connSettings, true } From d7e65a4d9676058ca90a9ded5a3e7b74d1454673 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 17 Jan 2025 14:16:37 -0800 Subject: [PATCH 054/126] save --- pkg/wshutil/wshrouter.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/wshutil/wshrouter.go b/pkg/wshutil/wshrouter.go index 2a99b19ce..1764fba50 100644 --- a/pkg/wshutil/wshrouter.go +++ b/pkg/wshutil/wshrouter.go @@ -196,20 +196,26 @@ func (router *WshRouter) getAnnouncedRoute(routeId string) string { func (router *WshRouter) sendRoutedMessage(msgBytes []byte, routeId string) bool { rpc := router.GetRpc(routeId) if rpc != nil { + log.Printf("[router] sending message to %q via rpc\n", routeId) rpc.SendRpcMessage(msgBytes) return true } upstream := router.GetUpstreamClient() if upstream != nil { + log.Printf("[router] sending message to %q via upstream\n", routeId) upstream.SendRpcMessage(msgBytes) return true } else { + log.Printf("[router] sending message to %q via announced route\n", routeId) // we are the upstream, so consult our announced routes map localRouteId := router.getAnnouncedRoute(routeId) + log.Printf("[router] local route id: %q\n", localRouteId) rpc := router.GetRpc(localRouteId) if rpc == nil { + log.Printf("[router] no rpc for local route id %q\n", localRouteId) return false } + log.Printf("[router] sending message to %q via local route\n", localRouteId) rpc.SendRpcMessage(msgBytes) return true } From 91c652583aa62a4219f9b189211b57b2119b19da Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 17 Jan 2025 14:30:09 -0800 Subject: [PATCH 055/126] add listbuckets --- go.mod | 30 +++++++------- go.sum | 56 +++++++++++++------------- pkg/remote/fileshare/s3fs/s3fs.go | 57 ++++++++++++++++++++++++++- pkg/remote/fileshare/wavefs/wavefs.go | 7 ++-- 4 files changed, 102 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 377eb1396..844234351 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.23.4 require ( github.com/alexflint/go-filemutex v1.3.0 - github.com/aws/aws-sdk-go-v2 v1.32.7 - github.com/aws/aws-sdk-go-v2/config v1.28.7 - github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 + github.com/aws/aws-sdk-go-v2 v1.33.0 + github.com/aws/aws-sdk-go-v2/config v1.29.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2 + github.com/aws/smithy-go v1.22.1 github.com/creack/pty v1.1.21 github.com/fsnotify/fsnotify v1.8.0 github.com/golang-jwt/jwt/v5 v5.2.1 @@ -44,20 +45,19 @@ require ( cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/longrunning v0.5.7 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.48 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.54 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 // indirect - github.com/aws/smithy-go v1.22.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect github.com/ebitengine/purego v0.8.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect diff --git a/go.sum b/go.sum index 5790b2fc0..649363d78 100644 --- a/go.sum +++ b/go.sum @@ -16,40 +16,40 @@ github.com/0xrawsec/golang-utils v1.3.2 h1:ww4jrtHRSnX9xrGzJYbalx5nXoZewy4zPxiY+ github.com/0xrawsec/golang-utils v1.3.2/go.mod h1:m7AzHXgdSAkFCD9tWWsApxNVxMlyy7anpPVOyT/yM7E= github.com/alexflint/go-filemutex v1.3.0 h1:LgE+nTUWnQCyRKbpoceKZsPQbs84LivvgwUymZXdOcM= github.com/alexflint/go-filemutex v1.3.0/go.mod h1:U0+VA/i30mGBlLCrFPGtTe9y6wGQfNAWPBTekHQ+c8A= -github.com/aws/aws-sdk-go-v2 v1.32.7 h1:ky5o35oENWi0JYWUZkB7WYvVPP+bcRF5/Iq7JWSb5Rw= -github.com/aws/aws-sdk-go-v2 v1.32.7/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs= +github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= -github.com/aws/aws-sdk-go-v2/config v1.28.7 h1:GduUnoTXlhkgnxTD93g1nv4tVPILbdNQOzav+Wpg7AE= -github.com/aws/aws-sdk-go-v2/config v1.28.7/go.mod h1:vZGX6GVkIE8uECSUHB6MWAUsd4ZcG2Yq/dMa4refR3M= -github.com/aws/aws-sdk-go-v2/credentials v1.17.48 h1:IYdLD1qTJ0zanRavulofmqut4afs45mOWEI+MzZtTfQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.48/go.mod h1:tOscxHN3CGmuX9idQ3+qbkzrjVIx32lqDSU1/0d/qXs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22 h1:kqOrpojG71DxJm/KDPO+Z/y1phm1JlC8/iT+5XRmAn8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.22/go.mod h1:NtSFajXVVL8TA2QNngagVZmUtXciyrHOt7xgz4faS/M= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26 h1:I/5wmGMffY4happ8NOCuIUEWGUvvFp5NSeQcXl9RHcI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.26/go.mod h1:FR8f4turZtNy6baO0KJ5FJUmXH/cSkI9fOngs0yl6mA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26 h1:zXFLuEuMMUOvEARXFUVJdfqZ4bvvSgdGRq/ATcrQxzM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.26/go.mod h1:3o2Wpy0bogG1kyOPrgkXA8pgIfEEv0+m19O9D5+W8y8= +github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= +github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26 h1:GeNJsIFHB+WW5ap2Tec4K6dzcVTsRbsT1Lra46Hv9ME= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.26/go.mod h1:zfgMpwHDXX2WGoG84xG2H+ZlPTkJUU4YUvx2svLQYWo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28 h1:7kpeALOUeThs2kEjlAxlADAVfxKmkYAedlpZ3kdoSJ4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28/go.mod h1:pyaOYEdp1MJWgtXLy6q80r3DhsVdOIOZNB9hdTcJIvI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7 h1:tB4tNw83KcajNAzaIMhkhVI2Nt8fAZd5A5ro113FEMY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.7/go.mod h1:lvpyBGkZ3tZ9iSsUIcC2EWp+0ywa7aK3BLT+FwZi+mQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7 h1:8eUsivBQzZHqe/3FE+cqwfH+0p5Jo8PFM/QYQSmeZ+M= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.7/go.mod h1:kLPQvGUmxn/fqiCrDeohwG33bq2pQpGeY62yRO6Nrh0= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7 h1:Hi0KGbrnr57bEHWM0bJ1QcBzxLrL/k2DHvGYhb8+W1w= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.7/go.mod h1:wKNgWgExdjjrm4qvfbTorkvocEstaoDl4WCvGfeCy9c= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0 h1:SAfh4pNx5LuTafKKWR02Y+hL3A+3TX8cTKG1OIAJaBk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.0/go.mod h1:r+xl5yzMk9083rMR+sJ5TYj9Tihvf/l1oxzZXDgGj2Q= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.8 h1:CvuUmnXI7ebaUAhbJcDy9YQx8wHR69eZ9I7q5hszt/g= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.8/go.mod h1:XDeGv1opzwm8ubxddF0cgqkZWsyOtw4lr6dxwmb6YQg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7 h1:F2rBfNAL5UyswqoeWv9zs74N/NanhK16ydHW1pahX6E= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.7/go.mod h1:JfyQ0g2JG8+Krq0EuZNnRwX0mU0HrwY/tG6JNfcqh4k= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.3 h1:Xgv/hyNgvLda/M9l9qxXc4UFSgppnRczLxlMs5Ae/QY= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.3/go.mod h1:5Gn+d+VaaRgsjewpMvGazt0WfcFO+Md4wLOuBfGR9Bc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2 h1:e6um6+DWYQP1XCa+E9YVtG/9v1qk5lyAOelMOVwSyO8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2/go.mod h1:dIW8puxSbYLSPv/ju0d9A3CpwXdtqvJtYKDMVmPLOWE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9 h1:2aInXbh02XsbO0KobPGMNXyv2QP73VDKsWPNJARj/+4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9/go.mod h1:dgXS1i+HgWnYkPXqNoPIPKeUsUUYHaUbThC90aDnNiE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2 h1:F3h8VYq9ZLBXYurmwrT8W0SPhgCcU0q+0WZJfT1dFt0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2/go.mod h1:jGJ/v7FIi7Ys9t54tmEFnrxuaWeJLpwNgKp2DXAVhOU= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index ae3552029..2f1d52c67 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -5,12 +5,17 @@ package s3fs import ( "context" + "errors" + "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshutil" ) type S3Client struct { @@ -37,12 +42,60 @@ func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, return nil } +func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { + ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData], 16) + go func() { + defer close(ch) + list, err := c.ListEntries(ctx, conn, opts) + if err != nil { + ch <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) + return + } + for i := 0; i < len(list); i += wshrpc.DirChunkSize { + ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: list[i:min(i+wshrpc.DirChunkSize, len(list))]}} + } + }() + return ch +} + func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { + if conn.Path == "" || conn.Path == "/" { + buckets, err := c.listBuckets(ctx) + if err != nil { + return nil, err + } + var entries []*wshrpc.FileInfo + for _, bucket := range buckets { + entries = append(entries, &wshrpc.FileInfo{ + Path: *bucket.Name, + IsDir: true, + }) + } + return entries, nil + } return nil, nil } -func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { - return nil +func (c S3Client) listBuckets(ctx context.Context) ([]types.Bucket, error) { + var err error + var output *s3.ListBucketsOutput + var buckets []types.Bucket + bucketPaginator := s3.NewListBucketsPaginator(c.client, &s3.ListBucketsInput{}) + for bucketPaginator.HasMorePages() { + output, err = bucketPaginator.NextPage(ctx) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { + fmt.Println("You don't have permission to list buckets for this account.") + err = apiErr + } else { + return nil, fmt.Errorf("Couldn't list buckets for your account. Here's why: %v\n", err) + } + break + } + buckets = append(buckets, output.Buckets...) + } + return buckets, nil } func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 84c23f26b..a8c7893f4 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -18,6 +18,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshutil" ) type WaveClient struct{} @@ -34,12 +35,12 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, defer close(ch) rtnData, err := c.Read(ctx, conn, data) if err != nil { - ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Error: err} + ch <- wshutil.RespErr[wshrpc.FileData](err) return } for { if ctx.Err() != nil { - ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Error: ctx.Err()} + ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) } dataLen := len(rtnData.Data64) if !rtnData.Info.IsDir { @@ -100,7 +101,7 @@ func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Conne defer close(ch) list, err := c.ListEntries(ctx, conn, opts) if err != nil { - ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Error: err} + ch <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) return } for i := 0; i < len(list); i += wshrpc.DirChunkSize { From 280e454a6b16e0ecbb9e5ef504d8478792a47085 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 17 Jan 2025 14:34:01 -0800 Subject: [PATCH 056/126] move listbuckets to awsconn --- pkg/remote/awsconn/awsconn.go | 26 ++++++++++++++++++++++++++ pkg/remote/fileshare/s3fs/s3fs.go | 29 ++--------------------------- pkg/wshutil/wshrouter.go | 2 +- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index f1e756945..357258cfd 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -3,6 +3,7 @@ package awsconn import ( "context" + "errors" "fmt" "log" "os" @@ -11,6 +12,9 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/wavetermdev/waveterm/pkg/wconfig" "gopkg.in/ini.v1" ) @@ -93,3 +97,25 @@ func ParseProfiles() map[string]struct{} { } return profiles } + +func ListBuckets(ctx context.Context, client *s3.Client) ([]types.Bucket, error) { + var err error + var output *s3.ListBucketsOutput + var buckets []types.Bucket + bucketPaginator := s3.NewListBucketsPaginator(client, &s3.ListBucketsInput{}) + for bucketPaginator.HasMorePages() { + output, err = bucketPaginator.NextPage(ctx) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { + fmt.Println("You don't have permission to list buckets for this account.") + err = apiErr + } else { + return nil, fmt.Errorf("Couldn't list buckets for your account. Here's why: %v\n", err) + } + break + } + buckets = append(buckets, output.Buckets...) + } + return buckets, nil +} diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 2f1d52c67..d32ba9ef4 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -5,13 +5,10 @@ package s3fs import ( "context" - "errors" - "fmt" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" + "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -60,7 +57,7 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { if conn.Path == "" || conn.Path == "/" { - buckets, err := c.listBuckets(ctx) + buckets, err := awsconn.ListBuckets(ctx, c.client) if err != nil { return nil, err } @@ -76,28 +73,6 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o return nil, nil } -func (c S3Client) listBuckets(ctx context.Context) ([]types.Bucket, error) { - var err error - var output *s3.ListBucketsOutput - var buckets []types.Bucket - bucketPaginator := s3.NewListBucketsPaginator(c.client, &s3.ListBucketsInput{}) - for bucketPaginator.HasMorePages() { - output, err = bucketPaginator.NextPage(ctx) - if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { - fmt.Println("You don't have permission to list buckets for this account.") - err = apiErr - } else { - return nil, fmt.Errorf("Couldn't list buckets for your account. Here's why: %v\n", err) - } - break - } - buckets = append(buckets, output.Buckets...) - } - return buckets, nil -} - func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { return nil, nil } diff --git a/pkg/wshutil/wshrouter.go b/pkg/wshutil/wshrouter.go index 1764fba50..0514acb61 100644 --- a/pkg/wshutil/wshrouter.go +++ b/pkg/wshutil/wshrouter.go @@ -196,7 +196,7 @@ func (router *WshRouter) getAnnouncedRoute(routeId string) string { func (router *WshRouter) sendRoutedMessage(msgBytes []byte, routeId string) bool { rpc := router.GetRpc(routeId) if rpc != nil { - log.Printf("[router] sending message to %q via rpc\n", routeId) + // log.Printf("[router] sending message to %q via rpc\n", routeId) rpc.SendRpcMessage(msgBytes) return true } From bc7616401534a31806ffd11048a15204ef736de5 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 17 Jan 2025 15:19:45 -0800 Subject: [PATCH 057/126] save --- pkg/remote/awsconn/awsconn.go | 59 +++++++++++++++++++++---------- pkg/remote/fileshare/fileshare.go | 25 +++++++------ pkg/remote/fileshare/s3fs/s3fs.go | 6 ++++ 3 files changed, 61 insertions(+), 29 deletions(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index 357258cfd..5e8396dd8 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -15,17 +15,19 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" + "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wconfig" "gopkg.in/ini.v1" ) const ( - ProfileConfigKey = "profile:config" - ProfilePrefix = "aws:" - TempFilePattern = "waveterm-awsconfig-%s" + ProfileConfigKey = "profile:config" + ProfileCredentialsKey = "profile:credentials" + ProfilePrefix = "aws:" + TempFilePattern = "waveterm-awsconfig-%s" ) -var connectionRe = regexp.MustCompile(`^aws:\/\/(.*)$`) +var connectionRe = regexp.MustCompile(`^(.*):\w+:\/\/.*$`) var tempfiles map[string]string = make(map[string]string) @@ -38,25 +40,23 @@ func GetConfig(ctx context.Context, profile string) (*aws.Config, error) { return nil, fmt.Errorf("invalid connection string: %s)", profile) } profile = connMatch[1] + log.Printf("GetConfig: profile=%s", profile) profiles, cerrs := wconfig.ReadWaveHomeConfigFile(wconfig.ProfilesFile) if len(cerrs) > 0 { return nil, fmt.Errorf("error reading config file: %v", cerrs[0]) } if profiles[profile] != nil { - connectionconfig := profiles.GetMap(profile) - if connectionconfig[ProfileConfigKey] != "" { - var tempfile string - if tempfiles[profile] != "" { - tempfile = tempfiles[profile] - } else { - awsConfig := connectionconfig.GetString(ProfileConfigKey, "") - tempfile, err := os.CreateTemp("", fmt.Sprintf(TempFilePattern, profile)) - if err != nil { - return nil, fmt.Errorf("error creating temp file: %v", err) - } - tempfile.WriteString(awsConfig) - } - optfns = append(optfns, config.WithSharedCredentialsFiles([]string{tempfile})) + configfilepath, _ := getTempFileFromConfig(profiles, ProfileConfigKey, profile) + credentialsfilepath, _ := getTempFileFromConfig(profiles, ProfileCredentialsKey, profile) + if configfilepath != "" { + log.Printf("configfilepath: %s", configfilepath) + optfns = append(optfns, config.WithSharedConfigFiles([]string{configfilepath})) + tempfiles[profile] = configfilepath + } + if credentialsfilepath != "" { + log.Printf("credentialsfilepath: %s", credentialsfilepath) + optfns = append(optfns, config.WithSharedCredentialsFiles([]string{credentialsfilepath})) + tempfiles[profile] = credentialsfilepath } } trimmedProfile := strings.TrimPrefix(profile, ProfilePrefix) @@ -69,6 +69,22 @@ func GetConfig(ctx context.Context, profile string) (*aws.Config, error) { return &cfg, nil } +func getTempFileFromConfig(config waveobj.MetaMapType, key string, profile string) (string, error) { + connectionconfig := config.GetMap(profile) + if connectionconfig[key] != "" { + awsConfig := connectionconfig.GetString(key, "") + if awsConfig != "" { + tempfile, err := os.CreateTemp("", fmt.Sprintf(TempFilePattern, profile)) + if err != nil { + return "", fmt.Errorf("error creating temp file: %v", err) + } + tempfile.WriteString(awsConfig) + return tempfile.Name(), nil + } + } + return "", nil +} + func ParseProfiles() map[string]struct{} { profiles := make(map[string]struct{}) fname := config.DefaultSharedConfigFilename() // Get aws.config default shared configuration file name @@ -102,9 +118,11 @@ func ListBuckets(ctx context.Context, client *s3.Client) ([]types.Bucket, error) var err error var output *s3.ListBucketsOutput var buckets []types.Bucket - bucketPaginator := s3.NewListBucketsPaginator(client, &s3.ListBucketsInput{}) + region := client.Options().Region + bucketPaginator := s3.NewListBucketsPaginator(client, &s3.ListBucketsInput{BucketRegion: ®ion}) for bucketPaginator.HasMorePages() { output, err = bucketPaginator.NextPage(ctx) + log.Printf("output: %v", output) if err != nil { var apiErr smithy.APIError if errors.As(err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { @@ -115,6 +133,9 @@ func ListBuckets(ctx context.Context, client *s3.Client) ([]types.Bucket, error) } break } + if output == nil { + break + } buckets = append(buckets, output.Buckets...) } return buckets, nil diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index a701e247d..d90ff428c 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -15,6 +15,10 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) +const ( + ErrorParsingConnection = "error creating fileshare client, could not parse connection %s" +) + // CreateFileShareClient creates a fileshare client based on the connection string // Returns the client and the parsed connection func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileShareClient, *connparse.Connection) { @@ -25,6 +29,7 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS return nil, nil } conntype := conn.GetType() + log.Printf("CreateFileShareClient: conntype=%s", conntype) if conntype == connparse.ConnectionTypeS3 { config, err := awsconn.GetConfig(ctx, connection) if err != nil { @@ -43,7 +48,7 @@ func Read(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileData, error) { log.Printf("Read: path=%s", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { - return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path) + return nil, fmt.Errorf(ErrorParsingConnection, data.Info.Path) } return client.Read(ctx, conn, data) } @@ -52,7 +57,7 @@ func ReadStream(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrE log.Printf("ReadStream: path=%s", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { - return wshutil.SendErrCh[wshrpc.FileData](fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path)) + return wshutil.SendErrCh[wshrpc.FileData](fmt.Errorf(ErrorParsingConnection, data.Info.Path)) } return client.ReadStream(ctx, conn, data) } @@ -61,7 +66,7 @@ func ReadTarStream(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) log.Printf("ReadTarStream: path=%s", data.Path) client, conn := CreateFileShareClient(ctx, data.Path) if conn == nil || client == nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Path)) + return wshutil.SendErrCh[[]byte](fmt.Errorf(ErrorParsingConnection, data.Path)) } return client.ReadTarStream(ctx, conn, data.Opts) } @@ -70,7 +75,7 @@ func ListEntries(ctx context.Context, path string, opts *wshrpc.FileListOpts) ([ log.Printf("ListEntries: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + return nil, fmt.Errorf(ErrorParsingConnection, path) } return client.ListEntries(ctx, conn, opts) } @@ -79,7 +84,7 @@ func ListEntriesStream(ctx context.Context, path string, opts *wshrpc.FileListOp log.Printf("ListEntriesStream: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf("error creating fileshare client, could not parse connection %s", path)) + return wshutil.SendErrCh[wshrpc.CommandRemoteListEntriesRtnData](fmt.Errorf(ErrorParsingConnection, path)) } return client.ListEntriesStream(ctx, conn, opts) } @@ -88,7 +93,7 @@ func Stat(ctx context.Context, path string) (*wshrpc.FileInfo, error) { log.Printf("Stat: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return nil, fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + return nil, fmt.Errorf(ErrorParsingConnection, path) } return client.Stat(ctx, conn) } @@ -97,7 +102,7 @@ func PutFile(ctx context.Context, data wshrpc.FileData) error { log.Printf("PutFile: path=%s", data.Info.Path) client, conn := CreateFileShareClient(ctx, data.Info.Path) if conn == nil || client == nil { - return fmt.Errorf("error creating fileshare client, could not parse connection %s", data.Info.Path) + return fmt.Errorf(ErrorParsingConnection, data.Info.Path) } return client.PutFile(ctx, conn, wshrpc.FileData{ Data64: data.Data64, @@ -110,7 +115,7 @@ func Mkdir(ctx context.Context, path string) error { log.Printf("Mkdir: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + return fmt.Errorf(ErrorParsingConnection, path) } return client.Mkdir(ctx, conn) } @@ -146,7 +151,7 @@ func Delete(ctx context.Context, path string) error { log.Printf("Delete: path=%s", path) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + return fmt.Errorf(ErrorParsingConnection, path) } return client.Delete(ctx, conn) } @@ -155,7 +160,7 @@ func Join(ctx context.Context, path string, parts ...string) (string, error) { log.Printf("Join: path=%s, parts=%v", path, parts) client, conn := CreateFileShareClient(ctx, path) if conn == nil || client == nil { - return "", fmt.Errorf("error creating fileshare client, could not parse connection %s", path) + return "", fmt.Errorf(ErrorParsingConnection, path) } return client.Join(ctx, conn, parts...) } diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index d32ba9ef4..a57cade8c 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -5,6 +5,7 @@ package s3fs import ( "context" + "log" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" @@ -48,6 +49,10 @@ func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connect ch <- wshutil.RespErr[wshrpc.CommandRemoteListEntriesRtnData](err) return } + if list == nil { + ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{}} + return + } for i := 0; i < len(list); i += wshrpc.DirChunkSize { ch <- wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]{Response: wshrpc.CommandRemoteListEntriesRtnData{FileInfo: list[i:min(i+wshrpc.DirChunkSize, len(list))]}} } @@ -63,6 +68,7 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o } var entries []*wshrpc.FileInfo for _, bucket := range buckets { + log.Printf("bucket: %v", *bucket.Name) entries = append(entries, &wshrpc.FileInfo{ Path: *bucket.Name, IsDir: true, From 29419faca6057cdbdd0e25f754a46bbaf44a2890 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 17 Jan 2025 16:40:01 -0800 Subject: [PATCH 058/126] it works! timing out but still --- cmd/server/main-server.go | 2 + cmd/wsh/cmd/wshcmd-connserver.go | 3 + pkg/remote/fileshare/wshfs/wshfs.go | 23 +++--- pkg/util/iochan/iochan.go | 19 +++-- pkg/wshrpc/wshremote/wshremote.go | 119 ++++++++++++---------------- 5 files changed, 83 insertions(+), 83 deletions(-) diff --git a/cmd/server/main-server.go b/cmd/server/main-server.go index d75e9c68d..06ce5639d 100644 --- a/cmd/server/main-server.go +++ b/cmd/server/main-server.go @@ -21,6 +21,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/panichandler" "github.com/wavetermdev/waveterm/pkg/remote/conncontroller" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/service" "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/util/shellutil" @@ -175,6 +176,7 @@ func shutdownActivityUpdate() { func createMainWshClient() { rpc := wshserver.GetMainRpcClient() + wshfs.RpcClient = rpc wshutil.DefaultRouter.RegisterRoute(wshutil.DefaultRoute, rpc, true) wps.Broker.SetClient(wshutil.DefaultRouter) localConnWsh := wshutil.MakeWshRpc(nil, nil, wshrpc.RpcContext{Conn: wshrpc.LocalConnName}, &wshremote.ServerImpl{}) diff --git a/cmd/wsh/cmd/wshcmd-connserver.go b/cmd/wsh/cmd/wshcmd-connserver.go index 98948ca1f..995eb0bb5 100644 --- a/cmd/wsh/cmd/wshcmd-connserver.go +++ b/cmd/wsh/cmd/wshcmd-connserver.go @@ -18,6 +18,7 @@ import ( "github.com/spf13/cobra" "github.com/wavetermdev/waveterm/pkg/panichandler" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/util/packetparser" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -180,6 +181,7 @@ func serverRunRouter(jwtToken string) error { if err != nil { return fmt.Errorf("error setting up connserver rpc client: %v", err) } + wshfs.RpcClient = client go runListener(unixListener, router) // run the sysinfo loop wshremote.RunSysInfoLoop(client, client.GetRpcContext().Conn) @@ -224,6 +226,7 @@ func serverRunNormal(jwtToken string) error { if err != nil { return err } + wshfs.RpcClient = RpcClient WriteStdout("running wsh connserver (%s)\n", RpcContext.Conn) go wshremote.RunSysInfoLoop(RpcClient, RpcContext.Conn) select {} // run forever diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 0c7440c97..97aa4cd6f 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -18,6 +18,9 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) +// This needs to be set by whoever initializes the client, either main-server or wshcmd-connserver +var RpcClient *wshutil.WshRpc + type WshClient struct{} var _ fstype.FileShareClient = WshClient{} @@ -79,12 +82,12 @@ func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, d byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+data.At.Size) } streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path, ByteRange: byteRange} - return wshclient.RemoteStreamFileCommand(wshclient.GetBareRpcClient(), streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteStreamFileCommand(RpcClient, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { log.Print("ReadTarStream wshfs") - return wshclient.RemoteTarStreamCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteTarStreamCommand(RpcClient, wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { @@ -101,36 +104,36 @@ func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, } func (c WshClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { - return wshclient.RemoteListEntriesCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteListEntriesCommand(RpcClient, wshrpc.CommandRemoteListEntriesData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { - return wshclient.RemoteFileInfoCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteFileInfoCommand(RpcClient, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { writeData := wshrpc.CommandRemoteWriteFileData{Path: conn.Path, Data64: data.Data64} - return wshclient.RemoteWriteFileCommand(wshclient.GetBareRpcClient(), writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteWriteFileCommand(RpcClient, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { - return wshclient.RemoteMkdirCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteMkdirCommand(RpcClient, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, recursive bool) error { - return wshclient.RemoteFileRenameCommand(wshclient.GetBareRpcClient(), [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) + return wshclient.RemoteFileRenameCommand(RpcClient, [2]string{srcConn.Path, destConn.Path}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(srcConn.Host)}) } func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return wshclient.RemoteFileCopyCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestPath: destConn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host)}) + return wshclient.RemoteFileCopyCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestPath: destConn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host)}) } func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { - return wshclient.RemoteFileDeleteCommand(wshclient.GetBareRpcClient(), conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteFileDeleteCommand(RpcClient, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { - finfo, err := wshclient.RemoteFileJoinCommand(wshclient.GetBareRpcClient(), append([]string{conn.Path}, parts...), &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + finfo, err := wshclient.RemoteFileJoinCommand(RpcClient, append([]string{conn.Path}, parts...), &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) if err != nil { return "", err } diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 4bb9f26c8..ec6f9f18f 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -25,13 +25,14 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func callback() return } - n, err := r.Read(buf) - if err != nil && err != io.EOF { + + if n, err := r.Read(buf); err != nil && err != io.EOF { ch <- wshrpc.RespOrErrorUnion[[]byte]{Error: fmt.Errorf("ReaderChan: read error: %v", err)} + log.Printf("ReaderChan: read error: %v", err) callback() return - } - if n > 0 { + } else if n > 0 { + log.Printf("ReaderChan: read %d bytes", n) ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} } } @@ -40,14 +41,20 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } // WriterChan reads from a channel and writes the data to an io.Writer -func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { +func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte], callback func()) { go func() { + defer callback() for resp := range ch { if resp.Error != nil { log.Printf("WriterChan: error: %v", resp.Error) return } - w.Write(resp.Response) + if n, err := w.Write(resp.Response); err != nil { + log.Printf("WriterChan: write error: %v", err) + return + } else { + log.Printf("WriterChan: wrote %d bytes", n) + } } }() } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index b85f9908f..1d87d8ed1 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -215,6 +215,7 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[[]byte] { path := data.Path opts := data.Opts + recursive := opts != nil && opts.Recursive log.Printf("RemoteTarStreamCommand: path=%s\n", path) path, err := wavebase.ExpandHomeDir(path) if err != nil { @@ -229,53 +230,50 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. tarWriter := tar.NewWriter(pipeWriter) iochanCtx, cancel := context.WithCancel(ctx) rtn := iochan.ReaderChan(iochanCtx, pipeReader, wshrpc.FileChunkSize, func() { + tarWriter.Close() pipeReader.Close() pipeWriter.Close() - tarWriter.Close() }) go func() { defer cancel() if finfo.IsDir() { log.Printf("creating tar stream for directory %q\n", path) - if opts != nil && opts.Recursive { - log.Printf("creating tar stream for directory %q recursively\n", path) - err := tarWriter.AddFS(os.DirFS(path)) - if err != nil { - rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) - return - } - log.Printf("added directory %q to tar stream\n", path) - log.Printf("returning tar stream\n") - } else { + if !recursive { rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) } - } else { - log.Printf("creating tar stream for file %q\n", path) - header, err := tar.FileInfoHeader(finfo, "") + } + log.Printf("creating tar stream for %q\n", path) + filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { + // generate tar header + header, err := tar.FileInfoHeader(fi, file) if err != nil { - rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar header for %q: %w", path, err)) - return + return err } - log.Printf("created tar header for file %q\n", path) - err = tarWriter.WriteHeader(header) - if err != nil { - rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot write tar header for %q: %w", path, err)) - return + + header.Name = strings.TrimPrefix(file, path) + if header.Name == "" { + header.Name = filepath.Base(file) } - log.Printf("wrote tar header for file %q\n", path) - file, err := os.Open(cleanedPath) - if err != nil { - rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot open file %q: %w", path, err)) - return + + // write header + if err := tarWriter.WriteHeader(header); err != nil { + return err } - log.Printf("opened file %q\n", path) - n, err := file.WriteTo(tarWriter) - if err != nil { - rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot write file %q to tar stream: %w", path, err)) - return + // if not a dir, write file content + if !fi.IsDir() { + data, err := os.Open(file) + if err != nil { + return err + } + if n, err := io.Copy(tarWriter, data); err != nil { + return err + } else { + log.Printf("wrote %d bytes to tar stream\n", n) + } } - log.Printf("wrote %d bytes to tar stream\n", n) - } + return nil + }) + log.Printf("returning tar stream\n") }() log.Printf("returning channel\n") return rtn @@ -288,25 +286,10 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C srcUri := data.SrcUri merge := opts != nil && opts.Merge overwrite := opts != nil && opts.Overwrite - recursive := opts != nil && opts.Recursive destPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(destPath)) destinfo, err := os.Stat(destPathCleaned) if err == nil { - if destinfo.IsDir() { - if !recursive { - return fmt.Errorf("destination %q is a directory, use recursive option", destPath) - } - if !merge { - if overwrite { - err := os.RemoveAll(destPathCleaned) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", destPath, err) - } - } else { - return fmt.Errorf("destination %q is a directory, use overwrite option", destPath) - } - } - } else { + if !destinfo.IsDir() { if !overwrite { return fmt.Errorf("destination %q already exists, use overwrite option", destPath) } else { @@ -316,35 +299,35 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } } - } else if !os.IsNotExist(err) { + } else if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat destination %q: %w", destPath, err) } log.Printf("copying %q to %q\n", srcUri, destPath) - // conn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) - // if err != nil { - // return fmt.Errorf("cannot parse source URI %q: %w", srcUri, err) - // } - // rpcOpts := &wshrpc.RpcOpts{Timeout: OneYear, Route: wshutil.MakeConnectionRouteId(conn.Host)} - // ioch := wshclient.FileStreamTarCommand(wshclient.GetBareRpcClient(), wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, rpcOpts) ioch := fileshare.ReadTarStream(ctx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() - tarReader := tar.NewReader(pipeReader) ctx, cancel := context.WithCancel(ctx) - iochan.WriterChan(ctx, pipeWriter, ioch) - defer func() { + iochan.WriterChan(ctx, pipeWriter, ioch, func() { + log.Printf("closing pipe writer\n") pipeWriter.Close() pipeReader.Close() cancel() - // rpcOpts.StreamCancelFn() - }() - for next, err := tarReader.Next(); err == nil; { + }) + tarReader := tar.NewReader(pipeReader) + for { + next, err := tarReader.Next() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("cannot read tar stream: %w", err) + } finfo := next.FileInfo() nextPath := filepath.Join(destPathCleaned, next.Name) destinfo, err = os.Stat(nextPath) - if err != nil && !os.IsNotExist(err) { + if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat file %q: %w", nextPath, err) } - log.Printf("copying %q to %q\n", next.Name, nextPath) + log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) if destinfo != nil { if destinfo.IsDir() { @@ -391,6 +374,10 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot create directory %q: %w", nextPath, err) } } else { + err := os.MkdirAll(filepath.Dir(nextPath), 0755) + if err != nil { + return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) + } file, err := os.Create(nextPath) if err != nil { return fmt.Errorf("cannot create new file %q: %w", nextPath, err) @@ -404,9 +391,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } } - if err != nil && err != io.EOF { - return fmt.Errorf("cannot read tar stream: %w", err) - } + log.Printf("copy complete\n") return nil } From e44ea1cfb2558629d6971689a63820ff3ee96fe6 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 17 Jan 2025 17:46:57 -0800 Subject: [PATCH 059/126] working, getting copy file error read/write on closed pipe, need to ignore --- pkg/util/iochan/iochan.go | 38 ++++++++++++++++++++++++------- pkg/wshrpc/wshremote/wshremote.go | 21 ++++++++++++----- 2 files changed, 45 insertions(+), 14 deletions(-) diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index ec6f9f18f..b66fae395 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -31,9 +31,15 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func log.Printf("ReaderChan: read error: %v", err) callback() return + } else if err == io.EOF { + log.Printf("ReaderChan: EOF") + return } else if n > 0 { log.Printf("ReaderChan: read %d bytes", n) ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} + } else { + log.Printf("ReaderChan: no data") + return } } }() @@ -43,17 +49,33 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func // WriterChan reads from a channel and writes the data to an io.Writer func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte], callback func()) { go func() { - defer callback() - for resp := range ch { - if resp.Error != nil { - log.Printf("WriterChan: error: %v", resp.Error) + defer func() { + log.Printf("WriterChan: closing channel") + callback() + }() + for { + if ctx.Err() != nil { + log.Printf("WriterChan: context error: %v", ctx.Err()) return } - if n, err := w.Write(resp.Response); err != nil { - log.Printf("WriterChan: write error: %v", err) + select { + case <-ctx.Done(): return - } else { - log.Printf("WriterChan: wrote %d bytes", n) + case resp, ok := <-ch: + if !ok { + log.Printf("WriterChan: channel closed") + return + } + if resp.Error != nil { + log.Printf("WriterChan: error: %v", resp.Error) + return + } + if n, err := w.Write(resp.Response); err != nil { + log.Printf("WriterChan: write error: %v", err) + return + } else { + log.Printf("WriterChan: wrote %d bytes", n) + } } } }() diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 1d87d8ed1..1b627f1e3 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -228,13 +228,16 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) - iochanCtx, cancel := context.WithCancel(ctx) - rtn := iochan.ReaderChan(iochanCtx, pipeReader, wshrpc.FileChunkSize, func() { - tarWriter.Close() + chanCtx, cancel := context.WithCancel(ctx) + rtn := iochan.ReaderChan(chanCtx, pipeReader, wshrpc.FileChunkSize, func() { pipeReader.Close() pipeWriter.Close() }) go func() { + if ctx.Err() != nil { + return + } + defer tarWriter.Close() defer cancel() if finfo.IsDir() { log.Printf("creating tar stream for directory %q\n", path) @@ -243,7 +246,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } } log.Printf("creating tar stream for %q\n", path) - filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { + err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { // generate tar header header, err := tar.FileInfoHeader(fi, file) if err != nil { @@ -252,7 +255,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. header.Name = strings.TrimPrefix(file, path) if header.Name == "" { - header.Name = filepath.Base(file) + return nil } // write header @@ -273,6 +276,9 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } return nil }) + if err != nil { + rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) + } log.Printf("returning tar stream\n") }() log.Printf("returning channel\n") @@ -306,14 +312,17 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C ioch := fileshare.ReadTarStream(ctx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() ctx, cancel := context.WithCancel(ctx) + defer cancel() iochan.WriterChan(ctx, pipeWriter, ioch, func() { log.Printf("closing pipe writer\n") pipeWriter.Close() pipeReader.Close() - cancel() }) tarReader := tar.NewReader(pipeReader) for { + if ctx.Err() != nil { + return ctx.Err() + } next, err := tarReader.Next() if err != nil { if errors.Is(err, io.EOF) { From fa94263eb4ebc49d37cbd35bf623fc35006753e2 Mon Sep 17 00:00:00 2001 From: sawka Date: Fri, 17 Jan 2025 18:59:43 -0800 Subject: [PATCH 060/126] don't allow unregister to block the main sending loop --- pkg/wshutil/wshrpc.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/wshutil/wshrpc.go b/pkg/wshutil/wshrpc.go index 8e23921d0..79db029c5 100644 --- a/pkg/wshutil/wshrpc.go +++ b/pkg/wshutil/wshrpc.go @@ -413,7 +413,13 @@ func (w *WshRpc) unregisterRpc(reqId string, err error) { ResId: reqId, Error: err.Error(), } - rd.ResCh <- errResp + // non-blocking send since we're about to close anyway + // likely the channel isn't being actively read + // this also prevents us from blocking the main loop (and holding the lock) + select { + case rd.ResCh <- errResp: + default: + } } delete(w.RpcMap, reqId) close(rd.ResCh) From 1fe895f3b6e5d6bef9c5ab8fed4d06e5d95ae6bf Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 20 Jan 2025 21:23:02 -0800 Subject: [PATCH 061/126] it's working! --- pkg/util/fileutil/fileutil.go | 62 ++++++++---- pkg/util/iochan/iochan.go | 49 +++++----- pkg/util/iochan/iochan_test.go | 45 +++++++-- pkg/util/utilfn/utilfn.go | 89 ----------------- pkg/wshrpc/wshremote/wshremote.go | 157 ++++++++++++++++-------------- 5 files changed, 187 insertions(+), 215 deletions(-) diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index a30b485e6..6f7562229 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -13,10 +13,53 @@ import ( "github.com/wavetermdev/waveterm/pkg/wavebase" ) +func FixPath(path string) (string, error) { + if strings.HasPrefix(path, "~") { + return filepath.Join(wavebase.GetHomeDir(), path[1:]), nil + } else if !filepath.IsAbs(path) { + log.Printf("FixPath: path is not absolute: %s", path) + path, err := filepath.Abs(path) + if err != nil { + return "", err + } + log.Printf("FixPath: fixed path: %s", path) + return path, nil + } else { + return path, nil + } +} + +const ( + winFlagSoftlink = uint32(0x8000) // FILE_ATTRIBUTE_REPARSE_POINT + winFlagJunction = uint32(0x80) // FILE_ATTRIBUTE_JUNCTION +) + +func WinSymlinkDir(path string, bits os.FileMode) bool { + // Windows compatibility layer doesn't expose symlink target type through fileInfo + // so we need to check file attributes and extension patterns + isFileSymlink := func(filepath string) bool { + if len(filepath) == 0 { + return false + } + return strings.LastIndex(filepath, ".") > strings.LastIndex(filepath, "/") + } + + flags := uint32(bits >> 12) + + if flags == winFlagSoftlink { + return !isFileSymlink(path) + } else if flags == winFlagJunction { + return true + } else { + return false + } +} + // on error just returns "" // does not return "application/octet-stream" as this is considered a detection failure // can pass an existing fileInfo to avoid re-statting the file // falls back to text/plain for 0 byte files + func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { if fileInfo == nil { statRtn, err := os.Stat(path) @@ -25,7 +68,8 @@ func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { } fileInfo = statRtn } - if fileInfo.IsDir() { + + if fileInfo.IsDir() || WinSymlinkDir(path, fileInfo.Mode()) { return "directory" } if fileInfo.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { @@ -69,19 +113,3 @@ func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { } return rtn } - -func FixPath(path string) (string, error) { - if strings.HasPrefix(path, "~") { - return filepath.Join(wavebase.GetHomeDir(), path[1:]), nil - } else if !filepath.IsAbs(path) { - log.Printf("FixPath: path is not absolute: %s", path) - path, err := filepath.Abs(path) - if err != nil { - return "", err - } - log.Printf("FixPath: fixed path: %s", path) - return path, nil - } else { - return path, nil - } -} diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index b66fae395..f62af0e57 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -17,19 +17,21 @@ import ( func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) chan wshrpc.RespOrErrorUnion[[]byte] { ch := make(chan wshrpc.RespOrErrorUnion[[]byte], 16) go func() { - defer close(ch) + defer func() { + log.Printf("ReaderChan: closing channel") + callback() + close(ch) + }() buf := make([]byte, chunkSize) for { if ctx.Err() != nil { log.Printf("ReaderChan: context error: %v", ctx.Err()) - callback() return } if n, err := r.Read(buf); err != nil && err != io.EOF { ch <- wshrpc.RespOrErrorUnion[[]byte]{Error: fmt.Errorf("ReaderChan: read error: %v", err)} log.Printf("ReaderChan: read error: %v", err) - callback() return } else if err == io.EOF { log.Printf("ReaderChan: EOF") @@ -37,9 +39,6 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } else if n > 0 { log.Printf("ReaderChan: read %d bytes", n) ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} - } else { - log.Printf("ReaderChan: no data") - return } } }() @@ -47,36 +46,32 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } // WriterChan reads from a channel and writes the data to an io.Writer -func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte], callback func()) { +func WriterChan(w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte], callback func(), errCallback func(error)) { go func() { defer func() { log.Printf("WriterChan: closing channel") callback() + drainChannel(ch) }() - for { - if ctx.Err() != nil { - log.Printf("WriterChan: context error: %v", ctx.Err()) + for resp := range ch { + if resp.Error != nil { + log.Printf("WriterChan: error: %v", resp.Error) + errCallback(resp.Error) return } - select { - case <-ctx.Done(): + if n, err := w.Write(resp.Response); err != nil { + log.Printf("WriterChan: write error: %v", err) + errCallback(resp.Error) return - case resp, ok := <-ch: - if !ok { - log.Printf("WriterChan: channel closed") - return - } - if resp.Error != nil { - log.Printf("WriterChan: error: %v", resp.Error) - return - } - if n, err := w.Write(resp.Response); err != nil { - log.Printf("WriterChan: write error: %v", err) - return - } else { - log.Printf("WriterChan: wrote %d bytes", n) - } + } else { + log.Printf("WriterChan: wrote %d bytes", n) } } }() } + +func drainChannel(ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { + for range ch { + <-ch + } +} diff --git a/pkg/util/iochan/iochan_test.go b/pkg/util/iochan/iochan_test.go index b1a7361b3..10ec061d8 100644 --- a/pkg/util/iochan/iochan_test.go +++ b/pkg/util/iochan/iochan_test.go @@ -7,26 +7,46 @@ import ( "context" "io" "testing" + "time" "github.com/wavetermdev/waveterm/pkg/util/iochan" ) +const ( + buflen = 1024 +) + func TestIochan_Basic(t *testing.T) { + // Write the packet to the source pipe from a goroutine srcPipeReader, srcPipeWriter := io.Pipe() packet := []byte("hello world") go func() { srcPipeWriter.Write(packet) srcPipeWriter.Close() }() - destPipeReader, destPipeWriter := io.Pipe() - defer destPipeReader.Close() - defer destPipeWriter.Close() - ioch := iochan.ReaderChan(context.TODO(), srcPipeReader, 1024, func() { + + // Initialize the reader channel + readerChanCallbackCalled := false + readerChanCallback := func() { srcPipeReader.Close() - srcPipeWriter.Close() - }) - iochan.WriterChan(context.TODO(), destPipeWriter, ioch) - buf := make([]byte, 1024) + readerChanCallbackCalled = true + } + defer readerChanCallback() // Ensure the callback is called + ioch := iochan.ReaderChan(context.TODO(), srcPipeReader, buflen, readerChanCallback) + + // Initialize the destination pipe and the writer channel + destPipeReader, destPipeWriter := io.Pipe() + writerChanCallbackCalled := false + writerChanCallback := func() { + destPipeReader.Close() + destPipeWriter.Close() + writerChanCallbackCalled = true + } + defer writerChanCallback() // Ensure the callback is called + iochan.WriterChan(destPipeWriter, ioch, writerChanCallback, func(err error) {}) + + // Read the packet from the destination pipe and compare it to the original packet + buf := make([]byte, buflen) n, err := destPipeReader.Read(buf) if err != nil { t.Fatalf("Read failed: %v", err) @@ -37,4 +57,13 @@ func TestIochan_Basic(t *testing.T) { if string(buf[:n]) != string(packet) { t.Fatalf("Read data mismatch: %s != %s", buf[:n], packet) } + + // Give the callbacks a chance to run before checking if they were called + time.Sleep(10 * time.Millisecond) + if !readerChanCallbackCalled { + t.Fatalf("ReaderChan callback not called") + } + if !writerChanCallbackCalled { + t.Fatalf("WriterChan callback not called") + } } diff --git a/pkg/util/utilfn/utilfn.go b/pkg/util/utilfn/utilfn.go index 7bec011ab..a9588c8ab 100644 --- a/pkg/util/utilfn/utilfn.go +++ b/pkg/util/utilfn/utilfn.go @@ -14,14 +14,10 @@ import ( "errors" "fmt" "io" - "io/fs" "math" mathrand "math/rand" - "mime" - "net/http" "os" "os/exec" - "path/filepath" "reflect" "regexp" "sort" @@ -618,91 +614,6 @@ func CopyToChannel(outputCh chan<- []byte, reader io.Reader) error { } } -const ( - winFlagSoftlink = uint32(0x8000) // FILE_ATTRIBUTE_REPARSE_POINT - winFlagJunction = uint32(0x80) // FILE_ATTRIBUTE_JUNCTION -) - -func WinSymlinkDir(path string, bits os.FileMode) bool { - // Windows compatibility layer doesn't expose symlink target type through fileInfo - // so we need to check file attributes and extension patterns - isFileSymlink := func(filepath string) bool { - if len(filepath) == 0 { - return false - } - return strings.LastIndex(filepath, ".") > strings.LastIndex(filepath, "/") - } - - flags := uint32(bits >> 12) - - if flags == winFlagSoftlink { - return !isFileSymlink(path) - } else if flags == winFlagJunction { - return true - } else { - return false - } -} - -// on error just returns "" -// does not return "application/octet-stream" as this is considered a detection failure -// can pass an existing fileInfo to avoid re-statting the file -// falls back to text/plain for 0 byte files - -func DetectMimeType(path string, fileInfo fs.FileInfo, extended bool) string { - if fileInfo == nil { - statRtn, err := os.Stat(path) - if err != nil { - return "" - } - fileInfo = statRtn - } - - if fileInfo.IsDir() || WinSymlinkDir(path, fileInfo.Mode()) { - return "directory" - } - if fileInfo.Mode()&os.ModeNamedPipe == os.ModeNamedPipe { - return "pipe" - } - charDevice := os.ModeDevice | os.ModeCharDevice - if fileInfo.Mode()&charDevice == charDevice { - return "character-special" - } - if fileInfo.Mode()&os.ModeDevice == os.ModeDevice { - return "block-special" - } - ext := filepath.Ext(path) - if mimeType, ok := StaticMimeTypeMap[ext]; ok { - return mimeType - } - if mimeType := mime.TypeByExtension(ext); mimeType != "" { - return mimeType - } - if fileInfo.Size() == 0 { - return "text/plain" - } - if !extended { - return "" - } - fd, err := os.Open(path) - if err != nil { - return "" - } - defer fd.Close() - buf := make([]byte, 512) - // ignore the error (EOF / UnexpectedEOF is fine, just process how much we got back) - n, _ := io.ReadAtLeast(fd, buf, 512) - if n == 0 { - return "" - } - buf = buf[:n] - rtn := http.DetectContentType(buf) - if rtn == "application/octet-stream" { - return "" - } - return rtn -} - func GetCmdExitCode(cmd *exec.Cmd, err error) int { if cmd == nil || cmd.ProcessState == nil { return GetExitCode(err) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 1b627f1e3..3677e0a96 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -228,8 +228,8 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) - chanCtx, cancel := context.WithCancel(ctx) - rtn := iochan.ReaderChan(chanCtx, pipeReader, wshrpc.FileChunkSize, func() { + readerCtx, _ := context.WithTimeout(context.Background(), time.Second*30) + rtn := iochan.ReaderChan(readerCtx, pipeReader, wshrpc.FileChunkSize, func() { pipeReader.Close() pipeWriter.Close() }) @@ -238,14 +238,14 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. return } defer tarWriter.Close() - defer cancel() + log.Printf("creating tar stream for %q\n", path) if finfo.IsDir() { - log.Printf("creating tar stream for directory %q\n", path) + log.Printf("%q is a directory, recursive: %v\n", path, recursive) if !recursive { rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) + return } } - log.Printf("creating tar stream for %q\n", path) err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { // generate tar header header, err := tar.FileInfoHeader(fi, file) @@ -274,6 +274,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. log.Printf("wrote %d bytes to tar stream\n", n) } } + time.Sleep(time.Millisecond * 10) return nil }) if err != nil { @@ -309,99 +310,107 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot stat destination %q: %w", destPath, err) } log.Printf("copying %q to %q\n", srcUri, destPath) - ioch := fileshare.ReadTarStream(ctx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) + readCtx, _ := context.WithTimeout(ctx, time.Second*30) + readCtx, cancel := context.WithCancelCause(readCtx) + ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() - ctx, cancel := context.WithCancel(ctx) - defer cancel() - iochan.WriterChan(ctx, pipeWriter, ioch, func() { + iochan.WriterChan(pipeWriter, ioch, func() { log.Printf("closing pipe writer\n") pipeWriter.Close() pipeReader.Close() - }) + }, cancel) tarReader := tar.NewReader(pipeReader) for { - if ctx.Err() != nil { - return ctx.Err() - } - next, err := tarReader.Next() - if err != nil { - if errors.Is(err, io.EOF) { - break + select { + case <-readCtx.Done(): + if readCtx.Err() != nil { + return context.Cause(readCtx) } - return fmt.Errorf("cannot read tar stream: %w", err) - } - finfo := next.FileInfo() - nextPath := filepath.Join(destPathCleaned, next.Name) - destinfo, err = os.Stat(nextPath) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("cannot stat file %q: %w", nextPath, err) - } - log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) - - if destinfo != nil { - if destinfo.IsDir() { - if !finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) + return nil + default: + next, err := tarReader.Next() + if err != nil { + if errors.Is(err, io.EOF) { + // Do one more check for context error before returning + if readCtx.Err() != nil { + return context.Cause(readCtx) + } + return nil + } + return fmt.Errorf("cannot read tar stream: %w", err) + } + finfo := next.FileInfo() + nextPath := filepath.Join(destPathCleaned, next.Name) + destinfo, err = os.Stat(nextPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("cannot stat file %q: %w", nextPath, err) + } + log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) + + if destinfo != nil { + if destinfo.IsDir() { + if !finfo.IsDir() { + if !overwrite { + return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) + } else { + err := os.Remove(nextPath) + if err != nil { + return fmt.Errorf("cannot remove file %q: %w", nextPath, err) + } + } + } else if !merge && !overwrite { + return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) + } else if overwrite { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + } + } + } else { + if finfo.IsDir() { + if !overwrite { + return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) + } else { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + } + } + } else if !overwrite { + return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) } else { err := os.Remove(nextPath) if err != nil { return fmt.Errorf("cannot remove file %q: %w", nextPath, err) } } - } else if !merge && !overwrite { - return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) - } else if overwrite { - err := os.RemoveAll(nextPath) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) - } } } else { if finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) - } else { - err := os.RemoveAll(nextPath) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) - } + log.Printf("creating directory %q\n", nextPath) + err := os.MkdirAll(nextPath, finfo.Mode()) + if err != nil { + return fmt.Errorf("cannot create directory %q: %w", nextPath, err) } - } else if !overwrite { - return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) } else { - err := os.Remove(nextPath) + err := os.MkdirAll(filepath.Dir(nextPath), 0755) if err != nil { - return fmt.Errorf("cannot remove file %q: %w", nextPath, err) + return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) } + file, err := os.Create(nextPath) + if err != nil { + return fmt.Errorf("cannot create new file %q: %w", nextPath, err) + } + _, err = io.Copy(file, tarReader) + if err != nil { + return fmt.Errorf("cannot write file %q: %w", nextPath, err) + } + file.Chmod(finfo.Mode()) + file.Close() } } - } else { - if finfo.IsDir() { - err := os.MkdirAll(nextPath, finfo.Mode()) - if err != nil { - return fmt.Errorf("cannot create directory %q: %w", nextPath, err) - } - } else { - err := os.MkdirAll(filepath.Dir(nextPath), 0755) - if err != nil { - return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) - } - file, err := os.Create(nextPath) - if err != nil { - return fmt.Errorf("cannot create new file %q: %w", nextPath, err) - } - _, err = io.Copy(file, tarReader) - if err != nil { - return fmt.Errorf("cannot write file %q: %w", nextPath, err) - } - file.Chmod(finfo.Mode()) - file.Close() - } } } - log.Printf("copy complete\n") - return nil } func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrpc.CommandRemoteListEntriesData) chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { From 5f42b976593e3c7f84730ceefd6b7a075bdebe78 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 20 Jan 2025 23:15:54 -0800 Subject: [PATCH 062/126] save --- pkg/util/iochan/iochan.go | 37 ++++++++++++++++++++----------- pkg/util/iochan/iochan_test.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 15 ++++++++++--- 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index f62af0e57..79b2f853b 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -25,6 +25,9 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func buf := make([]byte, chunkSize) for { if ctx.Err() != nil { + if ctx.Err() == context.Canceled { + return + } log.Printf("ReaderChan: context error: %v", ctx.Err()) return } @@ -37,7 +40,7 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func log.Printf("ReaderChan: EOF") return } else if n > 0 { - log.Printf("ReaderChan: read %d bytes", n) + // log.Printf("ReaderChan: read %d bytes", n) ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} } } @@ -46,25 +49,33 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } // WriterChan reads from a channel and writes the data to an io.Writer -func WriterChan(w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte], callback func(), errCallback func(error)) { +func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte], callback func(), errCallback func(error)) { go func() { defer func() { log.Printf("WriterChan: closing channel") callback() drainChannel(ch) }() - for resp := range ch { - if resp.Error != nil { - log.Printf("WriterChan: error: %v", resp.Error) - errCallback(resp.Error) - return - } - if n, err := w.Write(resp.Response); err != nil { - log.Printf("WriterChan: write error: %v", err) - errCallback(resp.Error) + for { + select { + case <-ctx.Done(): return - } else { - log.Printf("WriterChan: wrote %d bytes", n) + case resp, ok := <-ch: + if !ok { + return + } + if resp.Error != nil { + log.Printf("WriterChan: error: %v", resp.Error) + errCallback(resp.Error) + return + } + if _, err := w.Write(resp.Response); err != nil { + log.Printf("WriterChan: write error: %v", err) + errCallback(resp.Error) + return + } else { + // log.Printf("WriterChan: wrote %d bytes", n) + } } } }() diff --git a/pkg/util/iochan/iochan_test.go b/pkg/util/iochan/iochan_test.go index 10ec061d8..dd8eeb092 100644 --- a/pkg/util/iochan/iochan_test.go +++ b/pkg/util/iochan/iochan_test.go @@ -43,7 +43,7 @@ func TestIochan_Basic(t *testing.T) { writerChanCallbackCalled = true } defer writerChanCallback() // Ensure the callback is called - iochan.WriterChan(destPipeWriter, ioch, writerChanCallback, func(err error) {}) + iochan.WriterChan(context.TODO(), destPipeWriter, ioch, writerChanCallback, func(err error) {}) // Read the packet from the destination pipe and compare it to the original packet buf := make([]byte, buflen) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 3677e0a96..ffc47df50 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -233,8 +233,15 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. pipeReader.Close() pipeWriter.Close() }) + + var pathPrefix string + if finfo.IsDir() && strings.HasSuffix(cleanedPath, "/") { + pathPrefix = cleanedPath + } else { + pathPrefix = filepath.Dir(cleanedPath) + } go func() { - if ctx.Err() != nil { + if readerCtx.Err() != nil { return } defer tarWriter.Close() @@ -253,7 +260,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. return err } - header.Name = strings.TrimPrefix(file, path) + header.Name = strings.TrimPrefix(file, pathPrefix) if header.Name == "" { return nil } @@ -269,6 +276,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. return err } if n, err := io.Copy(tarWriter, data); err != nil { + log.Printf("error copying file %q: %v\n", file, err) return err } else { log.Printf("wrote %d bytes to tar stream\n", n) @@ -314,11 +322,12 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C readCtx, cancel := context.WithCancelCause(readCtx) ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() - iochan.WriterChan(pipeWriter, ioch, func() { + iochan.WriterChan(readCtx, pipeWriter, ioch, func() { log.Printf("closing pipe writer\n") pipeWriter.Close() pipeReader.Close() }, cancel) + defer cancel(nil) tarReader := tar.NewReader(pipeReader) for { select { From 367044e7fe0f61332d44205169c67bb9b4a24326 Mon Sep 17 00:00:00 2001 From: sawka Date: Mon, 20 Jan 2025 23:35:35 -0800 Subject: [PATCH 063/126] add panichandler to runServer --- pkg/wshutil/wshrpc.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/wshutil/wshrpc.go b/pkg/wshutil/wshrpc.go index 79db029c5..3d26397da 100644 --- a/pkg/wshutil/wshrpc.go +++ b/pkg/wshutil/wshrpc.go @@ -326,6 +326,9 @@ func (w *WshRpc) handleRequest(req *RpcMessage) { } func (w *WshRpc) runServer() { + defer func() { + panichandler.PanicHandler("wshrpc.runServer", recover()) + }() defer close(w.OutputCh) for msgBytes := range w.InputCh { if w.Debug { From 570885b80db53c5d48742f202b9c3ed8f7e62327 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 11:05:49 -0800 Subject: [PATCH 064/126] timeout --- cmd/wsh/cmd/wshcmd-file.go | 77 ++------------------------- frontend/types/gotypes.d.ts | 1 + pkg/remote/fileshare/wshfs/wshfs.go | 17 +++++- pkg/util/iochan/iochan.go | 34 ++++++------ pkg/wshrpc/wshclient/wshclientutil.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 12 ++++- pkg/wshrpc/wshrpctypes.go | 1 + 7 files changed, 50 insertions(+), 94 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 5c6a9c35d..0eda84d0d 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -33,6 +33,7 @@ const ( WaveFilePrefix = "wavefile://" DefaultFileTimeout = 5000 + TimeoutYear = int(365 * 24 * time.Hour) // 1 year UriHelpText = ` @@ -394,86 +395,14 @@ func fileCpRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("unable to parse dest path: %w", err) } log.Printf("Copying %s to %s; recursive: %v, merge: %v, force: %v", srcPath, destPath, recursive, merge, force) - err = wshclient.FileCopyCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcPath, DestUri: destPath, Opts: &wshrpc.FileCopyOpts{Recursive: recursive, Merge: merge, Overwrite: force}}, &wshrpc.RpcOpts{Timeout: fileTimeout}) + rpcOpts := &wshrpc.RpcOpts{Timeout: TimeoutYear} + err = wshclient.FileCopyCommand(RpcClient, wshrpc.CommandFileCopyData{SrcUri: srcPath, DestUri: destPath, Opts: &wshrpc.FileCopyOpts{Recursive: recursive, Merge: merge, Overwrite: force, Timeout: TimeoutYear}}, rpcOpts) if err != nil { return fmt.Errorf("copying file: %w", err) } return nil } -func copyFromWaveToLocal(src, dst string) error { - path, err := fixRelativePaths(src) - if err != nil { - return err - } - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} - - // Get file info first to check existence and get size - info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) - err = convertNotFoundErr(err) - if err == fs.ErrNotExist { - return fmt.Errorf("%s: no such file", src) - } - if err != nil { - return fmt.Errorf("getting file info: %w", err) - } - - // Create the destination file - f, err := os.Create(dst) - if err != nil { - return fmt.Errorf("creating local file: %w", err) - } - defer f.Close() - - err = streamReadFromFile(fileData, info.Size, f) - if err != nil { - return fmt.Errorf("reading wave file: %w", err) - } - - return nil -} - -func copyFromLocalToWave(src, dst string) error { - path, err := fixRelativePaths(dst) - if err != nil { - return err - } - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} - - // stat local file - stat, err := os.Stat(src) - if err == fs.ErrNotExist { - return fmt.Errorf("%s: no such file", src) - } - if err != nil { - return fmt.Errorf("stat local file: %w", err) - } - if stat.IsDir() { - return fmt.Errorf("%s: is a directory", src) - } - _, err = ensureFile(dst, fileData) - if err != nil { - return err - } - - file, err := os.Open(src) - if err != nil { - return fmt.Errorf("opening local file: %w", err) - } - defer file.Close() - - err = streamWriteToFile(fileData, file) - if err != nil { - return fmt.Errorf("writing wave file: %w", err) - } - - return nil -} - func filePrintColumns(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData]) error { width := 80 // default if we can't get terminal if w, _, err := term.GetSize(int(os.Stdout.Fd())); err == nil { diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index cc9c6cb37..9c7f983dc 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -371,6 +371,7 @@ declare global { overwrite?: boolean; recursive?: boolean; merge?: boolean; + timeout?: number; }; // wshrpc.FileData diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 97aa4cd6f..b26784073 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log" + "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" @@ -18,6 +19,10 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) +const ( + ThirtySeconds = int(30 * time.Second) +) + // This needs to be set by whoever initializes the client, either main-server or wshcmd-connserver var RpcClient *wshutil.WshRpc @@ -87,7 +92,11 @@ func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, d func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { log.Print("ReadTarStream wshfs") - return wshclient.RemoteTarStreamCommand(RpcClient, wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + timeout := opts.Timeout + if timeout == 0 { + timeout = ThirtySeconds + } + return wshclient.RemoteTarStreamCommand(RpcClient, wshrpc.CommandRemoteStreamTarData{Path: conn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host), Timeout: timeout}) } func (c WshClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { @@ -125,7 +134,11 @@ func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connec } func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return wshclient.RemoteFileCopyCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestPath: destConn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host)}) + timeout := opts.Timeout + if timeout == 0 { + timeout = ThirtySeconds + } + return wshclient.RemoteFileCopyCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestPath: destConn.Path, Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) } func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 79b2f853b..dbf5b15c8 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -6,42 +6,46 @@ package iochan import ( "context" + "errors" "fmt" "io" "log" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshutil" ) // ReaderChan reads from an io.Reader and sends the data to a channel func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) chan wshrpc.RespOrErrorUnion[[]byte] { - ch := make(chan wshrpc.RespOrErrorUnion[[]byte], 16) + ch := make(chan wshrpc.RespOrErrorUnion[[]byte], 32) go func() { defer func() { log.Printf("ReaderChan: closing channel") - callback() close(ch) + callback() }() buf := make([]byte, chunkSize) for { - if ctx.Err() != nil { + select { + case <-ctx.Done(): if ctx.Err() == context.Canceled { return } log.Printf("ReaderChan: context error: %v", ctx.Err()) return - } - - if n, err := r.Read(buf); err != nil && err != io.EOF { - ch <- wshrpc.RespOrErrorUnion[[]byte]{Error: fmt.Errorf("ReaderChan: read error: %v", err)} - log.Printf("ReaderChan: read error: %v", err) - return - } else if err == io.EOF { - log.Printf("ReaderChan: EOF") - return - } else if n > 0 { - // log.Printf("ReaderChan: read %d bytes", n) - ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} + default: + if n, err := r.Read(buf); err != nil { + if errors.Is(err, io.EOF) { + log.Printf("ReaderChan: EOF") + return + } + ch <- wshutil.RespErr[[]byte](fmt.Errorf("ReaderChan: read error: %v", err)) + log.Printf("ReaderChan: read error: %v", err) + return + } else if n > 0 { + // log.Printf("ReaderChan: read %d bytes", n) + ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} + } } } }() diff --git a/pkg/wshrpc/wshclient/wshclientutil.go b/pkg/wshrpc/wshclient/wshclientutil.go index 1a0cab961..327466b9f 100644 --- a/pkg/wshrpc/wshclient/wshclientutil.go +++ b/pkg/wshrpc/wshclient/wshclientutil.go @@ -52,7 +52,7 @@ func sendRpcRequestResponseStreamHelper[T any](w *wshutil.WshRpc, command string if opts == nil { opts = &wshrpc.RpcOpts{} } - respChan := make(chan wshrpc.RespOrErrorUnion[T]) + respChan := make(chan wshrpc.RespOrErrorUnion[T], 32) if w == nil { rtnErr(respChan, errors.New("nil wshrpc passed to wshclient")) return respChan diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index ffc47df50..25ff4275a 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -228,7 +228,11 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) - readerCtx, _ := context.WithTimeout(context.Background(), time.Second*30) + timeout := time.Millisecond * 100 + if opts.Timeout > 0 { + timeout = time.Duration(opts.Timeout) + } + readerCtx, _ := context.WithTimeout(context.Background(), timeout) rtn := iochan.ReaderChan(readerCtx, pipeReader, wshrpc.FileChunkSize, func() { pipeReader.Close() pipeWriter.Close() @@ -318,7 +322,11 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot stat destination %q: %w", destPath, err) } log.Printf("copying %q to %q\n", srcUri, destPath) - readCtx, _ := context.WithTimeout(ctx, time.Second*30) + timeout := time.Millisecond * 100 + if opts.Timeout > 0 { + timeout = time.Duration(opts.Timeout) + } + readCtx, _ := context.WithTimeout(ctx, timeout) readCtx, cancel := context.WithCancelCause(readCtx) ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index d1ba0c845..a5e1ae155 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -512,6 +512,7 @@ type FileCopyOpts struct { Overwrite bool `json:"overwrite,omitempty"` Recursive bool `json:"recursive,omitempty"` Merge bool `json:"merge,omitempty"` + Timeout int `json:"timeout,omitempty"` } type CommandRemoteStreamFileData struct { From 3283314f6c0447a644b213f20e4e6653caf8f86a Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 15:58:23 -0800 Subject: [PATCH 065/126] Disable S3, add fileappend --- cmd/wsh/cmd/wshcmd-file-util.go | 2 +- cmd/wsh/cmd/wshcmd-file.go | 8 +++--- frontend/app/store/wshclientapi.ts | 2 +- frontend/types/gotypes.d.ts | 7 ----- pkg/remote/fileshare/fileshare.go | 39 ++++++++++++++++++++------- pkg/remote/fileshare/wavefs/wavefs.go | 4 +-- pkg/remote/fileshare/wshfs/wshfs.go | 5 ++-- pkg/wshrpc/wshclient/wshclient.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 39 ++++++++++++++++++++++----- pkg/wshrpc/wshrpctypes.go | 10 ++----- pkg/wshrpc/wshserver/wshserver.go | 4 +++ 11 files changed, 79 insertions(+), 43 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 499772bf2..6c38280d8 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -102,7 +102,7 @@ func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) // Set up the ReadAt request fileData.At = &wshrpc.FileDataAt{ Offset: offset, - Size: int64(length), + Size: length, } // Read the chunk diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 0eda84d0d..06982a2df 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -88,10 +88,10 @@ func init() { fileCmd.AddCommand(fileRmCmd) fileCmd.AddCommand(fileInfoCmd) fileCmd.AddCommand(fileAppendCmd) - fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") - fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") - fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") - fileCmd.AddCommand(fileCpCmd) + // fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") + // fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") + // fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") + // fileCmd.AddCommand(fileCpCmd) } type waveFileRef struct { diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 1dcee0799..65663489d 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -308,7 +308,7 @@ class RpcApiType { } // command "remotewritefile" [call] - RemoteWriteFileCommand(client: WshClient, data: CommandRemoteWriteFileData, opts?: RpcOpts): Promise { + RemoteWriteFileCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("remotewritefile", data, opts); } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 9c7f983dc..7d479634b 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -226,13 +226,6 @@ declare global { opts?: FileCopyOpts; }; - // wshrpc.CommandRemoteWriteFileData - type CommandRemoteWriteFileData = { - path: string; - data64: string; - createmode?: number; - }; - // wshrpc.CommandResolveIdsData type CommandResolveIdsData = { blockid: string; diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index d90ff428c..d272f609c 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -5,10 +5,8 @@ import ( "fmt" "log" - "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare/s3fs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -31,16 +29,21 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS conntype := conn.GetType() log.Printf("CreateFileShareClient: conntype=%s", conntype) if conntype == connparse.ConnectionTypeS3 { - config, err := awsconn.GetConfig(ctx, connection) - if err != nil { - log.Printf("error getting aws config: %v", err) - return nil, nil - } - return s3fs.NewS3Client(config), conn + // TODO: enable s3fs + return nil, nil + // config, err := awsconn.GetConfig(ctx, connection) + // if err != nil { + // log.Printf("error getting aws config: %v", err) + // return nil, nil + // } + // return s3fs.NewS3Client(config), conn } else if conntype == connparse.ConnectionTypeWave { return wavefs.NewWaveClient(), conn - } else { + } else if conntype == connparse.ConnectionTypeWsh { return wshfs.NewWshClient(), conn + } else { + log.Printf("unsupported connection type: %s", conntype) + return nil, nil } } @@ -164,3 +167,21 @@ func Join(ctx context.Context, path string, parts ...string) (string, error) { } return client.Join(ctx, conn, parts...) } + +func Append(ctx context.Context, data wshrpc.FileData) error { + path := data.Info.Path + log.Printf("Append: path=%s", path) + client, conn := CreateFileShareClient(ctx, path) + if conn == nil || client == nil { + return fmt.Errorf(ErrorParsingConnection, path) + } + fdata, err := client.Read(ctx, conn, data) + if err != nil { + return err + } + data.Info = fdata.Info + data.At = &wshrpc.FileDataAt{ + Offset: fdata.Info.Size, + } + return client.PutFile(ctx, conn, data) +} diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index a8c7893f4..2395b697f 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -46,7 +46,7 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, if !rtnData.Info.IsDir { for i := 0; i < dataLen; i += wshrpc.FileChunkSize { dataEnd := min(i+wshrpc.FileChunkSize, dataLen) - ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: rtnData.Data64[i:dataEnd], Info: rtnData.Info, At: &wshrpc.FileDataAt{Offset: int64(i), Size: int64(dataEnd - i)}}} + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: rtnData.Data64[i:dataEnd], Info: rtnData.Info, At: &wshrpc.FileDataAt{Offset: int64(i), Size: dataEnd - i}}} } } else { for i := 0; i < len(rtnData.Entries); i += wshrpc.DirChunkSize { @@ -68,7 +68,7 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w return nil, fmt.Errorf("error cleaning path: %w", err) } if data.At != nil { - _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, data.At.Size) + _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, int64(data.At.Size)) if err == nil { return &wshrpc.FileData{Info: data.Info, Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil } else if err == fs.ErrNotExist { diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index b26784073..62a07f4af 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -84,7 +84,7 @@ func (c WshClient) Read(ctx context.Context, conn *connparse.Connection, data ws func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { byteRange := "" if data.At != nil && data.At.Size > 0 { - byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+data.At.Size) + byteRange = fmt.Sprintf("%d-%d", data.At.Offset, data.At.Offset+int64(data.At.Size)) } streamFileData := wshrpc.CommandRemoteStreamFileData{Path: conn.Path, ByteRange: byteRange} return wshclient.RemoteStreamFileCommand(RpcClient, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) @@ -121,8 +121,7 @@ func (c WshClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshrp } func (c WshClient) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { - writeData := wshrpc.CommandRemoteWriteFileData{Path: conn.Path, Data64: data.Data64} - return wshclient.RemoteWriteFileCommand(RpcClient, writeData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) + return wshclient.RemoteWriteFileCommand(RpcClient, data, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 348c7ffd6..0bdc3afdd 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -369,7 +369,7 @@ func RemoteTarStreamCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTa } // command "remotewritefile", wshserver.RemoteWriteFileCommand -func RemoteWriteFileCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteWriteFileData, opts *wshrpc.RpcOpts) error { +func RemoteWriteFileCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "remotewritefile", data, opts) return err } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 25ff4275a..862b3edaa 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -200,7 +200,7 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc } if len(data) > 0 { resp.Data64 = base64.StdEncoding.EncodeToString(data) - resp.At = &wshrpc.FileDataAt{Offset: byteRange.Start, Size: int64(len(data))} + resp.At = &wshrpc.FileDataAt{Offset: byteRange.Start, Size: len(data)} } log.Printf("callback -- sending response %d\n", len(resp.Data64)) ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: resp} @@ -648,25 +648,50 @@ func (impl *ServerImpl) RemoteMkdirCommand(ctx context.Context, path string) err return nil } -func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.CommandRemoteWriteFileData) error { - path, err := wavebase.ExpandHomeDir(data.Path) +func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileData) error { + path, err := wavebase.ExpandHomeDir(data.Info.Path) if err != nil { return err } - createMode := data.CreateMode + createMode := data.Info.Mode if createMode == 0 { createMode = 0644 } dataSize := base64.StdEncoding.DecodedLen(len(data.Data64)) dataBytes := make([]byte, dataSize) - n, err := base64.StdEncoding.Decode(dataBytes, []byte(data.Data64)) + decodedLen, err := base64.StdEncoding.Decode(dataBytes, []byte(data.Data64)) if err != nil { return fmt.Errorf("cannot decode base64 data: %w", err) } - err = os.WriteFile(path, dataBytes[:n], createMode) + finfo, err := os.Stat(path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("cannot stat file %q: %w", path, err) + } + fileSize := int64(0) + if finfo != nil { + fileSize = finfo.Size() + } + offset := fileSize + size := decodedLen + if data.At != nil { + if data.At.Offset > 0 { + offset = min(data.At.Offset, fileSize) + } + if data.At.Size > 0 { + size = min(data.At.Size, decodedLen) + } + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, createMode) + if err != nil { + return fmt.Errorf("cannot open file %q: %w", path, err) + } + offsetWriter := io.NewOffsetWriter(file, offset) + n, err := offsetWriter.Write(dataBytes[:size]) if err != nil { - return fmt.Errorf("cannot write file %q: %w", path, err) + return fmt.Errorf("cannot write to file %q: %w", path, err) } + log.Printf("wrote %d bytes to file %q at offset %q\n", n, path, offset) return nil } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index a5e1ae155..27c96da21 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -200,7 +200,7 @@ type WshRpcInterface interface { RemoteFileTouchCommand(ctx context.Context, path string) error RemoteFileRenameCommand(ctx context.Context, pathTuple [2]string) error RemoteFileDeleteCommand(ctx context.Context, path string) error - RemoteWriteFileCommand(ctx context.Context, data CommandRemoteWriteFileData) error + RemoteWriteFileCommand(ctx context.Context, data FileData) error RemoteFileJoinCommand(ctx context.Context, paths []string) (*FileInfo, error) RemoteMkdirCommand(ctx context.Context, path string) error RemoteStreamCpuDataCommand(ctx context.Context) chan RespOrErrorUnion[TimeSeriesData] @@ -363,7 +363,7 @@ type CommandBlockInputData struct { type FileDataAt struct { Offset int64 `json:"offset"` - Size int64 `json:"size,omitempty"` + Size int `json:"size,omitempty"` } type FileData struct { @@ -529,12 +529,6 @@ type CommandRemoteListEntriesRtnData struct { FileInfo []*FileInfo `json:"fileinfo,omitempty"` } -type CommandRemoteWriteFileData struct { - Path string `json:"path"` - Data64 string `json:"data64"` - CreateMode os.FileMode `json:"createmode,omitempty"` -} - type ConnRequest struct { Host string `json:"host"` Keywords wconfig.ConnKeywords `json:"keywords,omitempty"` diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 328c48edc..6b2b66195 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -328,6 +328,10 @@ func (ws *WshServer) FileStreamTarCommand(ctx context.Context, data wshrpc.Comma return fileshare.ReadTarStream(ctx, data) } +func (ws *WshServer) FileAppendCommand(ctx context.Context, data wshrpc.FileData) error { + return fileshare.Append(ctx, data) +} + func (ws *WshServer) FileAppendIJsonCommand(ctx context.Context, data wshrpc.CommandAppendIJsonData) error { tryCreate := true if data.FileName == blockcontroller.BlockFile_VDom && tryCreate { From e043a7b38e324d30a1f67d37ca23d6eb033979ad Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:29:18 -0800 Subject: [PATCH 066/126] clean up opts, fix append for wavefile --- cmd/wsh/cmd/wshcmd-file-util.go | 3 -- frontend/types/gotypes.d.ts | 1 - pkg/remote/connparse/connparse.go | 4 --- pkg/remote/fileshare/fileshare.go | 13 ++++----- pkg/remote/fileshare/wavefs/wavefs.go | 40 +++++++++++++++++++++------ pkg/wshrpc/wshremote/wshremote.go | 8 ++---- pkg/wshrpc/wshrpctypes.go | 9 +++--- pkg/wshrpc/wshserver/wshserver.go | 13 ++------- 8 files changed, 46 insertions(+), 45 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 6c38280d8..6be5732e4 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -8,7 +8,6 @@ import ( "fmt" "io" "io/fs" - "log" "strings" "github.com/wavetermdev/waveterm/pkg/remote/connparse" @@ -211,13 +210,11 @@ func streamFileList(zoneId string, path string, recursive bool, filesOnly bool) } func fixRelativePaths(path string) (string, error) { - log.Printf("Fixing relative paths: %s", path) conn, err := connparse.ParseURI(path) if err != nil { return "", err } if conn.Scheme == connparse.ConnectionTypeWsh { - log.Printf("wsh conn type %s; %s", conn.Host, conn.Path) fixedPath := conn.Path if conn.Host == connparse.ConnHostCurrent { conn.Host = RpcContext.Conn diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 7d479634b..41476013a 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -372,7 +372,6 @@ declare global { info?: FileInfo; data64?: string; entries?: FileInfo[]; - opts?: FileOptsType; at?: FileDataAt; }; diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index e2fa6f9fd..f5c88a87f 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -6,7 +6,6 @@ package connparse import ( "context" "fmt" - "log" "strings" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -105,7 +104,6 @@ func ParseURI(uri string) (*Connection, error) { host = wshrpc.LocalConnName remotePath = rest } else { - log.Printf("Fixing relative path to %s; %s", ConnHostCurrent, rest) host = ConnHostCurrent remotePath = rest } @@ -126,7 +124,6 @@ func ParseURI(uri string) (*Connection, error) { } if strings.HasPrefix(remotePath, "/~") { remotePath = strings.TrimPrefix(remotePath, "/") - log.Printf("Fixing relative path to %s; %s", host, remotePath) } } @@ -135,6 +132,5 @@ func ParseURI(uri string) (*Connection, error) { Host: host, Path: remotePath, } - log.Printf("Parsed connection: %v", conn) return conn, nil } diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index d272f609c..3693b5688 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -107,11 +107,7 @@ func PutFile(ctx context.Context, data wshrpc.FileData) error { if conn == nil || client == nil { return fmt.Errorf(ErrorParsingConnection, data.Info.Path) } - return client.PutFile(ctx, conn, wshrpc.FileData{ - Data64: data.Data64, - Opts: data.Opts, - At: data.At, - }) + return client.PutFile(ctx, conn, data) } func Mkdir(ctx context.Context, path string) error { @@ -175,13 +171,14 @@ func Append(ctx context.Context, data wshrpc.FileData) error { if conn == nil || client == nil { return fmt.Errorf(ErrorParsingConnection, path) } - fdata, err := client.Read(ctx, conn, data) + finfo, err := client.Stat(ctx, conn) if err != nil { return err } - data.Info = fdata.Info + data.Info = finfo data.At = &wshrpc.FileDataAt{ - Offset: fdata.Info.Size, + Offset: finfo.Size, } + log.Printf("Append: offset=%d", data.At.Offset) return client.PutFile(ctx, conn, data) } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 2395b697f..1aaca37a1 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -6,6 +6,7 @@ package wavefs import ( "context" "encoding/base64" + "errors" "fmt" "io/fs" "path" @@ -71,7 +72,7 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, int64(data.At.Size)) if err == nil { return &wshrpc.FileData{Info: data.Info, Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil - } else if err == fs.ErrNotExist { + } else if errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("NOTFOUND: %w", err) } else { return nil, fmt.Errorf("error reading blockfile: %w", err) @@ -80,7 +81,7 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w _, dataBuf, err := filestore.WFS.ReadFile(ctx, zoneId, fileName) if err == nil { return &wshrpc.FileData{Info: data.Info, Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil - } else if err != fs.ErrNotExist { + } else if !errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("error reading blockfile: %w", err) } } @@ -191,7 +192,7 @@ func (c WaveClient) Stat(ctx context.Context, conn *connparse.Connection) (*wshr } fileInfo, err := filestore.WFS.Stat(ctx, zoneId, fileName) if err != nil { - if err == fs.ErrNotExist { + if errors.Is(err, fs.ErrNotExist) { return nil, fmt.Errorf("NOTFOUND: %w", err) } return nil, fmt.Errorf("error getting file info: %w", err) @@ -212,9 +213,29 @@ func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, dat if err != nil { return fmt.Errorf("error cleaning path: %w", err) } - if data.At != nil { + _, err = filestore.WFS.Stat(ctx, zoneId, fileName) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("error getting blockfile info: %w", err) + } + var opts wshrpc.FileOptsType + var meta wshrpc.FileMeta + if data.Info != nil { + if data.Info.Opts != nil { + opts = *data.Info.Opts + } + if data.Info.Meta != nil { + meta = *data.Info.Meta + } + } + err := filestore.WFS.MakeFile(ctx, zoneId, fileName, meta, opts) + if err != nil { + return fmt.Errorf("error making blockfile: %w", err) + } + } + if data.At != nil && data.At.Offset >= 0 { err = filestore.WFS.WriteAt(ctx, zoneId, fileName, data.At.Offset, dataBuf) - if err == fs.ErrNotExist { + if errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("NOTFOUND: %w", err) } if err != nil { @@ -222,7 +243,7 @@ func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, dat } } else { err = filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf) - if err == fs.ErrNotExist { + if errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("NOTFOUND: %w", err) } if err != nil { @@ -292,8 +313,11 @@ func cleanPath(path string) (string, error) { if path == "" { return "", fmt.Errorf("path is empty") } - if strings.HasPrefix(path, "/") || strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { - return "", fmt.Errorf("path cannot start with /, ~, ., or ..") + if strings.HasPrefix(path, "/") { + path = path[1:] + } + if strings.HasPrefix(path, "~") || strings.HasPrefix(path, ".") || strings.HasPrefix(path, "..") { + return "", fmt.Errorf("wavefile path cannot start with ~, ., or ..") } var newParts []string for _, part := range strings.Split(path, "/") { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 862b3edaa..9515dd992 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -659,7 +659,7 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileD } dataSize := base64.StdEncoding.DecodedLen(len(data.Data64)) dataBytes := make([]byte, dataSize) - decodedLen, err := base64.StdEncoding.Decode(dataBytes, []byte(data.Data64)) + n, err := base64.StdEncoding.Decode(dataBytes, []byte(data.Data64)) if err != nil { return fmt.Errorf("cannot decode base64 data: %w", err) } @@ -672,14 +672,10 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileD fileSize = finfo.Size() } offset := fileSize - size := decodedLen if data.At != nil { if data.At.Offset > 0 { offset = min(data.At.Offset, fileSize) } - if data.At.Size > 0 { - size = min(data.At.Size, decodedLen) - } } file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, createMode) @@ -687,7 +683,7 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileD return fmt.Errorf("cannot open file %q: %w", path, err) } offsetWriter := io.NewOffsetWriter(file, offset) - n, err := offsetWriter.Write(dataBytes[:size]) + n, err = offsetWriter.Write(dataBytes[:n]) if err != nil { return fmt.Errorf("cannot write to file %q: %w", path, err) } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 27c96da21..64ad8cd90 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -367,11 +367,10 @@ type FileDataAt struct { } type FileData struct { - Info *FileInfo `json:"info,omitempty"` - Data64 string `json:"data64,omitempty"` - Entries []*FileInfo `json:"entries,omitempty"` - Opts *FileOptsType `json:"opts,omitempty"` - At *FileDataAt `json:"at,omitempty"` // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) + Info *FileInfo `json:"info,omitempty"` + Data64 string `json:"data64,omitempty"` + Entries []*FileInfo `json:"entries,omitempty"` + At *FileDataAt `json:"at,omitempty"` // if set, this turns read/write ops to ReadAt/WriteAt ops (len is only used for ReadAt) } type FileInfo struct { diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 6b2b66195..cd99be11f 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -280,16 +280,9 @@ func (ws *WshServer) ControllerAppendOutputCommand(ctx context.Context, data wsh return nil } -func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.FileCreateData) error { - var fileOpts wshrpc.FileOptsType - if data.Opts != nil { - fileOpts = *data.Opts - } - err := fileshare.PutFile(ctx, wshrpc.FileData{ - Info: &wshrpc.FileInfo{Path: data.Path}, - Data64: "", - Opts: &fileOpts, - }) +func (ws *WshServer) FileCreateCommand(ctx context.Context, data wshrpc.FileData) error { + data.Data64 = "" + err := fileshare.PutFile(ctx, data) if err != nil { return fmt.Errorf("error creating file: %w", err) } From 0fb579c394b30ebdee3a8d97b6240ff0fe1c6453 Mon Sep 17 00:00:00 2001 From: sawka Date: Tue, 21 Jan 2025 16:39:47 -0800 Subject: [PATCH 067/126] fix race condition around ctx.Done() and unregisterRpc() that can happen under extreme conditions. --- pkg/wshutil/wshrpc.go | 75 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/pkg/wshutil/wshrpc.go b/pkg/wshutil/wshrpc.go index 3d26397da..9948ff3ca 100644 --- a/pkg/wshutil/wshrpc.go +++ b/pkg/wshutil/wshrpc.go @@ -24,6 +24,7 @@ import ( const DefaultTimeoutMs = 5000 const RespChSize = 32 const DefaultMessageChSize = 32 +const CtxDoneChSize = 10 type ResponseFnType = func(any) error @@ -44,6 +45,7 @@ type WshRpc struct { clientId string InputCh chan []byte OutputCh chan []byte + CtxDoneCh chan string // for context cancellation, value is ResId RpcContext *atomic.Pointer[wshrpc.RpcContext] AuthToken string RpcMap map[string]*rpcData @@ -52,6 +54,7 @@ type WshRpc struct { ResponseHandlerMap map[string]*RpcResponseHandler // reqId => handler Debug bool DebugName string + ServerDone bool } type wshRpcContextKey struct{} @@ -206,6 +209,7 @@ func MakeWshRpc(inputCh chan []byte, outputCh chan []byte, rpcCtx wshrpc.RpcCont clientId: uuid.New().String(), InputCh: inputCh, OutputCh: outputCh, + CtxDoneCh: make(chan string, CtxDoneChSize), RpcMap: make(map[string]*rpcData), RpcContext: &atomic.Pointer[wshrpc.RpcContext]{}, EventListener: MakeEventListener(), @@ -328,12 +332,31 @@ func (w *WshRpc) handleRequest(req *RpcMessage) { func (w *WshRpc) runServer() { defer func() { panichandler.PanicHandler("wshrpc.runServer", recover()) + close(w.OutputCh) + w.setServerDone() }() - defer close(w.OutputCh) - for msgBytes := range w.InputCh { - if w.Debug { - log.Printf("[%s] received message: %s\n", w.DebugName, string(msgBytes)) +outer: + for { + var msgBytes []byte + var inputChMore bool + var resIdTimeout string + + select { + case msgBytes, inputChMore = <-w.InputCh: + if !inputChMore { + break outer + } + if w.Debug { + log.Printf("[%s] received message: %s\n", w.DebugName, string(msgBytes)) + } + case resIdTimeout = <-w.CtxDoneCh: + if w.Debug { + log.Printf("[%s] received request timeout: %s\n", w.DebugName, resIdTimeout) + } + w.unregisterRpc(resIdTimeout, fmt.Errorf("EC-TIME: timeout waiting for response")) + continue } + var msg RpcMessage err := json.Unmarshal(msgBytes, &msg) if err != nil { @@ -399,7 +422,7 @@ func (w *WshRpc) registerRpc(ctx context.Context, reqId string) chan *RpcMessage panichandler.PanicHandler("registerRpc:timeout", recover()) }() <-ctx.Done() - w.unregisterRpc(reqId, fmt.Errorf("EC-TIME: timeout waiting for response")) + w.retrySendTimeout(reqId) }() return rpcCh } @@ -651,6 +674,9 @@ func (handler *RpcResponseHandler) IsDone() bool { } func (w *WshRpc) SendComplexRequest(command string, data any, opts *wshrpc.RpcOpts) (rtnHandler *RpcRequestHandler, rtnErr error) { + if w.IsServerDone() { + return nil, errors.New("server is no longer running, cannot send new requests") + } if opts == nil { opts = &wshrpc.RpcOpts{} } @@ -690,3 +716,42 @@ func (w *WshRpc) SendComplexRequest(command string, data any, opts *wshrpc.RpcOp w.OutputCh <- barr return handler, nil } + +func (w *WshRpc) IsServerDone() bool { + w.Lock.Lock() + defer w.Lock.Unlock() + return w.ServerDone +} + +func (w *WshRpc) setServerDone() { + w.Lock.Lock() + defer w.Lock.Unlock() + w.ServerDone = true + close(w.CtxDoneCh) + for range w.CtxDoneCh { + // drain channel + } +} + +func (w *WshRpc) retrySendTimeout(resId string) { + for { + done := func() bool { + w.Lock.Lock() + defer w.Lock.Unlock() + if w.ServerDone { + return true + } + select { + case w.CtxDoneCh <- resId: + return true + default: + return false + } + }() + if done { + return + } + time.Sleep(100 * time.Millisecond) + } + +} From d37e7de27e345acece6a7bf85390b5affa09847f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:53:10 -0800 Subject: [PATCH 068/126] update filestore to use ints for size --- pkg/filestore/blockstore.go | 35 +++++++++++---------- pkg/filestore/blockstore_cache.go | 45 ++++++++++++++++----------- pkg/filestore/blockstore_test.go | 2 +- pkg/remote/fileshare/wavefs/wavefs.go | 2 +- 4 files changed, 47 insertions(+), 37 deletions(-) diff --git a/pkg/filestore/blockstore.go b/pkg/filestore/blockstore.go index d67dea93a..1e0f8e83c 100644 --- a/pkg/filestore/blockstore.go +++ b/pkg/filestore/blockstore.go @@ -12,6 +12,7 @@ import ( "fmt" "io/fs" "log" + "math" "sync" "sync/atomic" "time" @@ -42,7 +43,7 @@ const NoPartIdx = -1 var warningCount = &atomic.Int32{} var flushErrorCount = &atomic.Int32{} -var partDataSize int64 = DefaultPartDataSize // overridden in tests +var partDataSize int = DefaultPartDataSize // overridden in tests var stopFlush = &atomic.Bool{} var WFS *FileStore = &FileStore{ @@ -67,7 +68,7 @@ type WaveFile struct { // for circular files this is min(Size, MaxSize) func (f WaveFile) DataLength() int64 { if f.Opts.Circular { - return minInt64(f.Size, f.Opts.MaxSize) + return min(f.Size, f.Opts.MaxSize) } return f.Size } @@ -122,8 +123,9 @@ func (s *FileStore) MakeFile(ctx context.Context, zoneId string, name string, me return fmt.Errorf("circular file cannot be ijson") } if opts.Circular { - if opts.MaxSize%partDataSize != 0 { - opts.MaxSize = (opts.MaxSize/partDataSize + 1) * partDataSize + partDataSizeInt64 := int64(partDataSize) + if opts.MaxSize%partDataSizeInt64 != 0 { + opts.MaxSize = (opts.MaxSize/partDataSizeInt64 + 1) * partDataSizeInt64 } } if opts.IJsonBudget > 0 && !opts.IJson { @@ -249,7 +251,7 @@ func (s *FileStore) WriteAt(ctx context.Context, zoneId string, name string, off if offset > file.Size { return fmt.Errorf("offset is past the end of the file") } - partMap := file.computePartMap(offset, int64(len(data))) + partMap := file.computePartMap(offset, len(data)) incompleteParts := incompletePartsFromMap(partMap) err = entry.loadDataPartsIntoCache(ctx, incompleteParts) if err != nil { @@ -266,7 +268,7 @@ func (s *FileStore) AppendData(ctx context.Context, zoneId string, name string, if err != nil { return err } - partMap := entry.File.computePartMap(entry.File.Size, int64(len(data))) + partMap := entry.File.computePartMap(entry.File.Size, len(data)) incompleteParts := incompletePartsFromMap(partMap) if len(incompleteParts) > 0 { err = entry.loadDataPartsIntoCache(ctx, incompleteParts) @@ -332,7 +334,7 @@ func (s *FileStore) AppendIJson(ctx context.Context, zoneId string, name string, if !entry.File.Opts.IJson { return fmt.Errorf("file %s:%s is not an ijson file", zoneId, name) } - partMap := entry.File.computePartMap(entry.File.Size, int64(len(data))) + partMap := entry.File.computePartMap(entry.File.Size, len(data)) incompleteParts := incompletePartsFromMap(partMap) if len(incompleteParts) > 0 { err = entry.loadDataPartsIntoCache(ctx, incompleteParts) @@ -366,7 +368,7 @@ func (s *FileStore) GetAllZoneIds(ctx context.Context) ([]string, error) { // returns (offset, data, error) // we return the offset because the offset may have been adjusted if the size was too big (for circular files) -func (s *FileStore) ReadAt(ctx context.Context, zoneId string, name string, offset int64, size int64) (rtnOffset int64, rtnData []byte, rtnErr error) { +func (s *FileStore) ReadAt(ctx context.Context, zoneId string, name string, offset int64, size int) (rtnOffset int64, rtnData []byte, rtnErr error) { withLock(s, zoneId, name, func(entry *CacheEntry) error { rtnOffset, rtnData, rtnErr = entry.readAt(ctx, offset, size, false) return nil @@ -422,9 +424,9 @@ func (s *FileStore) FlushCache(ctx context.Context) (stats FlushStats, rtnErr er /////////////////////////////////// func (f *WaveFile) partIdxAtOffset(offset int64) int { - partIdx := int(offset / partDataSize) + partIdx := int(min(offset/int64(partDataSize), math.MaxInt)) if f.Opts.Circular { - maxPart := int(f.Opts.MaxSize / partDataSize) + maxPart := int(min(f.Opts.MaxSize/int64(partDataSize), math.MaxInt)) partIdx = partIdx % maxPart } return partIdx @@ -449,16 +451,17 @@ func getPartIdxsFromMap(partMap map[int]int) []int { } // returns a map of partIdx to amount of data to write to that part -func (file *WaveFile) computePartMap(startOffset int64, size int64) map[int]int { +func (file *WaveFile) computePartMap(startOffset int64, size int) map[int]int { partMap := make(map[int]int) - endOffset := startOffset + size - startFileOffset := startOffset - (startOffset % partDataSize) - for testOffset := startFileOffset; testOffset < endOffset; testOffset += partDataSize { + endOffset := startOffset + int64(size) + partDataSizeInt64 := int64(partDataSize) + startFileOffset := startOffset - (startOffset % partDataSizeInt64) + for testOffset := startFileOffset; testOffset < endOffset; testOffset += partDataSizeInt64 { partIdx := file.partIdxAtOffset(testOffset) partStartOffset := testOffset - partEndOffset := testOffset + partDataSize + partEndOffset := testOffset + partDataSizeInt64 partWriteStartOffset := 0 - partWriteEndOffset := int(partDataSize) + partWriteEndOffset := partDataSize if startOffset > partStartOffset && startOffset < partEndOffset { partWriteStartOffset = int(startOffset - partStartOffset) } diff --git a/pkg/filestore/blockstore_cache.go b/pkg/filestore/blockstore_cache.go index af8632022..dbb00628d 100644 --- a/pkg/filestore/blockstore_cache.go +++ b/pkg/filestore/blockstore_cache.go @@ -8,6 +8,7 @@ import ( "context" "fmt" "io/fs" + "math" "sync" "time" ) @@ -144,7 +145,7 @@ func withLockRtn[T any](s *FileStore, zoneId string, name string, fn func(*Cache } func (dce *DataCacheEntry) writeToPart(offset int64, data []byte) (int64, *DataCacheEntry) { - leftInPart := partDataSize - offset + leftInPart := int64(partDataSize) - offset toWrite := int64(len(data)) if toWrite > leftInPart { toWrite = leftInPart @@ -161,7 +162,7 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { entry.File.Size = 0 } if entry.File.Opts.Circular { - startCirFileOffset := entry.File.Size - entry.File.Opts.MaxSize + startCirFileOffset := entry.File.Size - int64(entry.File.Opts.MaxSize) if offset+int64(len(data)) <= startCirFileOffset { // write is before the start of the circular file return @@ -172,11 +173,12 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { data = data[truncateAmt:] offset += truncateAmt } - if int64(len(data)) > entry.File.Opts.MaxSize { + dataLen := int64(len(data)) + if dataLen > entry.File.Opts.MaxSize { // truncate data (from the front), update offset - truncateAmt := int64(len(data)) - entry.File.Opts.MaxSize + truncateAmt := int(max(dataLen-entry.File.Opts.MaxSize, 0)) data = data[truncateAmt:] - offset += truncateAmt + offset += int64(truncateAmt) } } endWriteOffset := offset + int64(len(data)) @@ -184,14 +186,19 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { entry.DataEntries = make(map[int]*DataCacheEntry) } for len(data) > 0 { - partIdx := int(offset / partDataSize) + partIdxI64 := offset / int64(partDataSize) + if partIdxI64 > math.MaxInt { + // too big + return + } + partIdx := int(partIdxI64) if entry.File.Opts.Circular { - maxPart := int(entry.File.Opts.MaxSize / partDataSize) + maxPart := int(min(entry.File.Opts.MaxSize/int64(partDataSize), math.MaxInt)) partIdx = partIdx % maxPart } - partOffset := offset % partDataSize + partOffset := int(offset % int64(partDataSize)) partData := entry.getOrCreateDataCacheEntry(partIdx) - nw, newDce := partData.writeToPart(partOffset, data) + nw, newDce := partData.writeToPart(int64(partOffset), data) entry.DataEntries[partIdx] = newDce data = data[nw:] offset += nw @@ -203,7 +210,7 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { } // returns (realOffset, data, error) -func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int64, readFull bool) (int64, []byte, error) { +func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int, readFull bool) (int64, []byte, error) { if offset < 0 { return 0, nil, fmt.Errorf("offset cannot be negative") } @@ -212,20 +219,20 @@ func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int64, r return 0, nil, err } if readFull { - size = file.Size - offset + size = int(min(file.Size-offset, math.MaxInt)) } - if offset+size > file.Size { - size = file.Size - offset + if offset+int64(size) > file.Size { + size = int(min(file.Size-offset, math.MaxInt)) } if file.Opts.Circular { realDataOffset := int64(0) - if file.Size > file.Opts.MaxSize { - realDataOffset = file.Size - file.Opts.MaxSize + if file.Size > int64(file.Opts.MaxSize) { + realDataOffset = file.Size - int64(file.Opts.MaxSize) } if offset < realDataOffset { truncateAmt := realDataOffset - offset offset += truncateAmt - size -= truncateAmt + size = int(max(int64(size)-truncateAmt, 0)) } if size <= 0 { return realDataOffset, nil, nil @@ -250,11 +257,11 @@ func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int64, r } else { partData = partDataEntry.Data[0:partDataSize] } - partOffset := curReadOffset % partDataSize - amtToRead := minInt64(partDataSize-partOffset, amtLeftToRead) + partOffset := int(curReadOffset % int64(partDataSize)) + amtToRead := min(partDataSize-partOffset, amtLeftToRead) rtnData = append(rtnData, partData[partOffset:partOffset+amtToRead]...) amtLeftToRead -= amtToRead - curReadOffset += amtToRead + curReadOffset += int64(amtToRead) } return offset, rtnData, nil } diff --git a/pkg/filestore/blockstore_test.go b/pkg/filestore/blockstore_test.go index 42fc7a343..2cda20647 100644 --- a/pkg/filestore/blockstore_test.go +++ b/pkg/filestore/blockstore_test.go @@ -306,7 +306,7 @@ func checkFileByteCount(t *testing.T, ctx context.Context, zoneId string, name s } func checkFileDataAt(t *testing.T, ctx context.Context, zoneId string, name string, offset int64, data string) { - _, rdata, err := WFS.ReadAt(ctx, zoneId, name, offset, int64(len(data))) + _, rdata, err := WFS.ReadAt(ctx, zoneId, name, offset, len(data)) if err != nil { t.Errorf("error reading data for file %q: %v", name, err) return diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 1aaca37a1..5382d7bbf 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -69,7 +69,7 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w return nil, fmt.Errorf("error cleaning path: %w", err) } if data.At != nil { - _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, int64(data.At.Size)) + _, dataBuf, err := filestore.WFS.ReadAt(ctx, zoneId, fileName, data.At.Offset, data.At.Size) if err == nil { return &wshrpc.FileData{Info: data.Info, Data64: base64.StdEncoding.EncodeToString(dataBuf)}, nil } else if errors.Is(err, fs.ErrNotExist) { From 91e80b3277366dab72bd5864d4a83c36751bddb6 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:55:06 -0800 Subject: [PATCH 069/126] Potential fix for code scanning alert no. 72: Arbitrary file access during archive extraction ("Zip Slip") Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- pkg/wshrpc/wshremote/wshremote.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 9515dd992..d47641020 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -356,6 +356,11 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } return fmt.Errorf("cannot read tar stream: %w", err) } + // Check for directory traversal + if strings.Contains(next.Name, "..") { + log.Printf("skipping file with unsafe path: %q\n", next.Name) + continue + } finfo := next.FileInfo() nextPath := filepath.Join(destPathCleaned, next.Name) destinfo, err = os.Stat(nextPath) From d9c1013a7bd7126eb7ee7fc8617053dcddd8ede3 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:55:58 -0800 Subject: [PATCH 070/126] Update pkg/remote/awsconn/awsconn.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/awsconn/awsconn.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index 5e8396dd8..261b604ca 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -106,6 +106,9 @@ func ParseProfiles() map[string]struct{} { f, err = ini.Load(fname) if err != nil { log.Printf("error reading aws credentials file: %v", err) + if profiles == nil { + profiles = make(map[string]struct{}) + } return profiles } for _, v := range f.Sections() { From 20e393cd13bff73f93d2329bf89e7d912fbf5745 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:56:12 -0800 Subject: [PATCH 071/126] Update pkg/remote/awsconn/awsconn.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/awsconn/awsconn.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index 261b604ca..2f49c1e53 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -51,12 +51,12 @@ func GetConfig(ctx context.Context, profile string) (*aws.Config, error) { if configfilepath != "" { log.Printf("configfilepath: %s", configfilepath) optfns = append(optfns, config.WithSharedConfigFiles([]string{configfilepath})) - tempfiles[profile] = configfilepath + tempfiles[profile+"_config"] = configfilepath } if credentialsfilepath != "" { log.Printf("credentialsfilepath: %s", credentialsfilepath) optfns = append(optfns, config.WithSharedCredentialsFiles([]string{credentialsfilepath})) - tempfiles[profile] = credentialsfilepath + tempfiles[profile+"_credentials"] = credentialsfilepath } } trimmedProfile := strings.TrimPrefix(profile, ProfilePrefix) From 74e8f60af5ae966dd1b317fba7029bb7bc98fd91 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:56:28 -0800 Subject: [PATCH 072/126] Update pkg/remote/awsconn/awsconn.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/awsconn/awsconn.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index 2f49c1e53..655e1fd17 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -78,7 +78,10 @@ func getTempFileFromConfig(config waveobj.MetaMapType, key string, profile strin if err != nil { return "", fmt.Errorf("error creating temp file: %v", err) } - tempfile.WriteString(awsConfig) + _, err = tempfile.WriteString(awsConfig) + if err != nil { + return "", fmt.Errorf("error writing to temp file: %v", err) + } return tempfile.Name(), nil } } From 8541f72668e00f2f96c8fceeb2f19a1912533db0 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:57:22 -0800 Subject: [PATCH 073/126] Update pkg/util/iochan/iochan.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/util/iochan/iochan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index dbf5b15c8..d2aaf7086 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -75,7 +75,7 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn } if _, err := w.Write(resp.Response); err != nil { log.Printf("WriterChan: write error: %v", err) - errCallback(resp.Error) + errCallback(err) return } else { // log.Printf("WriterChan: wrote %d bytes", n) From d7e8b47dbdb8e75a2bc3c402ceaece150cad5362 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:57:31 -0800 Subject: [PATCH 074/126] Update pkg/util/iochan/iochan.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/util/iochan/iochan.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index d2aaf7086..4145837fe 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -86,7 +86,5 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn } func drainChannel(ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { - for range ch { - <-ch - } + for range ch {} } From 935595046306e5afc42b1133404cefffe9a52cad Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 16:57:57 -0800 Subject: [PATCH 075/126] Update pkg/remote/fileshare/s3fs/s3fs.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/fileshare/s3fs/s3fs.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index a57cade8c..637af9d58 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -69,10 +69,12 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o var entries []*wshrpc.FileInfo for _, bucket := range buckets { log.Printf("bucket: %v", *bucket.Name) - entries = append(entries, &wshrpc.FileInfo{ - Path: *bucket.Name, - IsDir: true, - }) + if bucket.Name != nil { + entries = append(entries, &wshrpc.FileInfo{ + Path: *bucket.Name, + IsDir: true, + }) + } } return entries, nil } From 27d26c2f7cae2d81c499d492cdae27fc4800325e Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 17:05:14 -0800 Subject: [PATCH 076/126] Update pkg/wshrpc/wshrpctypes.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/wshrpc/wshrpctypes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 64ad8cd90..5f0dec399 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -21,7 +21,7 @@ import ( const ( // MaxFileSize is the maximum file size that can be read - MaxFileSize = 50 * 1024 * 1024 // 10M + MaxFileSize = 50 * 1024 * 1024 // 50M // MaxDirSize is the maximum number of entries that can be read in a directory MaxDirSize = 1024 // FileChunkSize is the size of the file chunk to read From d38c3aa339e1cd7cf7a65c04147a432a939e42ab Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 17:10:41 -0800 Subject: [PATCH 077/126] use time.Duration for timeouts --- cmd/wsh/cmd/wshcmd-file-util.go | 5 +++-- cmd/wsh/cmd/wshcmd-file.go | 6 +++--- pkg/remote/fileshare/wshfs/wshfs.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 4 ---- pkg/wshrpc/wshrpctypes.go | 15 ++++++++------- pkg/wshutil/wshrpc.go | 24 ++++++++++++------------ 6 files changed, 27 insertions(+), 29 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 6be5732e4..201aa080a 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -9,6 +9,7 @@ import ( "io" "io/fs" "strings" + "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/util/fileutil" @@ -80,7 +81,7 @@ func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { appendData := fileData appendData.Data64 = base64.StdEncoding.EncodeToString(chunk) - err = wshclient.FileAppendCommand(RpcClient, appendData, &wshrpc.RpcOpts{Timeout: fileTimeout}) + err = wshclient.FileAppendCommand(RpcClient, appendData, &wshrpc.RpcOpts{Timeout: time.Duration(fileTimeout)}) if err != nil { return fmt.Errorf("appending chunk to file: %w", err) } @@ -105,7 +106,7 @@ func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) } // Read the chunk - data, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) + data, err := wshclient.FileReadCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: time.Duration(fileTimeout)}) if err != nil { return fmt.Errorf("reading chunk at offset %d: %w", offset, err) } diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 06982a2df..ff7ed398e 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -33,7 +33,7 @@ const ( WaveFilePrefix = "wavefile://" DefaultFileTimeout = 5000 - TimeoutYear = int(365 * 24 * time.Hour) // 1 year + TimeoutYear = 365 * 24 * time.Hour // 1 year UriHelpText = ` @@ -321,7 +321,7 @@ func fileAppendRun(cmd *cobra.Command, args []string) error { if buf.Len() >= 8192 { // 8KB batch size fileData.Data64 = base64.StdEncoding.EncodeToString(buf.Bytes()) - err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) + err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: time.Duration(fileTimeout)}) if err != nil { return fmt.Errorf("appending to file: %w", err) } @@ -332,7 +332,7 @@ func fileAppendRun(cmd *cobra.Command, args []string) error { if buf.Len() > 0 { fileData.Data64 = base64.StdEncoding.EncodeToString(buf.Bytes()) - err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: fileTimeout}) + err = wshclient.FileAppendCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: time.Duration(fileTimeout)}) if err != nil { return fmt.Errorf("appending to file: %w", err) } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 62a07f4af..8f37deccf 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -20,7 +20,7 @@ import ( ) const ( - ThirtySeconds = int(30 * time.Second) + ThirtySeconds = 30 * time.Second ) // This needs to be set by whoever initializes the client, either main-server or wshcmd-connserver diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index d47641020..c3adeae4b 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -26,10 +26,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) -const ( - OneYear = int(time.Hour * 24 * 365) -) - type ServerImpl struct { LogWriter io.Writer } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 5f0dec399..5535eca00 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -11,6 +11,7 @@ import ( "log" "os" "reflect" + "time" "github.com/wavetermdev/waveterm/pkg/ijson" "github.com/wavetermdev/waveterm/pkg/vdom" @@ -233,9 +234,9 @@ type WshServerCommandMeta struct { } type RpcOpts struct { - Timeout int `json:"timeout,omitempty"` - NoResponse bool `json:"noresponse,omitempty"` - Route string `json:"route,omitempty"` + Timeout time.Duration `json:"timeout,omitempty"` + NoResponse bool `json:"noresponse,omitempty"` + Route string `json:"route,omitempty"` StreamCancelFn func() `json:"-"` // this is an *output* parameter, set by the handler } @@ -508,10 +509,10 @@ type CommandRemoteStreamTarData struct { } type FileCopyOpts struct { - Overwrite bool `json:"overwrite,omitempty"` - Recursive bool `json:"recursive,omitempty"` - Merge bool `json:"merge,omitempty"` - Timeout int `json:"timeout,omitempty"` + Overwrite bool `json:"overwrite,omitempty"` + Recursive bool `json:"recursive,omitempty"` + Merge bool `json:"merge,omitempty"` + Timeout time.Duration `json:"timeout,omitempty"` } type CommandRemoteStreamFileData struct { diff --git a/pkg/wshutil/wshrpc.go b/pkg/wshutil/wshrpc.go index 9948ff3ca..d67eedb0b 100644 --- a/pkg/wshutil/wshrpc.go +++ b/pkg/wshutil/wshrpc.go @@ -110,18 +110,18 @@ func (w *WshRpc) RecvRpcMessage() ([]byte, bool) { } type RpcMessage struct { - Command string `json:"command,omitempty"` - ReqId string `json:"reqid,omitempty"` - ResId string `json:"resid,omitempty"` - Timeout int `json:"timeout,omitempty"` - Route string `json:"route,omitempty"` // to route/forward requests to alternate servers - AuthToken string `json:"authtoken,omitempty"` // needed for routing unauthenticated requests (WshRpcMultiProxy) - Source string `json:"source,omitempty"` // source route id - Cont bool `json:"cont,omitempty"` // flag if additional requests/responses are forthcoming - Cancel bool `json:"cancel,omitempty"` // used to cancel a streaming request or response (sent from the side that is not streaming) - Error string `json:"error,omitempty"` - DataType string `json:"datatype,omitempty"` - Data any `json:"data,omitempty"` + Command string `json:"command,omitempty"` + ReqId string `json:"reqid,omitempty"` + ResId string `json:"resid,omitempty"` + Timeout time.Duration `json:"timeout,omitempty"` + Route string `json:"route,omitempty"` // to route/forward requests to alternate servers + AuthToken string `json:"authtoken,omitempty"` // needed for routing unauthenticated requests (WshRpcMultiProxy) + Source string `json:"source,omitempty"` // source route id + Cont bool `json:"cont,omitempty"` // flag if additional requests/responses are forthcoming + Cancel bool `json:"cancel,omitempty"` // used to cancel a streaming request or response (sent from the side that is not streaming) + Error string `json:"error,omitempty"` + DataType string `json:"datatype,omitempty"` + Data any `json:"data,omitempty"` } func (r *RpcMessage) IsRpcRequest() bool { From 3a31a0e21ae9a696d26a4b1524c87f26fe02b499 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 17:13:03 -0800 Subject: [PATCH 078/126] Update pkg/remote/fileshare/wavefs/wavefs.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/remote/fileshare/wavefs/wavefs.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 5382d7bbf..c204b224c 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -39,20 +39,19 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, ch <- wshutil.RespErr[wshrpc.FileData](err) return } - for { - if ctx.Err() != nil { - ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + if ctx.Err() != nil { + ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + return + } + dataLen := len(rtnData.Data64) + if !rtnData.Info.IsDir { + for i := 0; i < dataLen; i += wshrpc.FileChunkSize { + dataEnd := min(i+wshrpc.FileChunkSize, dataLen) + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: rtnData.Data64[i:dataEnd], Info: rtnData.Info, At: &wshrpc.FileDataAt{Offset: int64(i), Size: dataEnd - i}}} } - dataLen := len(rtnData.Data64) - if !rtnData.Info.IsDir { - for i := 0; i < dataLen; i += wshrpc.FileChunkSize { - dataEnd := min(i+wshrpc.FileChunkSize, dataLen) - ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: rtnData.Data64[i:dataEnd], Info: rtnData.Info, At: &wshrpc.FileDataAt{Offset: int64(i), Size: dataEnd - i}}} - } - } else { - for i := 0; i < len(rtnData.Entries); i += wshrpc.DirChunkSize { - ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: rtnData.Entries[i:min(i+wshrpc.DirChunkSize, len(rtnData.Entries))], Info: rtnData.Info}} - } + } else { + for i := 0; i < len(rtnData.Entries); i += wshrpc.DirChunkSize { + ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: rtnData.Entries[i:min(i+wshrpc.DirChunkSize, len(rtnData.Entries))], Info: rtnData.Info}} } } }() From a94014663b4917ab4cc76c8d82bdf570b3497002 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 17:14:47 -0800 Subject: [PATCH 079/126] fix readstream for loops --- pkg/remote/fileshare/wavefs/wavefs.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index c204b224c..c95cf5e95 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -39,18 +39,22 @@ func (c WaveClient) ReadStream(ctx context.Context, conn *connparse.Connection, ch <- wshutil.RespErr[wshrpc.FileData](err) return } - if ctx.Err() != nil { - ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) - return - } dataLen := len(rtnData.Data64) if !rtnData.Info.IsDir { for i := 0; i < dataLen; i += wshrpc.FileChunkSize { + if ctx.Err() != nil { + ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + return + } dataEnd := min(i+wshrpc.FileChunkSize, dataLen) ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Data64: rtnData.Data64[i:dataEnd], Info: rtnData.Info, At: &wshrpc.FileDataAt{Offset: int64(i), Size: dataEnd - i}}} } } else { for i := 0; i < len(rtnData.Entries); i += wshrpc.DirChunkSize { + if ctx.Err() != nil { + ch <- wshutil.RespErr[wshrpc.FileData](ctx.Err()) + return + } ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: wshrpc.FileData{Entries: rtnData.Entries[i:min(i+wshrpc.DirChunkSize, len(rtnData.Entries))], Info: rtnData.Info}} } } From a682ed6ea31dc69fb588fa3c4ac3c6241b4665c5 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 17:15:57 -0800 Subject: [PATCH 080/126] revert changes from rpc pr --- cmd/wsh/cmd/wshcmd-file.go | 8 ++++---- pkg/remote/fileshare/fileshare.go | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index ff7ed398e..3ddeb92e2 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -88,10 +88,10 @@ func init() { fileCmd.AddCommand(fileRmCmd) fileCmd.AddCommand(fileInfoCmd) fileCmd.AddCommand(fileAppendCmd) - // fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") - // fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") - // fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") - // fileCmd.AddCommand(fileCpCmd) + fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") + fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") + fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") + fileCmd.AddCommand(fileCpCmd) } type waveFileRef struct { diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 3693b5688..f8a80d8ec 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -5,8 +5,10 @@ import ( "fmt" "log" + "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/s3fs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -29,14 +31,12 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS conntype := conn.GetType() log.Printf("CreateFileShareClient: conntype=%s", conntype) if conntype == connparse.ConnectionTypeS3 { - // TODO: enable s3fs - return nil, nil - // config, err := awsconn.GetConfig(ctx, connection) - // if err != nil { - // log.Printf("error getting aws config: %v", err) - // return nil, nil - // } - // return s3fs.NewS3Client(config), conn + config, err := awsconn.GetConfig(ctx, connection) + if err != nil { + log.Printf("error getting aws config: %v", err) + return nil, nil + } + return s3fs.NewS3Client(config), conn } else if conntype == connparse.ConnectionTypeWave { return wavefs.NewWaveClient(), conn } else if conntype == connparse.ConnectionTypeWsh { From a429c6fecb1a6b6e1f5112d57a1cd39f946aae3f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 21 Jan 2025 17:23:54 -0800 Subject: [PATCH 081/126] save --- pkg/remote/awsconn/awsconn.go | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/pkg/remote/awsconn/awsconn.go b/pkg/remote/awsconn/awsconn.go index 655e1fd17..8f7731148 100644 --- a/pkg/remote/awsconn/awsconn.go +++ b/pkg/remote/awsconn/awsconn.go @@ -121,28 +121,13 @@ func ParseProfiles() map[string]struct{} { } func ListBuckets(ctx context.Context, client *s3.Client) ([]types.Bucket, error) { - var err error - var output *s3.ListBucketsOutput - var buckets []types.Bucket - region := client.Options().Region - bucketPaginator := s3.NewListBucketsPaginator(client, &s3.ListBucketsInput{BucketRegion: ®ion}) - for bucketPaginator.HasMorePages() { - output, err = bucketPaginator.NextPage(ctx) - log.Printf("output: %v", output) - if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) && apiErr.ErrorCode() == "AccessDenied" { - fmt.Println("You don't have permission to list buckets for this account.") - err = apiErr - } else { - return nil, fmt.Errorf("Couldn't list buckets for your account. Here's why: %v\n", err) - } - break - } - if output == nil { - break + output, err := client.ListBuckets(ctx, &s3.ListBucketsInput{}) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + return nil, fmt.Errorf("error listing buckets: %v", apiErr) } - buckets = append(buckets, output.Buckets...) + return nil, fmt.Errorf("error listing buckets: %v", err) } - return buckets, nil + return output.Buckets, nil } From 0da6d146e994b92bf54dcb5daf412608a4426f57 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 22 Jan 2025 16:03:28 -0800 Subject: [PATCH 082/126] fix merge errors --- .../app/view/preview/directorypreview.tsx | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/frontend/app/view/preview/directorypreview.tsx b/frontend/app/view/preview/directorypreview.tsx index 5c2453c40..14113ee8f 100644 --- a/frontend/app/view/preview/directorypreview.tsx +++ b/frontend/app/view/preview/directorypreview.tsx @@ -295,8 +295,13 @@ function DirectoryTable({ newPath = path.substring(0, lastInstance) + newName; console.log(`replacing ${fileName} with ${newName}: ${path}`); fireAndForget(async () => { - const connection = await globalStore.get(model.connection); - await FileService.Rename(connection, path, newPath); + await RpcApi.FileMoveCommand(TabRpcClient, { + srcuri: await model.formatRemoteUri(path), + desturi: await model.formatRemoteUri(newPath), + opts: { + recursive: true, + }, + }); model.refreshCallback(); }); } @@ -620,7 +625,11 @@ function TableBody({ label: "Delete", click: () => { fireAndForget(async () => { - await FileService.DeleteFile(conn, finfo.path).catch((e) => console.log(e)); + await RpcApi.FileDeleteCommand(TabRpcClient, { + info: { + path: await model.formatRemoteUri(finfo.path), + }, + }).catch((e) => console.log(e)); setRefreshVersion((current) => current + 1); }); }, @@ -833,8 +842,11 @@ function DirectoryPreview({ model }: DirectoryPreviewProps) { onSave: (newName: string) => { console.log(`newDirectory: ${newName}`); fireAndForget(async () => { - const connection = await globalStore.get(model.connection); - await FileService.Mkdir(connection, `${dirPath}/${newName}`); + await RpcApi.FileMkdirCommand(TabRpcClient, { + info: { + path: await model.formatRemoteUri(`${dirPath}/${newName}`), + }, + }); model.refreshCallback(); }); setEntryManagerProps(undefined); From f4077e079016983f2a33e3e5788014699843f038 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 22 Jan 2025 16:03:29 -0800 Subject: [PATCH 083/126] fix merge errors --- cmd/wsh/cmd/wshcmd-file-util.go | 9 --- cmd/wsh/cmd/wshcmd-file.go | 58 +----------------- frontend/app/store/wshclientapi.ts | 95 +++++++++--------------------- pkg/filestore/blockstore.go | 4 +- pkg/filestore/blockstore_test.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 28 +-------- pkg/wshutil/wshrpc.go | 1 - 7 files changed, 31 insertions(+), 166 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file-util.go b/cmd/wsh/cmd/wshcmd-file-util.go index 2dcc59c64..432cc1b1f 100644 --- a/cmd/wsh/cmd/wshcmd-file-util.go +++ b/cmd/wsh/cmd/wshcmd-file-util.go @@ -9,11 +9,7 @@ import ( "io" "io/fs" "strings" - "time" - "github.com/wavetermdev/waveterm/pkg/remote/connparse" - "github.com/wavetermdev/waveterm/pkg/util/fileutil" - "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" @@ -31,12 +27,10 @@ func convertNotFoundErr(err error) error { return err } -func ensureFile(origName string, fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { func ensureFile(origName string, fileData wshrpc.FileData) (*wshrpc.FileInfo, error) { info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { - err = wshclient.FileCreateCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = wshclient.FileCreateCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) if err != nil { return nil, fmt.Errorf("creating file: %w", err) @@ -95,7 +89,6 @@ func streamWriteToFile(fileData wshrpc.FileData, reader io.Reader) error { return nil } -func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) error { func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) error { const chunkSize = 32 * 1024 // 32KB chunks for offset := int64(0); offset < size; offset += chunkSize { @@ -106,7 +99,6 @@ func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) } // Set up the ReadAt request - fileData.At = &wshrpc.FileDataAt{ fileData.At = &wshrpc.FileDataAt{ Offset: offset, Size: length, @@ -120,7 +112,6 @@ func streamReadFromFile(fileData wshrpc.FileData, size int64, writer io.Writer) // Decode and write the chunk chunk, err := base64.StdEncoding.DecodeString(data.Data64) - chunk, err := base64.StdEncoding.DecodeString(data.Data64) if err != nil { return fmt.Errorf("decoding chunk at offset %d: %w", offset, err) } diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index c5dfd8094..a417afb0b 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -11,18 +11,17 @@ import ( "io" "io/fs" "log" - "log" "os" "path" "path/filepath" "strings" "text/tabwriter" - "text/tabwriter" "time" "github.com/spf13/cobra" "github.com/wavetermdev/waveterm/pkg/util/colprint" "github.com/wavetermdev/waveterm/pkg/util/utilfn" + "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "golang.org/x/term" @@ -125,11 +124,6 @@ func resolveWaveFile(ref *waveFileRef) (*waveobj.ORef, error) { } var fileListCmd = &cobra.Command{ - Use: "ls [uri]", - Aliases: []string{"list"}, - Short: "list files", - Long: "List files in a directory. By default, lists files in the current directory." + UriHelpText, - Example: " wsh file ls wsh://user@ec2/home/user/\n wsh file ls wavefile://client/configs/", Use: "ls [uri]", Aliases: []string{"list"}, Short: "list files", @@ -140,10 +134,6 @@ var fileListCmd = &cobra.Command{ } var fileCatCmd = &cobra.Command{ - Use: "cat [uri]", - Short: "display contents of a file", - Long: "Display the contents of a file." + UriHelpText, - Example: " wsh file cat wsh://user@ec2/home/user/config.txt\n wsh file cat wavefile://client/settings.json", Use: "cat [uri]", Short: "display contents of a file", Long: "Display the contents of a file." + UriHelpText, @@ -154,23 +144,16 @@ var fileCatCmd = &cobra.Command{ } var fileInfoCmd = &cobra.Command{ - Use: "info [uri]", Use: "info [uri]", Short: "show wave file information", Long: "Show information about a file." + UriHelpText, Example: " wsh file info wsh://user@ec2/home/user/config.txt\n wsh file info wavefile://client/settings.json", - Long: "Show information about a file." + UriHelpText, - Example: " wsh file info wsh://user@ec2/home/user/config.txt\n wsh file info wavefile://client/settings.json", Args: cobra.ExactArgs(1), RunE: activityWrap("file", fileInfoRun), PreRunE: preRunSetupRpcClient, } var fileRmCmd = &cobra.Command{ - Use: "rm [uri]", - Short: "remove a file", - Long: "Remove a file." + UriHelpText, - Example: " wsh file rm wsh://user@ec2/home/user/config.txt\n wsh file rm wavefile://client/settings.json", Use: "rm [uri]", Short: "remove a file", Long: "Remove a file." + UriHelpText, @@ -223,7 +206,6 @@ var fileMvCmd = &cobra.Command{ } func fileCatRun(cmd *cobra.Command, args []string) error { - path, err := fixRelativePaths(args[0]) path, err := fixRelativePaths(args[0]) if err != nil { return err @@ -231,22 +213,17 @@ func fileCatRun(cmd *cobra.Command, args []string) error { fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ Path: path}} - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} // Get file info first to check existence and get size info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: 2000}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { return fmt.Errorf("%s: no such file", path) - return fmt.Errorf("%s: no such file", path) } if err != nil { return fmt.Errorf("getting file info: %w", err) } - err = streamReadFromFile(fileData, info.Size, os.Stdout) err = streamReadFromFile(fileData, info.Size, os.Stdout) if err != nil { return fmt.Errorf("reading file: %w", err) @@ -256,7 +233,6 @@ func fileCatRun(cmd *cobra.Command, args []string) error { } func fileInfoRun(cmd *cobra.Command, args []string) error { - path, err := fixRelativePaths(args[0]) path, err := fixRelativePaths(args[0]) if err != nil { return err @@ -264,29 +240,16 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ Path: path}} - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} info, err := wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { return fmt.Errorf("%s: no such file", path) - return fmt.Errorf("%s: no such file", path) } if err != nil { return fmt.Errorf("getting file info: %w", err) } - WriteStdout("name:\t%s\n", info.Name) - if info.Mode != 0 { - WriteStdout("mode:\t%s\n", info.Mode.String()) - } - WriteStdout("mtime:\t%s\n", time.Unix(info.ModTime/1000, 0).Format(time.DateTime)) - if !info.IsDir { - WriteStdout("size:\t%d\n", info.Size) - } - if info.Meta != nil && len(*info.Meta) > 0 { WriteStdout("name:\t%s\n", info.Name) if info.Mode != 0 { WriteStdout("mode:\t%s\n", info.Mode.String()) @@ -297,8 +260,6 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { } if info.Meta != nil && len(*info.Meta) > 0 { WriteStdout("metadata:\n") - for k, v := range *info.Meta { - WriteStdout("\t\t\t%s: %v\n", k, v) for k, v := range *info.Meta { WriteStdout("\t\t\t%s: %v\n", k, v) } @@ -307,7 +268,6 @@ func fileInfoRun(cmd *cobra.Command, args []string) error { } func fileRmRun(cmd *cobra.Command, args []string) error { - path, err := fixRelativePaths(args[0]) path, err := fixRelativePaths(args[0]) if err != nil { return err @@ -315,15 +275,11 @@ func fileRmRun(cmd *cobra.Command, args []string) error { fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ Path: path}} - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} _, err = wshclient.FileInfoCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) err = convertNotFoundErr(err) if err == fs.ErrNotExist { return fmt.Errorf("%s: no such file", path) - return fmt.Errorf("%s: no such file", path) } if err != nil { return fmt.Errorf("getting file info: %w", err) @@ -338,7 +294,6 @@ func fileRmRun(cmd *cobra.Command, args []string) error { } func fileWriteRun(cmd *cobra.Command, args []string) error { - path, err := fixRelativePaths(args[0]) path, err := fixRelativePaths(args[0]) if err != nil { return err @@ -346,17 +301,12 @@ func fileWriteRun(cmd *cobra.Command, args []string) error { fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ Path: path}} - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} - _, err = ensureFile(path, fileData) _, err = ensureFile(path, fileData) if err != nil { return err } - err = streamWriteToFile(fileData, WrappedStdin) err = streamWriteToFile(fileData, WrappedStdin) if err != nil { return fmt.Errorf("writing file: %w", err) @@ -366,7 +316,6 @@ func fileWriteRun(cmd *cobra.Command, args []string) error { } func fileAppendRun(cmd *cobra.Command, args []string) error { - path, err := fixRelativePaths(args[0]) path, err := fixRelativePaths(args[0]) if err != nil { return err @@ -374,11 +323,7 @@ func fileAppendRun(cmd *cobra.Command, args []string) error { fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ Path: path}} - fileData := wshrpc.FileData{ - Info: &wshrpc.FileInfo{ - Path: path}} - info, err := ensureFile(path, fileData) info, err := ensureFile(path, fileData) if err != nil { return err @@ -529,7 +474,6 @@ func filePrintColumns(filesChan <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRem numCols = 1 } - return colprint.PrintColumnsArray( return colprint.PrintColumnsArray( filesChan, numCols, diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index d31950fda..2822ac298 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -23,11 +23,7 @@ class RpcApiType { } // command "authenticatetoken" [call] - AuthenticateTokenCommand( - client: WshClient, - data: CommandAuthenticateTokenData, - opts?: RpcOpts - ): Promise { + AuthenticateTokenCommand(client: WshClient, data: CommandAuthenticateTokenData, opts?: RpcOpts): Promise { return client.wshRpcCall("authenticatetoken", data, opts); } @@ -72,11 +68,7 @@ class RpcApiType { } // command "controllerappendoutput" [call] - ControllerAppendOutputCommand( - client: WshClient, - data: CommandControllerAppendOutputData, - opts?: RpcOpts - ): Promise { + ControllerAppendOutputCommand(client: WshClient, data: CommandControllerAppendOutputData, opts?: RpcOpts): Promise { return client.wshRpcCall("controllerappendoutput", data, opts); } @@ -131,11 +123,7 @@ class RpcApiType { } // command "eventreadhistory" [call] - EventReadHistoryCommand( - client: WshClient, - data: CommandEventReadHistoryData, - opts?: RpcOpts - ): Promise { + EventReadHistoryCommand(client: WshClient, data: CommandEventReadHistoryData, opts?: RpcOpts): Promise { return client.wshRpcCall("eventreadhistory", data, opts); } @@ -195,25 +183,27 @@ class RpcApiType { } // command "fileliststream" [responsestream] - FileListStreamCommand( - client: WshClient, - data: FileListData, - opts?: RpcOpts - ): AsyncGenerator { + FileListStreamCommand(client: WshClient, data: FileListData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("fileliststream", data, opts); } + // command "filemkdir" [call] + FileMkdirCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { + return client.wshRpcCall("filemkdir", data, opts); + } + + // command "filemove" [call] + FileMoveCommand(client: WshClient, data: CommandFileCopyData, opts?: RpcOpts): Promise { + return client.wshRpcCall("filemove", data, opts); + } + // command "fileread" [call] FileReadCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { return client.wshRpcCall("fileread", data, opts); } // command "filestreamtar" [responsestream] - FileStreamTarCommand( - client: WshClient, - data: CommandRemoteStreamTarData, - opts?: RpcOpts - ): AsyncGenerator { + FileStreamTarCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("filestreamtar", data, opts); } @@ -303,11 +293,7 @@ class RpcApiType { } // command "remotelistentries" [responsestream] - RemoteListEntriesCommand( - client: WshClient, - data: CommandRemoteListEntriesData, - opts?: RpcOpts - ): AsyncGenerator { + RemoteListEntriesCommand(client: WshClient, data: CommandRemoteListEntriesData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotelistentries", data, opts); } @@ -317,25 +303,17 @@ class RpcApiType { } // command "remotestreamcpudata" [responsestream] - RemoteStreamCpuDataCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { + RemoteStreamCpuDataCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotestreamcpudata", null, opts); } // command "remotestreamfile" [responsestream] - RemoteStreamFileCommand( - client: WshClient, - data: CommandRemoteStreamFileData, - opts?: RpcOpts - ): AsyncGenerator { + RemoteStreamFileCommand(client: WshClient, data: CommandRemoteStreamFileData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotestreamfile", data, opts); } // command "remotetarstream" [responsestream] - RemoteTarStreamCommand( - client: WshClient, - data: CommandRemoteStreamTarData, - opts?: RpcOpts - ): AsyncGenerator { + RemoteTarStreamCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotetarstream", data, opts); } @@ -345,11 +323,7 @@ class RpcApiType { } // command "resolveids" [call] - ResolveIdsCommand( - client: WshClient, - data: CommandResolveIdsData, - opts?: RpcOpts - ): Promise { + ResolveIdsCommand(client: WshClient, data: CommandResolveIdsData, opts?: RpcOpts): Promise { return client.wshRpcCall("resolveids", data, opts); } @@ -389,25 +363,17 @@ class RpcApiType { } // command "streamcpudata" [responsestream] - StreamCpuDataCommand( - client: WshClient, - data: CpuDataRequest, - opts?: RpcOpts - ): AsyncGenerator { + StreamCpuDataCommand(client: WshClient, data: CpuDataRequest, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("streamcpudata", data, opts); } // command "streamtest" [responsestream] - StreamTestCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { + StreamTestCommand(client: WshClient, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("streamtest", null, opts); } // command "streamwaveai" [responsestream] - StreamWaveAiCommand( - client: WshClient, - data: WaveAIStreamRequest, - opts?: RpcOpts - ): AsyncGenerator { + StreamWaveAiCommand(client: WshClient, data: WaveAIStreamRequest, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("streamwaveai", data, opts); } @@ -427,20 +393,12 @@ class RpcApiType { } // command "vdomrender" [responsestream] - VDomRenderCommand( - client: WshClient, - data: VDomFrontendUpdate, - opts?: RpcOpts - ): AsyncGenerator { + VDomRenderCommand(client: WshClient, data: VDomFrontendUpdate, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("vdomrender", data, opts); } // command "vdomurlrequest" [responsestream] - VDomUrlRequestCommand( - client: WshClient, - data: VDomUrlRequestData, - opts?: RpcOpts - ): AsyncGenerator { + VDomUrlRequestCommand(client: WshClient, data: VDomUrlRequestData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("vdomurlrequest", data, opts); } @@ -465,7 +423,7 @@ class RpcApiType { } // command "wshactivity" [call] - WshActivityCommand(client: WshClient, data: { [key: string]: number }, opts?: RpcOpts): Promise { + WshActivityCommand(client: WshClient, data: {[key: string]: number}, opts?: RpcOpts): Promise { return client.wshRpcCall("wshactivity", data, opts); } @@ -483,6 +441,7 @@ class RpcApiType { WslStatusCommand(client: WshClient, opts?: RpcOpts): Promise { return client.wshRpcCall("wslstatus", null, opts); } + } export const RpcApi = new RpcApiType(); diff --git a/pkg/filestore/blockstore.go b/pkg/filestore/blockstore.go index a38b3ed44..21cd2bf3d 100644 --- a/pkg/filestore/blockstore.go +++ b/pkg/filestore/blockstore.go @@ -373,7 +373,7 @@ func (s *FileStore) ReadAt(ctx context.Context, zoneId string, name string, offs return 0, nil, fmt.Errorf("size must be non-negative and less than MaxInt") } withLock(s, zoneId, name, func(entry *CacheEntry) error { - rtnOffset, rtnData, rtnErr = entry.readAt(ctx, offset, size, false) + rtnOffset, rtnData, rtnErr = entry.readAt(ctx, offset, int(size), false) return nil }) return @@ -424,8 +424,6 @@ func (s *FileStore) FlushCache(ctx context.Context) (stats FlushStats, rtnErr er return stats, nil } -/////////////////////////////////// - func (f *WaveFile) partIdxAtOffset(offset int64) int { partIdx := int(min(offset/int64(partDataSize), math.MaxInt)) if f.Opts.Circular { diff --git a/pkg/filestore/blockstore_test.go b/pkg/filestore/blockstore_test.go index 0845c519d..235242dc0 100644 --- a/pkg/filestore/blockstore_test.go +++ b/pkg/filestore/blockstore_test.go @@ -306,7 +306,7 @@ func checkFileByteCount(t *testing.T, ctx context.Context, zoneId string, name s } func checkFileDataAt(t *testing.T, ctx context.Context, zoneId string, name string, offset int64, data string) { - _, rdata, err := WFS.ReadAt(ctx, zoneId, name, offset, len(data)) + _, rdata, err := WFS.ReadAt(ctx, zoneId, name, offset, int64(len(data))) if err != nil { t.Errorf("error reading data for file %q: %v", name, err) return diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index cf19204d1..add1d4574 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -4,7 +4,6 @@ package wshremote import ( - "archive/tar" "archive/tar" "context" "encoding/base64" @@ -17,9 +16,8 @@ import ( "path/filepath" "strings" "time" - "time" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare" + "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/utilfn" @@ -117,7 +115,6 @@ func (impl *ServerImpl) remoteStreamFileDir(ctx context.Context, path string, by return nil } -func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType)) error { func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string, byteRange ByteRangeType, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType)) error { fd, err := os.Open(path) if err != nil { @@ -133,7 +130,6 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string filePos = byteRange.Start } buf := make([]byte, wshrpc.FileChunkSize) - buf := make([]byte, wshrpc.FileChunkSize) for { if ctx.Err() != nil { return ctx.Err() @@ -145,7 +141,6 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string } filePos += int64(n) dataCallback(nil, buf[:n], byteRange) - dataCallback(nil, buf[:n], byteRange) } if !byteRange.All && filePos >= byteRange.End { break @@ -160,7 +155,6 @@ func (impl *ServerImpl) remoteStreamFileRegular(ctx context.Context, path string return nil } -func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType)) error { func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrpc.CommandRemoteStreamFileData, dataCallback func(fileInfo []*wshrpc.FileInfo, data []byte, byteRange ByteRangeType)) error { byteRange, err := parseByteRange(data.ByteRange) if err != nil { @@ -175,11 +169,9 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp return fmt.Errorf("cannot stat file %q: %w", path, err) } dataCallback([]*wshrpc.FileInfo{finfo}, nil, byteRange) - dataCallback([]*wshrpc.FileInfo{finfo}, nil, byteRange) if finfo.NotFound { return nil } - if finfo.Size > wshrpc.MaxFileSize { if finfo.Size > wshrpc.MaxFileSize { return fmt.Errorf("file %q is too large to read, use /wave/stream-file", path) } @@ -190,8 +182,6 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp } } -func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc.CommandRemoteStreamFileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { - ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc.CommandRemoteStreamFileData) chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { ch := make(chan wshrpc.RespOrErrorUnion[wshrpc.FileData], 16) go func() { @@ -541,7 +531,6 @@ func (impl *ServerImpl) RemoteListEntriesCommand(ctx context.Context, data wshrp } func statToFileInfo(fullPath string, finfo fs.FileInfo, extended bool) *wshrpc.FileInfo { - mimeType := fileutil.DetectMimeType(fullPath, finfo, extended) mimeType := fileutil.DetectMimeType(fullPath, finfo, extended) rtn := &wshrpc.FileInfo{ Path: wavebase.ReplaceHomeDir(fullPath), @@ -554,16 +543,6 @@ func statToFileInfo(fullPath string, finfo fs.FileInfo, extended bool) *wshrpc.F IsDir: finfo.IsDir(), MimeType: mimeType, SupportsMkdir: true, - Path: wavebase.ReplaceHomeDir(fullPath), - Dir: computeDirPart(fullPath, finfo.IsDir()), - Name: finfo.Name(), - Size: finfo.Size(), - Mode: finfo.Mode(), - ModeStr: finfo.Mode().String(), - ModTime: finfo.ModTime().UnixMilli(), - IsDir: finfo.IsDir(), - MimeType: mimeType, - SupportsMkdir: true, } if finfo.IsDir() { rtn.Size = -1 @@ -734,15 +713,12 @@ func (impl *ServerImpl) RemoteMkdirCommand(ctx context.Context, path string) err return nil } -func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileData) error { - path, err := wavebase.ExpandHomeDir(data.Info.Path) func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileData) error { path, err := wavebase.ExpandHomeDir(data.Info.Path) if err != nil { return err } createMode := data.Info.Mode - createMode := data.Info.Mode if createMode == 0 { createMode = 0644 } @@ -786,10 +762,8 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileD } if err != nil { return fmt.Errorf("cannot write to file %q: %w", path, err) - return fmt.Errorf("cannot write to file %q: %w", path, err) } log.Printf("wrote %d bytes to file %q at offset %q\n", n, path, offset) - log.Printf("wrote %d bytes to file %q at offset %q\n", n, path, offset) return nil } diff --git a/pkg/wshutil/wshrpc.go b/pkg/wshutil/wshrpc.go index 4d1afcc70..84f9628b4 100644 --- a/pkg/wshutil/wshrpc.go +++ b/pkg/wshutil/wshrpc.go @@ -25,7 +25,6 @@ const DefaultTimeoutMs = 5000 const RespChSize = 32 const DefaultMessageChSize = 32 const CtxDoneChSize = 10 -const CtxDoneChSize = 10 type ResponseFnType = func(any) error From 532060cc3420e2c2a567948117b3ecfac4d1fd69 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 22 Jan 2025 16:05:12 -0800 Subject: [PATCH 084/126] more merge errors --- cmd/wsh/cmd/wshcmd-file.go | 13 --------- pkg/filestore/blockstore.go | 36 ++++++++++++------------- pkg/filestore/blockstore_cache.go | 45 +++++++++++++------------------ 3 files changed, 37 insertions(+), 57 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index a417afb0b..091dd74ac 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -21,7 +21,6 @@ import ( "github.com/spf13/cobra" "github.com/wavetermdev/waveterm/pkg/util/colprint" "github.com/wavetermdev/waveterm/pkg/util/utilfn" - "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "golang.org/x/term" @@ -105,24 +104,12 @@ func init() { fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") - fileCpCmd.Flags().BoolP("merge", "m", false, "merge directories") - fileCpCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") - fileCpCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") fileCmd.AddCommand(fileCpCmd) fileMvCmd.Flags().BoolP("recursive", "r", false, "move directories recursively") fileMvCmd.Flags().BoolP("force", "f", false, "force overwrite of existing files") fileCmd.AddCommand(fileMvCmd) } -type waveFileRef struct { - zoneId string - fileName string -} - -func resolveWaveFile(ref *waveFileRef) (*waveobj.ORef, error) { - return resolveSimpleId(ref.zoneId) -} - var fileListCmd = &cobra.Command{ Use: "ls [uri]", Aliases: []string{"list"}, diff --git a/pkg/filestore/blockstore.go b/pkg/filestore/blockstore.go index 21cd2bf3d..55ce70183 100644 --- a/pkg/filestore/blockstore.go +++ b/pkg/filestore/blockstore.go @@ -43,7 +43,7 @@ const NoPartIdx = -1 var warningCount = &atomic.Int32{} var flushErrorCount = &atomic.Int32{} -var partDataSize int = DefaultPartDataSize // overridden in tests +var partDataSize int64 = DefaultPartDataSize // overridden in tests var stopFlush = &atomic.Bool{} var WFS *FileStore = &FileStore{ @@ -68,7 +68,7 @@ type WaveFile struct { // for circular files this is min(Size, MaxSize) func (f WaveFile) DataLength() int64 { if f.Opts.Circular { - return min(f.Size, f.Opts.MaxSize) + return minInt64(f.Size, f.Opts.MaxSize) } return f.Size } @@ -123,9 +123,8 @@ func (s *FileStore) MakeFile(ctx context.Context, zoneId string, name string, me return fmt.Errorf("circular file cannot be ijson") } if opts.Circular { - partDataSizeInt64 := int64(partDataSize) - if opts.MaxSize%partDataSizeInt64 != 0 { - opts.MaxSize = (opts.MaxSize/partDataSizeInt64 + 1) * partDataSizeInt64 + if opts.MaxSize%partDataSize != 0 { + opts.MaxSize = (opts.MaxSize/partDataSize + 1) * partDataSize } } if opts.IJsonBudget > 0 && !opts.IJson { @@ -251,7 +250,7 @@ func (s *FileStore) WriteAt(ctx context.Context, zoneId string, name string, off if offset > file.Size { return fmt.Errorf("offset is past the end of the file") } - partMap := file.computePartMap(offset, len(data)) + partMap := file.computePartMap(offset, int64(len(data))) incompleteParts := incompletePartsFromMap(partMap) err = entry.loadDataPartsIntoCache(ctx, incompleteParts) if err != nil { @@ -268,7 +267,7 @@ func (s *FileStore) AppendData(ctx context.Context, zoneId string, name string, if err != nil { return err } - partMap := entry.File.computePartMap(entry.File.Size, len(data)) + partMap := entry.File.computePartMap(entry.File.Size, int64(len(data))) incompleteParts := incompletePartsFromMap(partMap) if len(incompleteParts) > 0 { err = entry.loadDataPartsIntoCache(ctx, incompleteParts) @@ -334,7 +333,7 @@ func (s *FileStore) AppendIJson(ctx context.Context, zoneId string, name string, if !entry.File.Opts.IJson { return fmt.Errorf("file %s:%s is not an ijson file", zoneId, name) } - partMap := entry.File.computePartMap(entry.File.Size, len(data)) + partMap := entry.File.computePartMap(entry.File.Size, int64(len(data))) incompleteParts := incompletePartsFromMap(partMap) if len(incompleteParts) > 0 { err = entry.loadDataPartsIntoCache(ctx, incompleteParts) @@ -373,7 +372,7 @@ func (s *FileStore) ReadAt(ctx context.Context, zoneId string, name string, offs return 0, nil, fmt.Errorf("size must be non-negative and less than MaxInt") } withLock(s, zoneId, name, func(entry *CacheEntry) error { - rtnOffset, rtnData, rtnErr = entry.readAt(ctx, offset, int(size), false) + rtnOffset, rtnData, rtnErr = entry.readAt(ctx, offset, size, false) return nil }) return @@ -424,10 +423,12 @@ func (s *FileStore) FlushCache(ctx context.Context) (stats FlushStats, rtnErr er return stats, nil } +/////////////////////////////////// + func (f *WaveFile) partIdxAtOffset(offset int64) int { - partIdx := int(min(offset/int64(partDataSize), math.MaxInt)) + partIdx := int(offset / partDataSize) if f.Opts.Circular { - maxPart := int(min(f.Opts.MaxSize/int64(partDataSize), math.MaxInt)) + maxPart := int(f.Opts.MaxSize / partDataSize) partIdx = partIdx % maxPart } return partIdx @@ -452,17 +453,16 @@ func getPartIdxsFromMap(partMap map[int]int) []int { } // returns a map of partIdx to amount of data to write to that part -func (file *WaveFile) computePartMap(startOffset int64, size int) map[int]int { +func (file *WaveFile) computePartMap(startOffset int64, size int64) map[int]int { partMap := make(map[int]int) - endOffset := startOffset + int64(size) - partDataSizeInt64 := int64(partDataSize) - startFileOffset := startOffset - (startOffset % partDataSizeInt64) - for testOffset := startFileOffset; testOffset < endOffset; testOffset += partDataSizeInt64 { + endOffset := startOffset + size + startFileOffset := startOffset - (startOffset % partDataSize) + for testOffset := startFileOffset; testOffset < endOffset; testOffset += partDataSize { partIdx := file.partIdxAtOffset(testOffset) partStartOffset := testOffset - partEndOffset := testOffset + partDataSizeInt64 + partEndOffset := testOffset + partDataSize partWriteStartOffset := 0 - partWriteEndOffset := partDataSize + partWriteEndOffset := int(partDataSize) if startOffset > partStartOffset && startOffset < partEndOffset { partWriteStartOffset = int(startOffset - partStartOffset) } diff --git a/pkg/filestore/blockstore_cache.go b/pkg/filestore/blockstore_cache.go index dbb00628d..af8632022 100644 --- a/pkg/filestore/blockstore_cache.go +++ b/pkg/filestore/blockstore_cache.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "io/fs" - "math" "sync" "time" ) @@ -145,7 +144,7 @@ func withLockRtn[T any](s *FileStore, zoneId string, name string, fn func(*Cache } func (dce *DataCacheEntry) writeToPart(offset int64, data []byte) (int64, *DataCacheEntry) { - leftInPart := int64(partDataSize) - offset + leftInPart := partDataSize - offset toWrite := int64(len(data)) if toWrite > leftInPart { toWrite = leftInPart @@ -162,7 +161,7 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { entry.File.Size = 0 } if entry.File.Opts.Circular { - startCirFileOffset := entry.File.Size - int64(entry.File.Opts.MaxSize) + startCirFileOffset := entry.File.Size - entry.File.Opts.MaxSize if offset+int64(len(data)) <= startCirFileOffset { // write is before the start of the circular file return @@ -173,12 +172,11 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { data = data[truncateAmt:] offset += truncateAmt } - dataLen := int64(len(data)) - if dataLen > entry.File.Opts.MaxSize { + if int64(len(data)) > entry.File.Opts.MaxSize { // truncate data (from the front), update offset - truncateAmt := int(max(dataLen-entry.File.Opts.MaxSize, 0)) + truncateAmt := int64(len(data)) - entry.File.Opts.MaxSize data = data[truncateAmt:] - offset += int64(truncateAmt) + offset += truncateAmt } } endWriteOffset := offset + int64(len(data)) @@ -186,19 +184,14 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { entry.DataEntries = make(map[int]*DataCacheEntry) } for len(data) > 0 { - partIdxI64 := offset / int64(partDataSize) - if partIdxI64 > math.MaxInt { - // too big - return - } - partIdx := int(partIdxI64) + partIdx := int(offset / partDataSize) if entry.File.Opts.Circular { - maxPart := int(min(entry.File.Opts.MaxSize/int64(partDataSize), math.MaxInt)) + maxPart := int(entry.File.Opts.MaxSize / partDataSize) partIdx = partIdx % maxPart } - partOffset := int(offset % int64(partDataSize)) + partOffset := offset % partDataSize partData := entry.getOrCreateDataCacheEntry(partIdx) - nw, newDce := partData.writeToPart(int64(partOffset), data) + nw, newDce := partData.writeToPart(partOffset, data) entry.DataEntries[partIdx] = newDce data = data[nw:] offset += nw @@ -210,7 +203,7 @@ func (entry *CacheEntry) writeAt(offset int64, data []byte, replace bool) { } // returns (realOffset, data, error) -func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int, readFull bool) (int64, []byte, error) { +func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int64, readFull bool) (int64, []byte, error) { if offset < 0 { return 0, nil, fmt.Errorf("offset cannot be negative") } @@ -219,20 +212,20 @@ func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int, rea return 0, nil, err } if readFull { - size = int(min(file.Size-offset, math.MaxInt)) + size = file.Size - offset } - if offset+int64(size) > file.Size { - size = int(min(file.Size-offset, math.MaxInt)) + if offset+size > file.Size { + size = file.Size - offset } if file.Opts.Circular { realDataOffset := int64(0) - if file.Size > int64(file.Opts.MaxSize) { - realDataOffset = file.Size - int64(file.Opts.MaxSize) + if file.Size > file.Opts.MaxSize { + realDataOffset = file.Size - file.Opts.MaxSize } if offset < realDataOffset { truncateAmt := realDataOffset - offset offset += truncateAmt - size = int(max(int64(size)-truncateAmt, 0)) + size -= truncateAmt } if size <= 0 { return realDataOffset, nil, nil @@ -257,11 +250,11 @@ func (entry *CacheEntry) readAt(ctx context.Context, offset int64, size int, rea } else { partData = partDataEntry.Data[0:partDataSize] } - partOffset := int(curReadOffset % int64(partDataSize)) - amtToRead := min(partDataSize-partOffset, amtLeftToRead) + partOffset := curReadOffset % partDataSize + amtToRead := minInt64(partDataSize-partOffset, amtLeftToRead) rtnData = append(rtnData, partData[partOffset:partOffset+amtToRead]...) amtLeftToRead -= amtToRead - curReadOffset += int64(amtToRead) + curReadOffset += amtToRead } return offset, rtnData, nil } From 4483554a49beef1d15a8080fecdbd12fa806e212 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 27 Jan 2025 17:39:15 -0800 Subject: [PATCH 085/126] save --- pkg/wshrpc/wshremote/wshremote.go | 195 +++++++++++++++--------------- 1 file changed, 100 insertions(+), 95 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 6e17ae806..a5edb00fa 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -18,6 +18,7 @@ import ( "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/utilfn" @@ -221,7 +222,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. opts = &wshrpc.FileCopyOpts{} } recursive := opts.Recursive - log.Printf("RemoteTarStreamCommand: path=%s\n", path) + logPrintfDev("RemoteTarStreamCommand: path=%s\n", path) path, err := wavebase.ExpandHomeDir(path) if err != nil { return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot expand path %q: %w", path, err)) @@ -273,6 +274,11 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if header.Name == "" { return nil } + if strings.HasPrefix(header.Name, "/") { + header.Name = header.Name[1:] + } + + log.Printf("file: %q; header: %v\n", file, header) // write header if err := tarWriter.WriteHeader(header); err != nil { @@ -311,7 +317,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } destUri := data.DestUri srcUri := data.SrcUri - // merge := opts.Merge + merge := opts.Merge overwrite := opts.Overwrite destConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, destUri) @@ -334,7 +340,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } else if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat destination %q: %w", destPathCleaned, err) } - logPrintfDev("copying %q to %q\n", srcUri, destUri) + log.Printf("copying %q to %q\n", srcUri, destUri) srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) if err != nil { return fmt.Errorf("cannot parse source URI %q: %w", srcUri, err) @@ -347,120 +353,119 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot copy file %q to %q: %w", srcPathCleaned, destPathCleaned, err) } } else { - return fmt.Errorf("cannot copy file %q to %q: source and destination must be on the same host", srcUri, destPathCleaned) - } - /* TODO: uncomment once ready for cross-connection copy - timeout := time.Millisecond * 100 - if opts.Timeout > 0 { - timeout = time.Duration(opts.Timeout) * time.Millisecond - } - readCtx, _ := context.WithTimeout(ctx, timeout) - readCtx, cancel := context.WithCancelCause(readCtx) - ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) - pipeReader, pipeWriter := io.Pipe() - iochan.WriterChan(readCtx, pipeWriter, ioch, func() { - log.Printf("closing pipe writer\n") - pipeWriter.Close() - pipeReader.Close() - }, cancel) - defer cancel(nil) - tarReader := tar.NewReader(pipeReader) - for { - select { - case <-readCtx.Done(): - if readCtx.Err() != nil { - return context.Cause(readCtx) - } - return nil - default: - next, err := tarReader.Next() - if err != nil { - if errors.Is(err, io.EOF) { + logPrintfDev("different hosts, streaming file\n") + timeout := time.Millisecond * 100 + if opts.Timeout > 0 { + timeout = time.Duration(opts.Timeout) * time.Millisecond + } + readCtx, _ := context.WithTimeout(ctx, timeout) + readCtx, cancel := context.WithCancelCause(readCtx) + ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) + pipeReader, pipeWriter := io.Pipe() + iochan.WriterChan(readCtx, pipeWriter, ioch, func() { + logPrintfDev("closing pipe writer\n") + pipeWriter.Close() + pipeReader.Close() + }, cancel) + defer cancel(nil) + tarReader := tar.NewReader(pipeReader) + for { + select { + case <-readCtx.Done(): + if readCtx.Err() != nil { + return context.Cause(readCtx) + } + return nil + default: + next, err := tarReader.Next() + if err != nil { // Do one more check for context error before returning if readCtx.Err() != nil { return context.Cause(readCtx) } - return nil + if errors.Is(err, io.EOF) { + return nil + } + return fmt.Errorf("cannot read tar stream: %w", err) } - return fmt.Errorf("cannot read tar stream: %w", err) - } - // Check for directory traversal - if strings.Contains(next.Name, "..") { - log.Printf("skipping file with unsafe path: %q\n", next.Name) - continue - } - finfo := next.FileInfo() - nextPath := filepath.Join(destPathCleaned, next.Name) - destinfo, err = os.Stat(nextPath) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("cannot stat file %q: %w", nextPath, err) - } - log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) - - if destinfo != nil { - if destinfo.IsDir() { - if !finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) + // Check for directory traversal + if strings.Contains(next.Name, "..") { + log.Printf("skipping file with unsafe path: %q\n", next.Name) + continue + } + finfo := next.FileInfo() + nextPath := filepath.Join(destPathCleaned, next.Name) + destinfo, err = os.Stat(nextPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("cannot stat file %q: %w", nextPath, err) + } + log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) + + if destinfo != nil { + if destinfo.IsDir() { + if !finfo.IsDir() { + if !overwrite { + return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) + } else { + err := os.Remove(nextPath) + if err != nil { + return fmt.Errorf("cannot remove file %q: %w", nextPath, err) + } + } + } else if !merge && !overwrite { + return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) + } else if overwrite { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + } + } + } else { + if finfo.IsDir() { + if !overwrite { + return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) + } else { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + } + } + } else if !overwrite { + return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) } else { err := os.Remove(nextPath) if err != nil { return fmt.Errorf("cannot remove file %q: %w", nextPath, err) } } - } else if !merge && !overwrite { - return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) - } else if overwrite { - err := os.RemoveAll(nextPath) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) - } } } else { if finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) - } else { - err := os.RemoveAll(nextPath) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) - } + logPrintfDev("creating directory %q\n", nextPath) + err := os.MkdirAll(nextPath, finfo.Mode()) + if err != nil { + return fmt.Errorf("cannot create directory %q: %w", nextPath, err) } - } else if !overwrite { - return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) } else { - err := os.Remove(nextPath) + err := os.MkdirAll(filepath.Dir(nextPath), 0755) if err != nil { - return fmt.Errorf("cannot remove file %q: %w", nextPath, err) + return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) } + file, err := os.Create(nextPath) + if err != nil { + return fmt.Errorf("cannot create new file %q: %w", nextPath, err) + } + _, err = io.Copy(file, tarReader) + if err != nil { + return fmt.Errorf("cannot write file %q: %w", nextPath, err) + } + file.Chmod(finfo.Mode()) + file.Close() } } - } else { - if finfo.IsDir() { - log.Printf("creating directory %q\n", nextPath) - err := os.MkdirAll(nextPath, finfo.Mode()) - if err != nil { - return fmt.Errorf("cannot create directory %q: %w", nextPath, err) - } - } else { - err := os.MkdirAll(filepath.Dir(nextPath), 0755) - if err != nil { - return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) - } - file, err := os.Create(nextPath) - if err != nil { - return fmt.Errorf("cannot create new file %q: %w", nextPath, err) - } - _, err = io.Copy(file, tarReader) - if err != nil { - return fmt.Errorf("cannot write file %q: %w", nextPath, err) - } - file.Chmod(finfo.Mode()) - file.Close() - } } } - }*/ + } return nil } From e9b42095a619fe883771b0ed8912813ae1ba71f9 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 27 Jan 2025 17:45:17 -0800 Subject: [PATCH 086/126] add cancel handles --- pkg/wshrpc/wshremote/wshremote.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index a5edb00fa..8951db2b4 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -238,10 +238,11 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } - readerCtx, _ := context.WithTimeout(context.Background(), timeout) + readerCtx, cancel := context.WithTimeout(context.Background(), timeout) rtn := iochan.ReaderChan(readerCtx, pipeReader, wshrpc.FileChunkSize, func() { pipeReader.Close() pipeWriter.Close() + cancel() }) var pathPrefix string @@ -358,8 +359,9 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } - readCtx, _ := context.WithTimeout(ctx, timeout) - readCtx, cancel := context.WithCancelCause(readCtx) + readCtx, cancel := context.WithCancelCause(ctx) + readCtx, timeoutCancel := context.WithTimeoutCause(readCtx, timeout, fmt.Errorf("timeout copying file %q to %q", srcUri, destUri)) + defer timeoutCancel() ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(readCtx, pipeWriter, ioch, func() { @@ -367,7 +369,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C pipeWriter.Close() pipeReader.Close() }, cancel) - defer cancel(nil) tarReader := tar.NewReader(pipeReader) for { select { From fc5a9683c05633ebf19dce472fc5729b80668b9d Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 27 Jan 2025 19:02:44 -0800 Subject: [PATCH 087/126] add debugging --- pkg/util/iochan/iochan.go | 6 +++-- pkg/wshrpc/wshremote/wshremote.go | 44 +++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 4145837fe..cfb4875da 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -28,6 +28,7 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func for { select { case <-ctx.Done(): + log.Printf("ReaderChan: context done") if ctx.Err() == context.Canceled { return } @@ -57,8 +58,8 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn go func() { defer func() { log.Printf("WriterChan: closing channel") - callback() drainChannel(ch) + callback() }() for { select { @@ -86,5 +87,6 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn } func drainChannel(ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { - for range ch {} + for range ch { + } } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 8951db2b4..55e37b864 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -240,9 +240,8 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } readerCtx, cancel := context.WithTimeout(context.Background(), timeout) rtn := iochan.ReaderChan(readerCtx, pipeReader, wshrpc.FileChunkSize, func() { + logPrintfDev("closing pipe reader\n") pipeReader.Close() - pipeWriter.Close() - cancel() }) var pathPrefix string @@ -252,10 +251,33 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. pathPrefix = filepath.Dir(cleanedPath) } go func() { + defer func() { + logPrintfDev("closing tar writer\n") + for { + if err := tarWriter.Close(); err != nil { + logPrintfDev("error closing tar writer: %v\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + logPrintfDev("closed tar writer\n") + break + } + logPrintfDev("closing pipe writer\n") + for { + if err := pipeWriter.Close(); err != nil { + logPrintfDev("error closing pipe writer: %v\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + logPrintfDev("closed pipe writer\n") + break + } + log.Printf("closing context\n") + cancel() + }() if readerCtx.Err() != nil { return } - defer tarWriter.Close() logPrintfDev("creating tar stream for %q\n", path) if finfo.IsDir() { logPrintfDev("%q is a directory, recursive: %v\n", path, recursive) @@ -365,11 +387,23 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(readCtx, pipeWriter, ioch, func() { - logPrintfDev("closing pipe writer\n") + log.Printf("copy closing pipe writer\n") pipeWriter.Close() - pipeReader.Close() }, cancel) tarReader := tar.NewReader(pipeReader) + defer func() { + log.Printf("copy closing pipe reader\n") + for { + if err := pipeReader.Close(); err != nil { + log.Printf("error closing pipe reader: %v\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + log.Printf("copy closed pipe reader\n") + timeoutCancel() + break + } + }() for { select { case <-readCtx.Done(): From ae650c24c73c0c3a46a69cd3ecba547f88931fb1 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 27 Jan 2025 19:09:16 -0800 Subject: [PATCH 088/126] save --- pkg/wshrpc/wshremote/wshremote.go | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 55e37b864..78c135fca 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -241,7 +241,16 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. readerCtx, cancel := context.WithTimeout(context.Background(), timeout) rtn := iochan.ReaderChan(readerCtx, pipeReader, wshrpc.FileChunkSize, func() { logPrintfDev("closing pipe reader\n") - pipeReader.Close() + log.Printf("closing pipe reader\n") + for { + if err := pipeReader.Close(); err != nil { + log.Printf("error closing pipe reader: %v\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + log.Printf("closed pipe reader\n") + break + } }) var pathPrefix string @@ -388,7 +397,16 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(readCtx, pipeWriter, ioch, func() { log.Printf("copy closing pipe writer\n") - pipeWriter.Close() + for { + if err := pipeWriter.Close(); err != nil { + log.Printf("error closing pipe writer: %v\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + log.Printf("copy closed pipe writer\n") + timeoutCancel() + break + } }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { From ef772e72d12c5683c853417f86e992b5025619ba Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 27 Jan 2025 19:25:39 -0800 Subject: [PATCH 089/126] add recursive delete --- cmd/wsh/cmd/wshcmd-file.go | 7 ++++++- frontend/app/store/wshclientapi.ts | 4 ++-- frontend/types/gotypes.d.ts | 6 ++++++ pkg/remote/fileshare/fileshare.go | 8 ++++---- pkg/remote/fileshare/fstype/fstype.go | 2 +- pkg/remote/fileshare/s3fs/s3fs.go | 2 +- pkg/remote/fileshare/wavefs/wavefs.go | 2 +- pkg/remote/fileshare/wshfs/wshfs.go | 4 ++-- pkg/wshrpc/wshclient/wshclient.go | 4 ++-- pkg/wshrpc/wshremote/wshremote.go | 20 ++++++++++++++++---- pkg/wshrpc/wshrpctypes.go | 9 +++++++-- 11 files changed, 48 insertions(+), 20 deletions(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 091dd74ac..8bacc1ae3 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -98,6 +98,7 @@ func init() { fileCmd.AddCommand(fileListCmd) fileCmd.AddCommand(fileCatCmd) fileCmd.AddCommand(fileWriteCmd) + fileRmCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") fileCmd.AddCommand(fileRmCmd) fileCmd.AddCommand(fileInfoCmd) fileCmd.AddCommand(fileAppendCmd) @@ -259,6 +260,10 @@ func fileRmRun(cmd *cobra.Command, args []string) error { if err != nil { return err } + recursive, err := cmd.Flags().GetBool("recursive") + if err != nil { + return err + } fileData := wshrpc.FileData{ Info: &wshrpc.FileInfo{ Path: path}} @@ -272,7 +277,7 @@ func fileRmRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("getting file info: %w", err) } - err = wshclient.FileDeleteCommand(RpcClient, fileData, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) + err = wshclient.FileDeleteCommand(RpcClient, wshrpc.CommandDeleteFileData{Path: path, Recursive: recursive}, &wshrpc.RpcOpts{Timeout: DefaultFileTimeout}) if err != nil { return fmt.Errorf("removing file: %w", err) } diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 2822ac298..6ef3b88a1 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -168,7 +168,7 @@ class RpcApiType { } // command "filedelete" [call] - FileDeleteCommand(client: WshClient, data: FileData, opts?: RpcOpts): Promise { + FileDeleteCommand(client: WshClient, data: CommandDeleteFileData, opts?: RpcOpts): Promise { return client.wshRpcCall("filedelete", data, opts); } @@ -258,7 +258,7 @@ class RpcApiType { } // command "remotefiledelete" [call] - RemoteFileDeleteCommand(client: WshClient, data: string, opts?: RpcOpts): Promise { + RemoteFileDeleteCommand(client: WshClient, data: CommandDeleteFileData, opts?: RpcOpts): Promise { return client.wshRpcCall("remotefiledelete", data, opts); } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index 41423efb2..c85fe17fd 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -166,6 +166,12 @@ declare global { blockid: string; }; + // wshrpc.CommandDeleteFileData + type CommandDeleteFileData = { + path: string; + recursive: boolean; + }; + // wshrpc.CommandDisposeData type CommandDisposeData = { routeid: string; diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 3b2458cad..7a0ac4df1 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -133,12 +133,12 @@ func Copy(ctx context.Context, data wshrpc.CommandFileCopyData) error { return destClient.Copy(ctx, srcConn, destConn, data.Opts) } -func Delete(ctx context.Context, path string) error { - client, conn := CreateFileShareClient(ctx, path) +func Delete(ctx context.Context, data wshrpc.CommandDeleteFileData) error { + client, conn := CreateFileShareClient(ctx, data.Path) if conn == nil || client == nil { - return fmt.Errorf(ErrorParsingConnection, path) + return fmt.Errorf(ErrorParsingConnection, data.Path) } - return client.Delete(ctx, conn) + return client.Delete(ctx, conn, data.Recursive) } func Join(ctx context.Context, path string, parts ...string) (string, error) { diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 3d42c0b03..0af71cbbd 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -34,7 +34,7 @@ type FileShareClient interface { // Copy copies the file from srcConn to destConn Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error // Delete deletes the entry at the given path - Delete(ctx context.Context, conn *connparse.Connection) error + Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error // Join joins the given parts to the connection path Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) // GetConnectionType returns the type of connection for the fileshare diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 32d0636d5..9ba46ff01 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -105,7 +105,7 @@ func (c S3Client) Copy(ctx context.Context, srcConn, destConn *connparse.Connect return nil } -func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection) error { +func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { return nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 7be121547..84c6353b7 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -357,7 +357,7 @@ func (c WaveClient) Copy(ctx context.Context, srcConn, destConn *connparse.Conne return nil } -func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection) error { +func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { zoneId := conn.Host if zoneId == "" { return fmt.Errorf("zoneid not found in connection") diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 000c58b66..8d32265c3 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -167,8 +167,8 @@ func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connec return wshclient.RemoteFileCopyCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) } -func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection) error { - return wshclient.RemoteFileDeleteCommand(RpcClient, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) +func (c WshClient) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { + return wshclient.RemoteFileDeleteCommand(RpcClient, wshrpc.CommandDeleteFileData{Path: conn.Path, Recursive: recursive}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } func (c WshClient) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index 2480484ec..c3f78614f 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -207,7 +207,7 @@ func FileCreateCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.Rpc } // command "filedelete", wshserver.FileDeleteCommand -func FileDeleteCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOpts) error { +func FileDeleteCommand(w *wshutil.WshRpc, data wshrpc.CommandDeleteFileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "filedelete", data, opts) return err } @@ -313,7 +313,7 @@ func RemoteFileCopyCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteFileCopyD } // command "remotefiledelete", wshserver.RemoteFileDeleteCommand -func RemoteFileDeleteCommand(w *wshutil.WshRpc, data string, opts *wshrpc.RpcOpts) error { +func RemoteFileDeleteCommand(w *wshutil.WshRpc, data wshrpc.CommandDeleteFileData, opts *wshrpc.RpcOpts) error { _, err := sendRpcRequestCallHelper[any](w, "remotefiledelete", data, opts) return err } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 78c135fca..e16a019d3 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -838,15 +838,27 @@ func (*ServerImpl) RemoteWriteFileCommand(ctx context.Context, data wshrpc.FileD return nil } -func (*ServerImpl) RemoteFileDeleteCommand(ctx context.Context, path string) error { - expandedPath, err := wavebase.ExpandHomeDir(path) +func (*ServerImpl) RemoteFileDeleteCommand(ctx context.Context, data wshrpc.CommandDeleteFileData) error { + expandedPath, err := wavebase.ExpandHomeDir(data.Path) if err != nil { - return fmt.Errorf("cannot delete file %q: %w", path, err) + return fmt.Errorf("cannot delete file %q: %w", data.Path, err) } cleanedPath := filepath.Clean(expandedPath) + err = os.Remove(cleanedPath) if err != nil { - return fmt.Errorf("cannot delete file %q: %w", path, err) + finfo, _ := os.Stat(cleanedPath) + if finfo != nil && finfo.IsDir() { + if !data.Recursive { + return fmt.Errorf("cannot delete directory %q, recursive option not specified", data.Path) + } + err = os.RemoveAll(cleanedPath) + if err != nil { + return fmt.Errorf("cannot delete directory %q: %w", data.Path, err) + } + } else { + return fmt.Errorf("cannot delete file %q: %w", data.Path, err) + } } return nil } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 7fc5121dc..49af0d944 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -150,7 +150,7 @@ type WshRpcInterface interface { WaitForRouteCommand(ctx context.Context, data CommandWaitForRouteData) (bool, error) FileMkdirCommand(ctx context.Context, data FileData) error FileCreateCommand(ctx context.Context, data FileData) error - FileDeleteCommand(ctx context.Context, data FileData) error + FileDeleteCommand(ctx context.Context, data CommandDeleteFileData) error FileAppendCommand(ctx context.Context, data FileData) error FileAppendIJsonCommand(ctx context.Context, data CommandAppendIJsonData) error FileWriteCommand(ctx context.Context, data FileData) error @@ -205,7 +205,7 @@ type WshRpcInterface interface { RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) RemoteFileTouchCommand(ctx context.Context, path string) error RemoteFileMoveCommand(ctx context.Context, data CommandRemoteFileCopyData) error - RemoteFileDeleteCommand(ctx context.Context, path string) error + RemoteFileDeleteCommand(ctx context.Context, data CommandDeleteFileData) error RemoteWriteFileCommand(ctx context.Context, data FileData) error RemoteFileJoinCommand(ctx context.Context, paths []string) (*FileInfo, error) RemoteMkdirCommand(ctx context.Context, path string) error @@ -498,6 +498,11 @@ type CpuDataType struct { Value float64 `json:"value"` } +type CommandDeleteFileData struct { + Path string `json:"path"` + Recursive bool `json:"recursive"` +} + type CommandFileCopyData struct { SrcUri string `json:"srcuri"` DestUri string `json:"desturi"` From 8e718cdf4ed0b7e96eee8a17b12b8e9a4763effc Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Mon, 27 Jan 2025 19:26:27 -0800 Subject: [PATCH 090/126] fix bug --- pkg/wshrpc/wshserver/wshserver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index fd575872b..196f0608a 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -307,8 +307,8 @@ func (ws *WshServer) FileMkdirCommand(ctx context.Context, data wshrpc.FileData) return fileshare.Mkdir(ctx, data.Info.Path) } -func (ws *WshServer) FileDeleteCommand(ctx context.Context, data wshrpc.FileData) error { - return fileshare.Delete(ctx, data.Info.Path) +func (ws *WshServer) FileDeleteCommand(ctx context.Context, data wshrpc.CommandDeleteFileData) error { + return fileshare.Delete(ctx, data) } func (ws *WshServer) FileInfoCommand(ctx context.Context, data wshrpc.FileData) (*wshrpc.FileInfo, error) { From fabf85a65845131cdec61f7dc02bdd4725e72c3b Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 10:46:11 -0800 Subject: [PATCH 091/126] avoid buffer reuse --- pkg/util/iochan/iochan.go | 4 +--- pkg/wshrpc/wshremote/wshremote.go | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index cfb4875da..0d648741e 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -24,7 +24,6 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func close(ch) callback() }() - buf := make([]byte, chunkSize) for { select { case <-ctx.Done(): @@ -35,6 +34,7 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func log.Printf("ReaderChan: context error: %v", ctx.Err()) return default: + buf := make([]byte, chunkSize) if n, err := r.Read(buf); err != nil { if errors.Is(err, io.EOF) { log.Printf("ReaderChan: EOF") @@ -78,8 +78,6 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn log.Printf("WriterChan: write error: %v", err) errCallback(err) return - } else { - // log.Printf("WriterChan: wrote %d bytes", n) } } } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 34a6d190c..e3e9a9019 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -454,8 +454,9 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } if errors.Is(err, io.EOF) { return nil + } else { + return fmt.Errorf("cannot read tar stream: %w", err) } - return fmt.Errorf("cannot read tar stream: %w", err) } // Check for directory traversal if strings.Contains(next.Name, "..") { From 5c6feb12c30d925dcbd237c6c19fb0ec12d4f793 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 10:50:31 -0800 Subject: [PATCH 092/126] remove artificial delay --- pkg/wshrpc/wshremote/wshremote.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index e3e9a9019..7865421f8 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -326,7 +326,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. header.Name = header.Name[1:] } - log.Printf("file: %q; header: %v\n", file, header) + // log.Printf("file: %q; header: %v\n", file, header) // write header if err := tarWriter.WriteHeader(header); err != nil { @@ -345,7 +345,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. logPrintfDev("wrote %d bytes to tar stream\n", n) } } - time.Sleep(time.Millisecond * 10) return nil }) if err != nil { @@ -469,7 +468,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat file %q: %w", nextPath, err) } - log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) + // log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) if destinfo != nil { if destinfo.IsDir() { From 41df9d50b0874dcf9d54877c64bb847a78869d87 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 10:59:17 -0800 Subject: [PATCH 093/126] remove annoying logs --- pkg/wshrpc/wshremote/wshremote.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 7865421f8..e64c98a68 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -338,11 +338,9 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } - if n, err := io.Copy(tarWriter, data); err != nil { + if _, err := io.Copy(tarWriter, data); err != nil { log.Printf("error copying file %q: %v\n", file, err) return err - } else { - logPrintfDev("wrote %d bytes to tar stream\n", n) } } return nil @@ -408,6 +406,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C readCtx, cancel := context.WithCancelCause(ctx) readCtx, timeoutCancel := context.WithTimeoutCause(readCtx, timeout, fmt.Errorf("timeout copying file %q to %q", srcUri, destUri)) defer timeoutCancel() + copyStart := time.Now() ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(readCtx, pipeWriter, ioch, func() { @@ -437,12 +436,14 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C break } }() + numFiles := 0 for { select { case <-readCtx.Done(): if readCtx.Err() != nil { return context.Cause(readCtx) } + log.Printf("copy complete: %d files copied in %vms\n", numFiles, time.Since(copyStart).Milliseconds()) return nil default: next, err := tarReader.Next() @@ -452,11 +453,13 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return context.Cause(readCtx) } if errors.Is(err, io.EOF) { + log.Printf("copy complete: %d files copied in %vms\n", numFiles, time.Since(copyStart).Milliseconds()) return nil } else { return fmt.Errorf("cannot read tar stream: %w", err) } } + numFiles++ // Check for directory traversal if strings.Contains(next.Name, "..") { log.Printf("skipping file with unsafe path: %q\n", next.Name) From b10cec1d5ccbab09bbb719b77ad9a4c46162df42 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 11:07:01 -0800 Subject: [PATCH 094/126] save --- pkg/wshrpc/wshremote/wshremote.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index e64c98a68..4abcf278a 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -437,13 +437,14 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } }() numFiles := 0 + numSkipped := 0 for { select { case <-readCtx.Done(): if readCtx.Err() != nil { return context.Cause(readCtx) } - log.Printf("copy complete: %d files copied in %vms\n", numFiles, time.Since(copyStart).Milliseconds()) + log.Printf("copy complete: %d files copied in %dms, %d skipped\n", numFiles, time.Since(copyStart).Milliseconds(), numSkipped) return nil default: next, err := tarReader.Next() @@ -453,18 +454,19 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return context.Cause(readCtx) } if errors.Is(err, io.EOF) { - log.Printf("copy complete: %d files copied in %vms\n", numFiles, time.Since(copyStart).Milliseconds()) + log.Printf("copy complete: %d files copied in %dms, %d skipped\n", numFiles, time.Since(copyStart).Milliseconds(), numSkipped) return nil } else { return fmt.Errorf("cannot read tar stream: %w", err) } } - numFiles++ // Check for directory traversal if strings.Contains(next.Name, "..") { log.Printf("skipping file with unsafe path: %q\n", next.Name) + numSkipped++ continue } + numFiles++ finfo := next.FileInfo() nextPath := filepath.Join(destPathCleaned, next.Name) destinfo, err = os.Stat(nextPath) From 536ffc3630055047531f8ad4f9503ad4904c08a2 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 11:54:13 -0800 Subject: [PATCH 095/126] remove file limit for streaming, add timeout --- pkg/web/web.go | 80 +++++++++++++++++-------------- pkg/wshrpc/wshremote/wshremote.go | 4 -- 2 files changed, 45 insertions(+), 39 deletions(-) diff --git a/pkg/web/web.go b/pkg/web/web.go index 35d9445c6..6cc1f999b 100644 --- a/pkg/web/web.go +++ b/pkg/web/web.go @@ -243,11 +243,12 @@ func handleLocalStreamFile(w http.ResponseWriter, r *http.Request, path string, } } -func handleRemoteStreamFile(w http.ResponseWriter, _ *http.Request, conn string, path string, no404 bool) error { +func handleRemoteStreamFile(w http.ResponseWriter, req *http.Request, conn string, path string, no404 bool) error { client := wshserver.GetMainRpcClient() streamFileData := wshrpc.CommandRemoteStreamFileData{Path: path} route := wshutil.MakeConnectionRouteId(conn) - rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, &wshrpc.RpcOpts{Route: route}) + rpcOpts := &wshrpc.RpcOpts{Route: route, Timeout: 60 * 1000} + rtnCh := wshclient.RemoteStreamFileCommand(client, streamFileData, rpcOpts) firstPk := true var fileInfo *wshrpc.FileInfo loopDone := false @@ -261,45 +262,54 @@ func handleRemoteStreamFile(w http.ResponseWriter, _ *http.Request, conn string, } }() }() - for respUnion := range rtnCh { - if respUnion.Error != nil { - return respUnion.Error - } - if firstPk { - firstPk = false - if respUnion.Response.Info == nil { - return fmt.Errorf("stream file protocol error, fileinfo is empty") + ctx := req.Context() + for { + select { + case <-ctx.Done(): + rpcOpts.StreamCancelFn() + return ctx.Err() + case respUnion, ok := <-rtnCh: + if !ok { + loopDone = true + return nil + } + if respUnion.Error != nil { + return respUnion.Error } - fileInfo = respUnion.Response.Info - if fileInfo.NotFound { - if no404 { - serveTransparentGIF(w) - return nil - } else { - return fmt.Errorf("file not found: %q", path) + if firstPk { + firstPk = false + if respUnion.Response.Info == nil { + return fmt.Errorf("stream file protocol error, fileinfo is empty") + } + fileInfo = respUnion.Response.Info + if fileInfo.NotFound { + if no404 { + serveTransparentGIF(w) + return nil + } else { + return fmt.Errorf("file not found: %q", path) + } + } + if fileInfo.IsDir { + return fmt.Errorf("cannot stream directory: %q", path) } + w.Header().Set(ContentTypeHeaderKey, fileInfo.MimeType) + w.Header().Set(ContentLengthHeaderKey, fmt.Sprintf("%d", fileInfo.Size)) + continue } - if fileInfo.IsDir { - return fmt.Errorf("cannot stream directory: %q", path) + if respUnion.Response.Data64 == "" { + continue + } + decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(respUnion.Response.Data64))) + _, err := io.Copy(w, decoder) + if err != nil { + log.Printf("error streaming file %q: %v\n", path, err) + // not sure what to do here, the headers have already been sent. + // just return + return nil } - w.Header().Set(ContentTypeHeaderKey, fileInfo.MimeType) - w.Header().Set(ContentLengthHeaderKey, fmt.Sprintf("%d", fileInfo.Size)) - continue - } - if respUnion.Response.Data64 == "" { - continue - } - decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader([]byte(respUnion.Response.Data64))) - _, err := io.Copy(w, decoder) - if err != nil { - log.Printf("error streaming file %q: %v\n", path, err) - // not sure what to do here, the headers have already been sent. - // just return - return nil } } - loopDone = true - return nil } func handleStreamFile(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 4abcf278a..b4a996301 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -191,9 +191,6 @@ func (impl *ServerImpl) remoteStreamFileInternal(ctx context.Context, data wshrp if finfo.NotFound { return nil } - if finfo.Size > wshrpc.MaxFileSize { - return fmt.Errorf("file %q is too large to read, use /wave/stream-file", path) - } if finfo.IsDir { return impl.remoteStreamFileDir(ctx, path, byteRange, dataCallback) } else { @@ -221,7 +218,6 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc resp.Data64 = base64.StdEncoding.EncodeToString(data) resp.At = &wshrpc.FileDataAt{Offset: byteRange.Start, Size: len(data)} } - logPrintfDev("callback -- sending response %d\n", len(resp.Data64)) ch <- wshrpc.RespOrErrorUnion[wshrpc.FileData]{Response: resp} }) if err != nil { From b1f2175c71b33e56a7a055d7f9202c4b66cd315b Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 11:56:03 -0800 Subject: [PATCH 096/126] comment out s3 again --- pkg/remote/fileshare/fileshare.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 7a0ac4df1..56738d325 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -5,10 +5,8 @@ import ( "fmt" "log" - "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare/s3fs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -29,12 +27,12 @@ func CreateFileShareClient(ctx context.Context, connection string) (fstype.FileS } conntype := conn.GetType() if conntype == connparse.ConnectionTypeS3 { - config, err := awsconn.GetConfig(ctx, connection) - if err != nil { - log.Printf("error getting aws config: %v", err) - return nil, nil - } - return s3fs.NewS3Client(config), conn + // config, err := awsconn.GetConfig(ctx, connection) + // if err != nil { + // log.Printf("error getting aws config: %v", err) + // return nil, nil + // } + return nil, nil } else if conntype == connparse.ConnectionTypeWave { return wavefs.NewWaveClient(), conn } else if conntype == connparse.ConnectionTypeWsh { From e3eea074038ddacecd862a9c0e3ec916d9d11067 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 12:59:23 -0800 Subject: [PATCH 097/126] Add sha sum check to copy --- cmd/generatego/main-generatego.go | 1 + frontend/app/store/wshclientapi.ts | 4 +- frontend/types/gotypes.d.ts | 6 +++ pkg/remote/fileshare/fileshare.go | 5 ++- pkg/remote/fileshare/fstype/fstype.go | 3 +- pkg/remote/fileshare/s3fs/s3fs.go | 3 +- pkg/remote/fileshare/wavefs/wavefs.go | 30 +------------- pkg/remote/fileshare/wshfs/wshfs.go | 3 +- pkg/util/iochan/iochan.go | 48 ++++++++++++++-------- pkg/util/iochan/iochantypes/iochantypes.go | 6 +++ pkg/wshrpc/wshclient/wshclient.go | 9 ++-- pkg/wshrpc/wshremote/wshremote.go | 11 ++--- pkg/wshrpc/wshrpctypes.go | 5 ++- pkg/wshrpc/wshserver/wshserver.go | 3 +- 14 files changed, 72 insertions(+), 65 deletions(-) create mode 100644 pkg/util/iochan/iochantypes/iochantypes.go diff --git a/cmd/generatego/main-generatego.go b/cmd/generatego/main-generatego.go index c93faa352..62e65d4f7 100644 --- a/cmd/generatego/main-generatego.go +++ b/cmd/generatego/main-generatego.go @@ -30,6 +30,7 @@ func GenerateWshClient() error { "github.com/wavetermdev/waveterm/pkg/waveobj", "github.com/wavetermdev/waveterm/pkg/wps", "github.com/wavetermdev/waveterm/pkg/vdom", + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes", }) wshDeclMap := wshrpc.GenerateWshCommandDeclMap() for _, key := range utilfn.GetOrderedMapKeys(wshDeclMap) { diff --git a/frontend/app/store/wshclientapi.ts b/frontend/app/store/wshclientapi.ts index 6ef3b88a1..9c614af4e 100644 --- a/frontend/app/store/wshclientapi.ts +++ b/frontend/app/store/wshclientapi.ts @@ -203,7 +203,7 @@ class RpcApiType { } // command "filestreamtar" [responsestream] - FileStreamTarCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { + FileStreamTarCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("filestreamtar", data, opts); } @@ -313,7 +313,7 @@ class RpcApiType { } // command "remotetarstream" [responsestream] - RemoteTarStreamCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { + RemoteTarStreamCommand(client: WshClient, data: CommandRemoteStreamTarData, opts?: RpcOpts): AsyncGenerator { return client.wshRpcStream("remotetarstream", data, opts); } diff --git a/frontend/types/gotypes.d.ts b/frontend/types/gotypes.d.ts index c85fe17fd..400e86935 100644 --- a/frontend/types/gotypes.d.ts +++ b/frontend/types/gotypes.d.ts @@ -589,6 +589,12 @@ declare global { // waveobj.ORef type ORef = string; + // iochantypes.Packet + type Packet = { + Data: string; + Checksum: string; + }; + // wshrpc.PathCommandData type PathCommandData = { pathtype: string; diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index 56738d325..bbafae751 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -9,6 +9,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wavefs" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" ) @@ -59,10 +60,10 @@ func ReadStream(ctx context.Context, data wshrpc.FileData) <-chan wshrpc.RespOrE return client.ReadStream(ctx, conn, data) } -func ReadTarStream(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func ReadTarStream(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { client, conn := CreateFileShareClient(ctx, data.Path) if conn == nil || client == nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf(ErrorParsingConnection, data.Path)) + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf(ErrorParsingConnection, data.Path)) } return client.ReadTarStream(ctx, conn, data.Opts) } diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 0af71cbbd..6b42902c7 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -7,6 +7,7 @@ import ( "context" "github.com/wavetermdev/waveterm/pkg/remote/connparse" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" ) @@ -18,7 +19,7 @@ type FileShareClient interface { // ReadStream returns a stream of file data at the given path. If it's a directory, then the list of entries ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] // ReadTarStream returns a stream of tar data at the given path - ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] + ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] // ListEntries returns the list of entries at the given path, or nothing if the path is a file ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) // ListEntriesStream returns a stream of entries at the given path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 9ba46ff01..e73c409c7 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -12,6 +12,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/awsconn" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" ) @@ -36,7 +37,7 @@ func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, da return nil } -func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { return nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 84c6353b7..9b13bb3a3 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -15,6 +15,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" @@ -95,7 +96,7 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w return &wshrpc.FileData{Info: data.Info, Entries: list}, nil } -func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { return nil } @@ -265,33 +266,6 @@ func (c WaveClient) PutFile(ctx context.Context, conn *connparse.Connection, dat return nil } -/* - - path := data.Info.Path - log.Printf("Append: path=%s", path) - client, conn := CreateFileShareClient(ctx, path) - if conn == nil || client == nil { - return fmt.Errorf(ErrorParsingConnection, path) - } - finfo, err := client.Stat(ctx, conn) - if err != nil { - return err - } - if data.Info == nil { - data.Info = &wshrpc.FileInfo{} - } - oldInfo := data.Info - data.Info = finfo - if oldInfo.Opts != nil { - data.Info.Opts = oldInfo.Opts - } - data.At = &wshrpc.FileDataAt{ - Offset: finfo.Size, - } - log.Printf("Append: offset=%d", data.At.Offset) - return client.PutFile(ctx, conn, data) -*/ - func (c WaveClient) AppendFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { dataBuf, err := base64.StdEncoding.DecodeString(data.Data64) if err != nil { diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 8d32265c3..4fcceabb0 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -12,6 +12,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshutil" @@ -86,7 +87,7 @@ func (c WshClient) ReadStream(ctx context.Context, conn *connparse.Connection, d return wshclient.RemoteStreamFileCommand(RpcClient, streamFileData, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } -func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (c WshClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { timeout := opts.Timeout if timeout == 0 { timeout = ThirtySeconds diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 0d648741e..893aff9ab 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -5,47 +5,49 @@ package iochan import ( + "bytes" "context" + "crypto/sha256" "errors" "fmt" "io" - "log" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/wshrpc" "github.com/wavetermdev/waveterm/pkg/wshutil" ) // ReaderChan reads from an io.Reader and sends the data to a channel -func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) chan wshrpc.RespOrErrorUnion[[]byte] { - ch := make(chan wshrpc.RespOrErrorUnion[[]byte], 32) +func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { + ch := make(chan wshrpc.RespOrErrorUnion[iochantypes.Packet], 32) + sha256Hash := sha256.New() go func() { defer func() { - log.Printf("ReaderChan: closing channel") close(ch) callback() }() for { select { case <-ctx.Done(): - log.Printf("ReaderChan: context done") if ctx.Err() == context.Canceled { return } - log.Printf("ReaderChan: context error: %v", ctx.Err()) return default: buf := make([]byte, chunkSize) if n, err := r.Read(buf); err != nil { if errors.Is(err, io.EOF) { - log.Printf("ReaderChan: EOF") + ch <- wshrpc.RespOrErrorUnion[iochantypes.Packet]{Response: iochantypes.Packet{Checksum: sha256Hash.Sum(nil)}} // send the checksum return } - ch <- wshutil.RespErr[[]byte](fmt.Errorf("ReaderChan: read error: %v", err)) - log.Printf("ReaderChan: read error: %v", err) + ch <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("ReaderChan: read error: %v", err)) return } else if n > 0 { - // log.Printf("ReaderChan: read %d bytes", n) - ch <- wshrpc.RespOrErrorUnion[[]byte]{Response: buf[:n]} + if _, err := sha256Hash.Write(buf[:n]); err != nil { + ch <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("ReaderChan: error writing to sha256 hash: %v", err)) + return + } + ch <- wshrpc.RespOrErrorUnion[iochantypes.Packet]{Response: iochantypes.Packet{Data: buf[:n]}} } } } @@ -54,10 +56,10 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } // WriterChan reads from a channel and writes the data to an io.Writer -func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[[]byte], callback func(), errCallback func(error)) { +func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], callback func(), errCallback func(error)) { + sha256Hash := sha256.New() go func() { defer func() { - log.Printf("WriterChan: closing channel") drainChannel(ch) callback() }() @@ -70,13 +72,23 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn return } if resp.Error != nil { - log.Printf("WriterChan: error: %v", resp.Error) errCallback(resp.Error) return } - if _, err := w.Write(resp.Response); err != nil { - log.Printf("WriterChan: write error: %v", err) - errCallback(err) + if _, err := sha256Hash.Write(resp.Response.Data); err != nil { + errCallback(fmt.Errorf("WriterChan: error writing to sha256 hash: %v", err)) + return + } + // The checksum is sent as the last packet + if resp.Response.Checksum != nil { + localChecksum := sha256Hash.Sum(nil) + if !bytes.Equal(localChecksum, resp.Response.Checksum) { + errCallback(fmt.Errorf("WriterChan: checksum mismatch")) + } + return + } + if _, err := w.Write(resp.Response.Data); err != nil { + errCallback(fmt.Errorf("WriterChan: write error: %v", err)) return } } @@ -84,7 +96,7 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn }() } -func drainChannel(ch <-chan wshrpc.RespOrErrorUnion[[]byte]) { +func drainChannel(ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet]) { for range ch { } } diff --git a/pkg/util/iochan/iochantypes/iochantypes.go b/pkg/util/iochan/iochantypes/iochantypes.go new file mode 100644 index 000000000..00ea42b0b --- /dev/null +++ b/pkg/util/iochan/iochantypes/iochantypes.go @@ -0,0 +1,6 @@ +package iochantypes + +type Packet struct { + Data []byte + Checksum []byte +} diff --git a/pkg/wshrpc/wshclient/wshclient.go b/pkg/wshrpc/wshclient/wshclient.go index c3f78614f..599cd79c5 100644 --- a/pkg/wshrpc/wshclient/wshclient.go +++ b/pkg/wshrpc/wshclient/wshclient.go @@ -12,6 +12,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" "github.com/wavetermdev/waveterm/pkg/vdom" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" ) // command "activity", wshserver.ActivityCommand @@ -248,8 +249,8 @@ func FileReadCommand(w *wshutil.WshRpc, data wshrpc.FileData, opts *wshrpc.RpcOp } // command "filestreamtar", wshserver.FileStreamTarCommand -func FileStreamTarCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTarData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { - return sendRpcRequestResponseStreamHelper[[]uint8](w, "filestreamtar", data, opts) +func FileStreamTarCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTarData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { + return sendRpcRequestResponseStreamHelper[iochantypes.Packet](w, "filestreamtar", data, opts) } // command "filewrite", wshserver.FileWriteCommand @@ -376,8 +377,8 @@ func RemoteStreamFileCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamF } // command "remotetarstream", wshserver.RemoteTarStreamCommand -func RemoteTarStreamCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTarData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[[]uint8] { - return sendRpcRequestResponseStreamHelper[[]uint8](w, "remotetarstream", data, opts) +func RemoteTarStreamCommand(w *wshutil.WshRpc, data wshrpc.CommandRemoteStreamTarData, opts *wshrpc.RpcOpts) chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { + return sendRpcRequestResponseStreamHelper[iochantypes.Packet](w, "remotetarstream", data, opts) } // command "remotewritefile", wshserver.RemoteWriteFileCommand diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index b4a996301..385f52244 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -21,6 +21,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" @@ -227,7 +228,7 @@ func (impl *ServerImpl) RemoteStreamFileCommand(ctx context.Context, data wshrpc return ch } -func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { path := data.Path opts := data.Opts if opts == nil { @@ -237,12 +238,12 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. logPrintfDev("RemoteTarStreamCommand: path=%s\n", path) path, err := wavebase.ExpandHomeDir(path) if err != nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot expand path %q: %w", path, err)) + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("cannot expand path %q: %w", path, err)) } cleanedPath := filepath.Clean(wavebase.ExpandHomeDirSafe(path)) finfo, err := os.Stat(cleanedPath) if err != nil { - return wshutil.SendErrCh[[]byte](fmt.Errorf("cannot stat file %q: %w", path, err)) + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("cannot stat file %q: %w", path, err)) } pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) @@ -303,7 +304,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if finfo.IsDir() { logPrintfDev("%q is a directory, recursive: %v\n", path, recursive) if !recursive { - rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) + rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) return } } @@ -342,7 +343,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. return nil }) if err != nil { - rtn <- wshutil.RespErr[[]byte](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) + rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) } logPrintfDev("returning tar stream\n") }() diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index 49af0d944..e2fc7403a 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -13,6 +13,7 @@ import ( "reflect" "github.com/wavetermdev/waveterm/pkg/ijson" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/vdom" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wconfig" @@ -155,7 +156,7 @@ type WshRpcInterface interface { FileAppendIJsonCommand(ctx context.Context, data CommandAppendIJsonData) error FileWriteCommand(ctx context.Context, data FileData) error FileReadCommand(ctx context.Context, data FileData) (*FileData, error) - FileStreamTarCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[[]byte] + FileStreamTarCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[iochantypes.Packet] FileMoveCommand(ctx context.Context, data CommandFileCopyData) error FileCopyCommand(ctx context.Context, data CommandFileCopyData) error FileInfoCommand(ctx context.Context, data FileData) (*FileInfo, error) @@ -199,7 +200,7 @@ type WshRpcInterface interface { // remotes RemoteStreamFileCommand(ctx context.Context, data CommandRemoteStreamFileData) chan RespOrErrorUnion[FileData] - RemoteTarStreamCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[[]byte] + RemoteTarStreamCommand(ctx context.Context, data CommandRemoteStreamTarData) <-chan RespOrErrorUnion[iochantypes.Packet] RemoteFileCopyCommand(ctx context.Context, data CommandRemoteFileCopyData) error RemoteListEntriesCommand(ctx context.Context, data CommandRemoteListEntriesData) chan RespOrErrorUnion[CommandRemoteListEntriesRtnData] RemoteFileInfoCommand(ctx context.Context, path string) (*FileInfo, error) diff --git a/pkg/wshrpc/wshserver/wshserver.go b/pkg/wshrpc/wshserver/wshserver.go index 196f0608a..ff403cd73 100644 --- a/pkg/wshrpc/wshserver/wshserver.go +++ b/pkg/wshrpc/wshserver/wshserver.go @@ -28,6 +28,7 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/fileshare" "github.com/wavetermdev/waveterm/pkg/telemetry" "github.com/wavetermdev/waveterm/pkg/util/envutil" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/util/shellutil" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" @@ -339,7 +340,7 @@ func (ws *WshServer) FileMoveCommand(ctx context.Context, data wshrpc.CommandFil return fileshare.Move(ctx, data) } -func (ws *WshServer) FileStreamTarCommand(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[[]byte] { +func (ws *WshServer) FileStreamTarCommand(ctx context.Context, data wshrpc.CommandRemoteStreamTarData) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { return fileshare.ReadTarStream(ctx, data) } From 5b1e0e12343d2c2ecb604dadecd94469b3a64576 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 13:06:54 -0800 Subject: [PATCH 098/126] clean up logging --- pkg/wshrpc/wshremote/wshremote.go | 46 +++++++++++-------------------- 1 file changed, 16 insertions(+), 30 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 385f52244..46239a3a0 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -253,15 +253,12 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } readerCtx, cancel := context.WithTimeout(context.Background(), timeout) rtn := iochan.ReaderChan(readerCtx, pipeReader, wshrpc.FileChunkSize, func() { - logPrintfDev("closing pipe reader\n") - log.Printf("closing pipe reader\n") for { if err := pipeReader.Close(); err != nil { - log.Printf("error closing pipe reader: %v\n", err) + log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) time.Sleep(time.Millisecond * 10) continue } - log.Printf("closed pipe reader\n") break } }) @@ -274,35 +271,28 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } go func() { defer func() { - logPrintfDev("closing tar writer\n") for { if err := tarWriter.Close(); err != nil { - logPrintfDev("error closing tar writer: %v\n", err) + logPrintfDev("error closing tar writer: %v, trying again in 10ms\n", err) time.Sleep(time.Millisecond * 10) continue } - logPrintfDev("closed tar writer\n") break } - logPrintfDev("closing pipe writer\n") for { if err := pipeWriter.Close(); err != nil { - logPrintfDev("error closing pipe writer: %v\n", err) + logPrintfDev("error closing pipe writer: %v, trying again in 10ms\n", err) time.Sleep(time.Millisecond * 10) continue } - logPrintfDev("closed pipe writer\n") break } - log.Printf("closing context\n") cancel() }() if readerCtx.Err() != nil { return } - logPrintfDev("creating tar stream for %q\n", path) if finfo.IsDir() { - logPrintfDev("%q is a directory, recursive: %v\n", path, recursive) if !recursive { rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) return @@ -323,8 +313,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. header.Name = header.Name[1:] } - // log.Printf("file: %q; header: %v\n", file, header) - // write header if err := tarWriter.WriteHeader(header); err != nil { return err @@ -345,9 +333,9 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) } - logPrintfDev("returning tar stream\n") + log.Printf("RemoteTarStreamCommand: done\n") }() - logPrintfDev("returning channel\n") + log.Printf("RemoteTarStreamCommand: returning channel\n") return rtn } @@ -382,20 +370,17 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } else if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat destination %q: %w", destPathCleaned, err) } - log.Printf("copying %q to %q\n", srcUri, destUri) srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) if err != nil { return fmt.Errorf("cannot parse source URI %q: %w", srcUri, err) } if srcConn.Host == destConn.Host { - logPrintfDev("same host, copying file\n") srcPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(srcConn.Path)) err := os.Rename(srcPathCleaned, destPathCleaned) if err != nil { return fmt.Errorf("cannot copy file %q to %q: %w", srcPathCleaned, destPathCleaned, err) } } else { - logPrintfDev("different hosts, streaming file\n") timeout := time.Millisecond * 100 if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond @@ -407,41 +392,41 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(readCtx, pipeWriter, ioch, func() { - log.Printf("copy closing pipe writer\n") for { if err := pipeWriter.Close(); err != nil { - log.Printf("error closing pipe writer: %v\n", err) + log.Printf("error closing pipe writer: %v, trying again in 10ms\n", err) time.Sleep(time.Millisecond * 10) continue } - log.Printf("copy closed pipe writer\n") timeoutCancel() break } }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { - log.Printf("copy closing pipe reader\n") for { if err := pipeReader.Close(); err != nil { - log.Printf("error closing pipe reader: %v\n", err) + log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) time.Sleep(time.Millisecond * 10) continue } - log.Printf("copy closed pipe reader\n") timeoutCancel() break } }() numFiles := 0 numSkipped := 0 + totalBytes := int64(0) + logResult := func() { + log.Printf("RemoteFileCopyCommand: done; %d files copied in %dms, total of %d bytes, %d files skipped\n", numFiles, time.Since(copyStart).Milliseconds(), totalBytes, numSkipped) + } for { select { case <-readCtx.Done(): if readCtx.Err() != nil { return context.Cause(readCtx) } - log.Printf("copy complete: %d files copied in %dms, %d skipped\n", numFiles, time.Since(copyStart).Milliseconds(), numSkipped) + logResult() return nil default: next, err := tarReader.Next() @@ -451,7 +436,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return context.Cause(readCtx) } if errors.Is(err, io.EOF) { - log.Printf("copy complete: %d files copied in %dms, %d skipped\n", numFiles, time.Since(copyStart).Milliseconds(), numSkipped) + logResult() return nil } else { return fmt.Errorf("cannot read tar stream: %w", err) @@ -470,7 +455,9 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat file %q: %w", nextPath, err) } - // log.Printf("new file: name %q; dest %q\n", next.Name, nextPath) + if !finfo.IsDir() { + totalBytes += finfo.Size() + } if destinfo != nil { if destinfo.IsDir() { @@ -512,7 +499,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } else { if finfo.IsDir() { - logPrintfDev("creating directory %q\n", nextPath) err := os.MkdirAll(nextPath, finfo.Mode()) if err != nil { return fmt.Errorf("cannot create directory %q: %w", nextPath, err) From 78e6d328ac6b6cc6d0b4441139ffed5632db9f22 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 13:10:43 -0800 Subject: [PATCH 099/126] add remote move impl --- pkg/wshrpc/wshremote/wshremote.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 46239a3a0..f20e0c072 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -742,22 +742,25 @@ func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.C } else if !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("cannot stat destination %q: %w", destUri, err) } - logPrintfDev("moving %q to %q\n", srcUri, destUri) srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, srcUri) if err != nil { return fmt.Errorf("cannot parse source URI %q: %w", srcUri, err) } - logPrintfDev("source host: %q, destination host: %q\n", srcConn.Host, destConn.Host) if srcConn.Host == destConn.Host { - logPrintfDev("moving file on same host\n") srcPathCleaned := filepath.Clean(wavebase.ExpandHomeDirSafe(srcConn.Path)) - logPrintfDev("moving %q to %q\n", srcPathCleaned, destPathCleaned) err := os.Rename(srcPathCleaned, destPathCleaned) if err != nil { return fmt.Errorf("cannot move file %q to %q: %w", srcPathCleaned, destPathCleaned, err) } } else { - return fmt.Errorf("cannot move file %q to %q: source and destination must be on the same host", srcUri, destUri) + err := impl.RemoteFileCopyCommand(ctx, data) + if err != nil { + return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) + } + err = impl.RemoteFileDeleteCommand(ctx, wshrpc.CommandDeleteFileData{Path: srcUri, Recursive: data.Opts.Recursive}) + if err != nil { + return fmt.Errorf("cannot delete %q: %w", srcUri, err) + } } return nil } From 2d83a7f688a78e0a00d4a3eaccf8c46c6d73ae47 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 15:04:42 -0800 Subject: [PATCH 100/126] fix delete in mv --- pkg/wshrpc/wshremote/wshremote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index f20e0c072..c59c85003 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -757,7 +757,7 @@ func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.C if err != nil { return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) } - err = impl.RemoteFileDeleteCommand(ctx, wshrpc.CommandDeleteFileData{Path: srcUri, Recursive: data.Opts.Recursive}) + err = fileshare.Delete(ctx, wshrpc.CommandDeleteFileData{Path: srcUri, Recursive: data.Opts.Recursive}) if err != nil { return fmt.Errorf("cannot delete %q: %w", srcUri, err) } From 8ed10f7adde926d3eb18e361986cc71a0371a8fb Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 16:36:15 -0800 Subject: [PATCH 101/126] route tar stream call to wavesrv, consolidate tar copy to new package pt 1 --- pkg/remote/fileshare/fileshare.go | 32 ++++++--- pkg/remote/fileshare/fstype/fstype.go | 10 +-- pkg/remote/fileshare/s3fs/s3fs.go | 8 ++- pkg/remote/fileshare/wavefs/wavefs.go | 55 +++++++++++++++- pkg/remote/fileshare/wshfs/wshfs.go | 11 +++- pkg/util/tarcopy/tarcopy.go | 69 +++++++++++++++++++ pkg/wshrpc/wshremote/wshremote.go | 95 +++++++-------------------- pkg/wshrpc/wshrpctypes.go | 1 + 8 files changed, 187 insertions(+), 94 deletions(-) create mode 100644 pkg/util/tarcopy/tarcopy.go diff --git a/pkg/remote/fileshare/fileshare.go b/pkg/remote/fileshare/fileshare.go index bbafae751..9473db55a 100644 --- a/pkg/remote/fileshare/fileshare.go +++ b/pkg/remote/fileshare/fileshare.go @@ -109,27 +109,39 @@ func Mkdir(ctx context.Context, path string) error { } func Move(ctx context.Context, data wshrpc.CommandFileCopyData) error { - srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, data.SrcUri) - if err != nil { - return fmt.Errorf("error parsing source connection %s: %v", data.SrcUri, err) + srcClient, srcConn := CreateFileShareClient(ctx, data.SrcUri) + if srcConn == nil || srcClient == nil { + return fmt.Errorf("error creating fileshare client, could not parse source connection %s", data.SrcUri) } destClient, destConn := CreateFileShareClient(ctx, data.DestUri) if destConn == nil || destClient == nil { - return fmt.Errorf("error creating fileshare client, could not parse connection %s or %s", data.SrcUri, data.DestUri) + return fmt.Errorf("error creating fileshare client, could not parse destination connection %s", data.DestUri) + } + if srcConn.Host != destConn.Host { + err := destClient.CopyRemote(ctx, srcConn, destConn, srcClient, data.Opts) + if err != nil { + return fmt.Errorf("cannot copy %q to %q: %w", data.SrcUri, data.DestUri, err) + } + return srcClient.Delete(ctx, srcConn, data.Opts.Recursive) + } else { + return srcClient.MoveInternal(ctx, srcConn, destConn, data.Opts) } - return destClient.Move(ctx, srcConn, destConn, data.Opts) } func Copy(ctx context.Context, data wshrpc.CommandFileCopyData) error { - srcConn, err := connparse.ParseURIAndReplaceCurrentHost(ctx, data.SrcUri) - if err != nil { - return fmt.Errorf("error parsing source connection %s: %v", data.SrcUri, err) + srcClient, srcConn := CreateFileShareClient(ctx, data.SrcUri) + if srcConn == nil || srcClient == nil { + return fmt.Errorf("error creating fileshare client, could not parse source connection %s", data.SrcUri) } destClient, destConn := CreateFileShareClient(ctx, data.DestUri) if destConn == nil || destClient == nil { - return fmt.Errorf("error creating fileshare client, could not parse connection %s or %s", data.SrcUri, data.DestUri) + return fmt.Errorf("error creating fileshare client, could not parse destination connection %s", data.DestUri) + } + if srcConn.Host != destConn.Host { + return destClient.CopyRemote(ctx, srcConn, destConn, srcClient, data.Opts) + } else { + return srcClient.CopyInternal(ctx, srcConn, destConn, data.Opts) } - return destClient.Copy(ctx, srcConn, destConn, data.Opts) } func Delete(ctx context.Context, data wshrpc.CommandDeleteFileData) error { diff --git a/pkg/remote/fileshare/fstype/fstype.go b/pkg/remote/fileshare/fstype/fstype.go index 6b42902c7..3c3d6fceb 100644 --- a/pkg/remote/fileshare/fstype/fstype.go +++ b/pkg/remote/fileshare/fstype/fstype.go @@ -30,10 +30,12 @@ type FileShareClient interface { AppendFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error // Mkdir creates a directory at the given path Mkdir(ctx context.Context, conn *connparse.Connection) error - // Move moves the file from srcConn to destConn - Move(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error - // Copy copies the file from srcConn to destConn - Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error + // Move moves the file within the same connection + MoveInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error + // Copy copies the file within the same connection + CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error + // CopyRemote copies the file between different connections + CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient FileShareClient, opts *wshrpc.FileCopyOpts) error // Delete deletes the entry at the given path Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error // Join joins the given parts to the connection path diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index e73c409c7..057b287e4 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -98,11 +98,15 @@ func (c S3Client) Mkdir(ctx context.Context, conn *connparse.Connection) error { return nil } -func (c S3Client) Move(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { +func (c S3Client) MoveInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { return nil } -func (c S3Client) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { +func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { + return nil +} + +func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { return nil } diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 9b13bb3a3..9be690ffa 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -320,14 +320,63 @@ func (c WaveClient) AppendFile(ctx context.Context, conn *connparse.Connection, // WaveFile does not support directories, only prefix-based listing func (c WaveClient) Mkdir(ctx context.Context, conn *connparse.Connection) error { - return nil + return errors.ErrUnsupported } -func (c WaveClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { +func (c WaveClient) MoveInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { + if srcConn.Host != destConn.Host { + return fmt.Errorf("move internal, src and dest hosts do not match") + } + err := c.CopyInternal(ctx, srcConn, destConn, opts) + if err != nil { + return fmt.Errorf("error copying blockfile: %w", err) + } + err = c.Delete(ctx, srcConn, opts.Recursive) + if err != nil { + return fmt.Errorf("error deleting blockfile: %w", err) + } return nil } -func (c WaveClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { +func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { + if srcConn.Host == destConn.Host { + host := srcConn.Host + srcFileName, err := cleanPath(srcConn.Path) + if err != nil { + return fmt.Errorf("error cleaning source path: %w", err) + } + destFileName, err := cleanPath(destConn.Path) + if err != nil { + return fmt.Errorf("error cleaning destination path: %w", err) + } + err = filestore.WFS.MakeFile(ctx, host, destFileName, wshrpc.FileMeta{}, wshrpc.FileOpts{}) + if err != nil { + return fmt.Errorf("error making source blockfile: %w", err) + } + _, dataBuf, err := filestore.WFS.ReadFile(ctx, host, srcFileName) + if err != nil { + return fmt.Errorf("error reading source blockfile: %w", err) + } + err = filestore.WFS.WriteFile(ctx, host, destFileName, dataBuf) + if err != nil { + return fmt.Errorf("error writing to destination blockfile: %w", err) + } + wps.Broker.Publish(wps.WaveEvent{ + Event: wps.Event_BlockFile, + Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, host).String()}, + Data: &wps.WSFileEventData{ + ZoneId: host, + FileName: destFileName, + FileOp: wps.FileOp_Invalidate, + }, + }) + return nil + } else { + return fmt.Errorf("copy between different hosts not supported") + } +} + +func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { return nil } diff --git a/pkg/remote/fileshare/wshfs/wshfs.go b/pkg/remote/fileshare/wshfs/wshfs.go index 4fcceabb0..61816ea57 100644 --- a/pkg/remote/fileshare/wshfs/wshfs.go +++ b/pkg/remote/fileshare/wshfs/wshfs.go @@ -146,7 +146,10 @@ func (c WshClient) Mkdir(ctx context.Context, conn *connparse.Connection) error return wshclient.RemoteMkdirCommand(RpcClient, conn.Path, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(conn.Host)}) } -func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { +func (c WshClient) MoveInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { + if srcConn.Host != destConn.Host { + return fmt.Errorf("move internal, src and dest hosts do not match") + } if opts == nil { opts = &wshrpc.FileCopyOpts{} } @@ -157,7 +160,11 @@ func (c WshClient) Move(ctx context.Context, srcConn, destConn *connparse.Connec return wshclient.RemoteFileMoveCommand(RpcClient, wshrpc.CommandRemoteFileCopyData{SrcUri: srcConn.GetFullURI(), DestUri: destConn.GetFullURI(), Opts: opts}, &wshrpc.RpcOpts{Route: wshutil.MakeConnectionRouteId(destConn.Host), Timeout: timeout}) } -func (c WshClient) Copy(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { +func (c WshClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, _ fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { + return c.CopyInternal(ctx, srcConn, destConn, opts) +} + +func (c WshClient) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { if opts == nil { opts = &wshrpc.FileCopyOpts{} } diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go new file mode 100644 index 000000000..e69ab67cb --- /dev/null +++ b/pkg/util/tarcopy/tarcopy.go @@ -0,0 +1,69 @@ +package tarcopy + +import ( + "archive/tar" + "context" + "io" + "io/fs" + "log" + "strings" + "time" + + "github.com/wavetermdev/waveterm/pkg/util/iochan" + "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" + "github.com/wavetermdev/waveterm/pkg/wshrpc" +) + +func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string) error, writer io.Writer, close func()) { + pipeReader, pipeWriter := io.Pipe() + tarWriter := tar.NewWriter(pipeWriter) + rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { + for { + if err := pipeReader.Close(); err != nil { + log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + break + } + }) + + return rtnChan, func(fi fs.FileInfo, file string) error { + // generate tar header + header, err := tar.FileInfoHeader(fi, file) + if err != nil { + return err + } + + header.Name = strings.TrimPrefix(file, pathPrefix) + if header.Name == "" { + return nil + } + if strings.HasPrefix(header.Name, "/") { + header.Name = header.Name[1:] + } + + // write header + if err := tarWriter.WriteHeader(header); err != nil { + return err + } + return nil + }, tarWriter, func() { + for { + if err := tarWriter.Close(); err != nil { + log.Printf("TarCopySrc: error closing tar writer: %v, trying again in 10ms\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + break + } + for { + if err := pipeWriter.Close(); err != nil { + log.Printf("TarCopySrc: error closing pipe writer: %v, trying again in 10ms\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + break + } + } +} diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index c59c85003..ae395652b 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -18,13 +18,15 @@ import ( "time" "github.com/wavetermdev/waveterm/pkg/remote/connparse" - "github.com/wavetermdev/waveterm/pkg/remote/fileshare" + "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" + "github.com/wavetermdev/waveterm/pkg/util/tarcopy" "github.com/wavetermdev/waveterm/pkg/util/utilfn" "github.com/wavetermdev/waveterm/pkg/wavebase" "github.com/wavetermdev/waveterm/pkg/wshrpc" + "github.com/wavetermdev/waveterm/pkg/wshrpc/wshclient" "github.com/wavetermdev/waveterm/pkg/wshutil" ) @@ -245,23 +247,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("cannot stat file %q: %w", path, err)) } - pipeReader, pipeWriter := io.Pipe() - tarWriter := tar.NewWriter(pipeWriter) - timeout := time.Millisecond * 100 - if opts.Timeout > 0 { - timeout = time.Duration(opts.Timeout) * time.Millisecond - } - readerCtx, cancel := context.WithTimeout(context.Background(), timeout) - rtn := iochan.ReaderChan(readerCtx, pipeReader, wshrpc.FileChunkSize, func() { - for { - if err := pipeReader.Close(); err != nil { - log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) - continue - } - break - } - }) var pathPrefix string if finfo.IsDir() && strings.HasSuffix(cleanedPath, "/") { @@ -269,52 +254,29 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } else { pathPrefix = filepath.Dir(cleanedPath) } + if finfo.IsDir() { + if !recursive { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) + } + } + + timeout := time.Millisecond * 100 + if opts.Timeout > 0 { + timeout = time.Duration(opts.Timeout) * time.Millisecond + } + readerCtx, cancel := context.WithTimeout(context.Background(), timeout) + rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, wshrpc.FileChunkSize, pathPrefix) + go func() { defer func() { - for { - if err := tarWriter.Close(); err != nil { - logPrintfDev("error closing tar writer: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) - continue - } - break - } - for { - if err := pipeWriter.Close(); err != nil { - logPrintfDev("error closing pipe writer: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) - continue - } - break - } + tarClose() cancel() }() if readerCtx.Err() != nil { return } - if finfo.IsDir() { - if !recursive { - rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, errors.New("directory copy requires recursive option"))) - return - } - } err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { - // generate tar header - header, err := tar.FileInfoHeader(fi, file) - if err != nil { - return err - } - - header.Name = strings.TrimPrefix(file, pathPrefix) - if header.Name == "" { - return nil - } - if strings.HasPrefix(header.Name, "/") { - header.Name = header.Name[1:] - } - - // write header - if err := tarWriter.WriteHeader(header); err != nil { + if err = writeHeader(fi, file); err != nil { return err } // if not a dir, write file content @@ -323,7 +285,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if err != nil { return err } - if _, err := io.Copy(tarWriter, data); err != nil { + if _, err := io.Copy(fileWriter, data); err != nil { log.Printf("error copying file %q: %v\n", file, err) return err } @@ -389,7 +351,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C readCtx, timeoutCancel := context.WithTimeoutCause(readCtx, timeout, fmt.Errorf("timeout copying file %q to %q", srcUri, destUri)) defer timeoutCancel() copyStart := time.Now() - ioch := fileshare.ReadTarStream(readCtx, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}) + ioch := wshclient.FileStreamTarCommand(wshfs.RpcClient, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}, &wshrpc.RpcOpts{Timeout: opts.Timeout}) pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(readCtx, pipeWriter, ioch, func() { for { @@ -490,11 +452,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C } } else if !overwrite { return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) - } else { - err := os.Remove(nextPath) - if err != nil { - return fmt.Errorf("cannot remove file %q: %w", nextPath, err) - } } } } else { @@ -508,7 +465,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil { return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) } - file, err := os.Create(nextPath) + file, err := os.OpenFile(nextPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, finfo.Mode()) if err != nil { return fmt.Errorf("cannot create new file %q: %w", nextPath, err) } @@ -516,7 +473,6 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil { return fmt.Errorf("cannot write file %q: %w", nextPath, err) } - file.Chmod(finfo.Mode()) file.Close() } } @@ -753,14 +709,7 @@ func (impl *ServerImpl) RemoteFileMoveCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot move file %q to %q: %w", srcPathCleaned, destPathCleaned, err) } } else { - err := impl.RemoteFileCopyCommand(ctx, data) - if err != nil { - return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) - } - err = fileshare.Delete(ctx, wshrpc.CommandDeleteFileData{Path: srcUri, Recursive: data.Opts.Recursive}) - if err != nil { - return fmt.Errorf("cannot delete %q: %w", srcUri, err) - } + return fmt.Errorf("cannot move file %q to %q: different hosts", srcUri, destUri) } return nil } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index e2fc7403a..b601ddf42 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -66,6 +66,7 @@ const ( Command_FileRead = "fileread" Command_FileMove = "filemove" Command_FileCopy = "filecopy" + Command_FileStreamTar = "filestreamtar" Command_EventPublish = "eventpublish" Command_EventRecv = "eventrecv" Command_EventSub = "eventsub" From 52348c4c7b268f0c0e9878b339cc4ff622c32d50 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 16:53:07 -0800 Subject: [PATCH 102/126] move dest of copy to tarcopy --- pkg/util/tarcopy/tarcopy.go | 55 ++++++++++ pkg/wshrpc/wshremote/wshremote.go | 175 +++++++++++------------------- 2 files changed, 120 insertions(+), 110 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index e69ab67cb..f1a4e9a67 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -3,6 +3,8 @@ package tarcopy import ( "archive/tar" "context" + "errors" + "fmt" "io" "io/fs" "log" @@ -67,3 +69,56 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh } } } + +func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader) error) error { + pipeReader, pipeWriter := io.Pipe() + iochan.WriterChan(ctx, pipeWriter, ch, func() { + for { + if err := pipeWriter.Close(); err != nil { + log.Printf("error closing pipe writer: %v, trying again in 10ms\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + cancel(nil) + break + } + }, cancel) + tarReader := tar.NewReader(pipeReader) + defer func() { + for { + if err := pipeReader.Close(); err != nil { + log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) + time.Sleep(time.Millisecond * 10) + continue + } + cancel(nil) + break + } + }() + for { + select { + case <-ctx.Done(): + if ctx.Err() != nil { + return context.Cause(ctx) + } + return nil + default: + next, err := tarReader.Next() + if err != nil { + // Do one more check for context error before returning + if ctx.Err() != nil { + return context.Cause(ctx) + } + if errors.Is(err, io.EOF) { + return nil + } else { + return fmt.Errorf("cannot read tar stream: %w", err) + } + } + err = readNext(next, tarReader) + if err != nil { + return err + } + } + } +} diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index ae395652b..0d9cc7d85 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -20,7 +20,6 @@ import ( "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/wshfs" "github.com/wavetermdev/waveterm/pkg/util/fileutil" - "github.com/wavetermdev/waveterm/pkg/util/iochan" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" "github.com/wavetermdev/waveterm/pkg/util/tarcopy" "github.com/wavetermdev/waveterm/pkg/util/utilfn" @@ -352,132 +351,88 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C defer timeoutCancel() copyStart := time.Now() ioch := wshclient.FileStreamTarCommand(wshfs.RpcClient, wshrpc.CommandRemoteStreamTarData{Path: srcUri, Opts: opts}, &wshrpc.RpcOpts{Timeout: opts.Timeout}) - pipeReader, pipeWriter := io.Pipe() - iochan.WriterChan(readCtx, pipeWriter, ioch, func() { - for { - if err := pipeWriter.Close(); err != nil { - log.Printf("error closing pipe writer: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) - continue - } - timeoutCancel() - break - } - }, cancel) - tarReader := tar.NewReader(pipeReader) - defer func() { - for { - if err := pipeReader.Close(); err != nil { - log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) - continue - } - timeoutCancel() - break - } - }() numFiles := 0 numSkipped := 0 totalBytes := int64(0) - logResult := func() { - log.Printf("RemoteFileCopyCommand: done; %d files copied in %dms, total of %d bytes, %d files skipped\n", numFiles, time.Since(copyStart).Milliseconds(), totalBytes, numSkipped) - } - for { - select { - case <-readCtx.Done(): - if readCtx.Err() != nil { - return context.Cause(readCtx) - } - logResult() + err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { + // Check for directory traversal + if strings.Contains(next.Name, "..") { + log.Printf("skipping file with unsafe path: %q\n", next.Name) + numSkipped++ return nil - default: - next, err := tarReader.Next() - if err != nil { - // Do one more check for context error before returning - if readCtx.Err() != nil { - return context.Cause(readCtx) - } - if errors.Is(err, io.EOF) { - logResult() - return nil - } else { - return fmt.Errorf("cannot read tar stream: %w", err) - } - } - // Check for directory traversal - if strings.Contains(next.Name, "..") { - log.Printf("skipping file with unsafe path: %q\n", next.Name) - numSkipped++ - continue - } - numFiles++ - finfo := next.FileInfo() - nextPath := filepath.Join(destPathCleaned, next.Name) - destinfo, err = os.Stat(nextPath) - if err != nil && !errors.Is(err, fs.ErrNotExist) { - return fmt.Errorf("cannot stat file %q: %w", nextPath, err) - } - if !finfo.IsDir() { - totalBytes += finfo.Size() - } + } + numFiles++ + finfo := next.FileInfo() + nextPath := filepath.Join(destPathCleaned, next.Name) + destinfo, err = os.Stat(nextPath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("cannot stat file %q: %w", nextPath, err) + } + if !finfo.IsDir() { + totalBytes += finfo.Size() + } - if destinfo != nil { - if destinfo.IsDir() { - if !finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) - } else { - err := os.Remove(nextPath) - if err != nil { - return fmt.Errorf("cannot remove file %q: %w", nextPath, err) - } - } - } else if !merge && !overwrite { - return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) - } else if overwrite { - err := os.RemoveAll(nextPath) + if destinfo != nil { + if destinfo.IsDir() { + if !finfo.IsDir() { + if !overwrite { + return fmt.Errorf("cannot create directory %q, file exists at path, overwrite not specified", nextPath) + } else { + err := os.Remove(nextPath) if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + return fmt.Errorf("cannot remove file %q: %w", nextPath, err) } } - } else { - if finfo.IsDir() { - if !overwrite { - return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) - } else { - err := os.RemoveAll(nextPath) - if err != nil { - return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) - } - } - } else if !overwrite { - return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) + } else if !merge && !overwrite { + return fmt.Errorf("cannot create directory %q, directory exists at path, neither overwrite nor merge specified", nextPath) + } else if overwrite { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) } } } else { if finfo.IsDir() { - err := os.MkdirAll(nextPath, finfo.Mode()) - if err != nil { - return fmt.Errorf("cannot create directory %q: %w", nextPath, err) - } - } else { - err := os.MkdirAll(filepath.Dir(nextPath), 0755) - if err != nil { - return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) - } - file, err := os.OpenFile(nextPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, finfo.Mode()) - if err != nil { - return fmt.Errorf("cannot create new file %q: %w", nextPath, err) - } - _, err = io.Copy(file, tarReader) - if err != nil { - return fmt.Errorf("cannot write file %q: %w", nextPath, err) + if !overwrite { + return fmt.Errorf("cannot create file %q, directory exists at path, overwrite not specified", nextPath) + } else { + err := os.RemoveAll(nextPath) + if err != nil { + return fmt.Errorf("cannot remove directory %q: %w", nextPath, err) + } } - file.Close() + } else if !overwrite { + return fmt.Errorf("cannot create file %q, file exists at path, overwrite not specified", nextPath) } } + } else { + if finfo.IsDir() { + err := os.MkdirAll(nextPath, finfo.Mode()) + if err != nil { + return fmt.Errorf("cannot create directory %q: %w", nextPath, err) + } + } else { + err := os.MkdirAll(filepath.Dir(nextPath), 0755) + if err != nil { + return fmt.Errorf("cannot create parent directory %q: %w", filepath.Dir(nextPath), err) + } + file, err := os.OpenFile(nextPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, finfo.Mode()) + if err != nil { + return fmt.Errorf("cannot create new file %q: %w", nextPath, err) + } + _, err = io.Copy(file, reader) + if err != nil { + return fmt.Errorf("cannot write file %q: %w", nextPath, err) + } + file.Close() + } } + return nil + }) + if err != nil { + return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) } + log.Printf("RemoteFileCopyCommand: done; %d files copied in %dms, total of %d bytes, %d files skipped\n", numFiles, time.Since(copyStart).Milliseconds(), totalBytes, numSkipped) } return nil } From f7351ad5390f11bcab457180a7edaf6c6db9085a Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 16:55:12 -0800 Subject: [PATCH 103/126] clean filepath in tarcopy --- pkg/util/tarcopy/tarcopy.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index f1a4e9a67..271e69669 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -8,6 +8,7 @@ import ( "io" "io/fs" "log" + "path/filepath" "strings" "time" @@ -37,7 +38,7 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh return err } - header.Name = strings.TrimPrefix(file, pathPrefix) + header.Name = filepath.Clean(strings.TrimPrefix(file, pathPrefix)) if header.Name == "" { return nil } From a8daf5299acec4aaa8e226febf8104d214f728e1 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 16:55:52 -0800 Subject: [PATCH 104/126] update error msg --- pkg/util/tarcopy/tarcopy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 271e69669..b90938777 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -23,7 +23,7 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { for { if err := pipeReader.Close(); err != nil { - log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) + log.Printf("TarCopySrc: error closing pipe reader: %v, trying again in 10ms\n", err) time.Sleep(time.Millisecond * 10) continue } @@ -76,7 +76,7 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan iochan.WriterChan(ctx, pipeWriter, ch, func() { for { if err := pipeWriter.Close(); err != nil { - log.Printf("error closing pipe writer: %v, trying again in 10ms\n", err) + log.Printf("TarCopyDest: error closing pipe writer: %v, trying again in 10ms\n", err) time.Sleep(time.Millisecond * 10) continue } From 0b4691a6c17a4c0043346bf1dfe9857d58979c08 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 16:56:45 -0800 Subject: [PATCH 105/126] Update cmd/wsh/cmd/wshcmd-file.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- cmd/wsh/cmd/wshcmd-file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/wsh/cmd/wshcmd-file.go b/cmd/wsh/cmd/wshcmd-file.go index 8bacc1ae3..62e60474e 100644 --- a/cmd/wsh/cmd/wshcmd-file.go +++ b/cmd/wsh/cmd/wshcmd-file.go @@ -98,7 +98,7 @@ func init() { fileCmd.AddCommand(fileListCmd) fileCmd.AddCommand(fileCatCmd) fileCmd.AddCommand(fileWriteCmd) - fileRmCmd.Flags().BoolP("recursive", "r", false, "copy directories recursively") + fileRmCmd.Flags().BoolP("recursive", "r", false, "remove directories recursively") fileCmd.AddCommand(fileRmCmd) fileCmd.AddCommand(fileInfoCmd) fileCmd.AddCommand(fileAppendCmd) From d8770a36ccf996498f2198a556dcaa1aa7c49127 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 17:00:33 -0800 Subject: [PATCH 106/126] handle err in walk --- pkg/wshrpc/wshremote/wshremote.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 0d9cc7d85..206ae71f4 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -275,6 +275,12 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. return } err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { + if readerCtx.Err() != nil { + return readerCtx.Err() + } + if err != nil { + return err + } if err = writeHeader(fi, file); err != nil { return err } From 2db3666fdc4bac3bce71983cef891e5e49f950ae Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 17:01:22 -0800 Subject: [PATCH 107/126] remove unnecessary ctx check --- pkg/wshrpc/wshremote/wshremote.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 206ae71f4..ed49bc858 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -271,9 +271,6 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. tarClose() cancel() }() - if readerCtx.Err() != nil { - return - } err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { if readerCtx.Err() != nil { return readerCtx.Err() From 3485489f9a93ba0d8b017ba297a434620f50f75a Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 17:30:38 -0800 Subject: [PATCH 108/126] Add wavefs tar stream --- pkg/remote/fileshare/wavefs/wavefs.go | 97 ++++++++++++++++++++++++++- pkg/util/fileutil/fileutil.go | 46 +++++++++++++ pkg/util/iochan/iochan.go | 10 +-- 3 files changed, 147 insertions(+), 6 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 9be690ffa..8ddea96bf 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -4,6 +4,7 @@ package wavefs import ( + "archive/tar" "context" "encoding/base64" "errors" @@ -11,11 +12,14 @@ import ( "io/fs" "path" "strings" + "time" "github.com/wavetermdev/waveterm/pkg/filestore" "github.com/wavetermdev/waveterm/pkg/remote/connparse" "github.com/wavetermdev/waveterm/pkg/remote/fileshare/fstype" + "github.com/wavetermdev/waveterm/pkg/util/fileutil" "github.com/wavetermdev/waveterm/pkg/util/iochan/iochantypes" + "github.com/wavetermdev/waveterm/pkg/util/tarcopy" "github.com/wavetermdev/waveterm/pkg/util/wavefileutil" "github.com/wavetermdev/waveterm/pkg/waveobj" "github.com/wavetermdev/waveterm/pkg/wps" @@ -97,7 +101,54 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w } func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { - return nil + pathPrefix, err := cleanPath(conn.Path) + if err != nil { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error cleaning path: %w", err)) + } + list, err := c.ListEntries(ctx, conn, nil) + if err != nil { + return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error listing blockfiles: %w", err)) + } + + timeout := time.Millisecond * 100 + if opts.Timeout > 0 { + timeout = time.Duration(opts.Timeout) * time.Millisecond + } + readerCtx, cancel := context.WithTimeout(context.Background(), timeout) + rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, wshrpc.FileChunkSize, pathPrefix) + + go func() { + defer func() { + tarClose() + cancel() + }() + for _, file := range list { + if readerCtx.Err() != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](readerCtx.Err()) + return + } + + if err = writeHeader(fileutil.ToFsFileInfo(file), file.Path); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("error writing tar header: %w", err)) + return + } + if file.IsDir { + continue + } + + _, dataBuf, err := filestore.WFS.ReadFile(ctx, conn.Host, file.Path) + if err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("error reading blockfile: %w", err)) + return + } + if _, err = fileWriter.Write(dataBuf); err != nil { + rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("error writing tar data: %w", err)) + return + } + } + }() + + return rtn } func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { @@ -377,6 +428,50 @@ func (c WaveClient) CopyInternal(ctx context.Context, srcConn, destConn *connpar } func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { + zoneId := destConn.Host + if zoneId == "" { + return fmt.Errorf("zoneid not found in connection") + } + readCtx, cancel := context.WithCancelCause(ctx) + ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) + err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { + if next.Typeflag == tar.TypeDir { + return nil + } + fileName, err := cleanPath(path.Join(destConn.Path, next.Name)) + _, err = filestore.WFS.Stat(ctx, zoneId, fileName) + if err != nil { + if !errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("error getting blockfile info: %w", err) + } + err := filestore.WFS.MakeFile(ctx, zoneId, fileName, nil, wshrpc.FileOpts{}) + if err != nil { + return fmt.Errorf("error making blockfile: %w", err) + } + } + dataBuf := make([]byte, next.Size) + n, err := reader.Read(dataBuf) + if err != nil { + return fmt.Errorf("error reading tar data: %w", err) + } + err = filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf[:n]) + if err != nil { + return fmt.Errorf("error writing to blockfile: %w", err) + } + wps.Broker.Publish(wps.WaveEvent{ + Event: wps.Event_BlockFile, + Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, zoneId).String()}, + Data: &wps.WSFileEventData{ + ZoneId: zoneId, + FileName: fileName, + FileOp: wps.FileOp_Invalidate, + }, + }) + return nil + }) + if err != nil { + return fmt.Errorf("error copying tar stream: %w", err) + } return nil } diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index b5bbfbb1b..72a979eae 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -13,8 +13,10 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/wavetermdev/waveterm/pkg/wavebase" + "github.com/wavetermdev/waveterm/pkg/wshrpc" ) func FixPath(path string) (string, error) { @@ -164,3 +166,47 @@ func IsInitScriptPath(input string) bool { return true } + +type FsFileInfo struct { + NameInternal string + ModeInternal os.FileMode + SizeInternal int64 + ModTimeInternal int64 + IsDirInternal bool +} + +func (f FsFileInfo) Name() string { + return f.NameInternal +} + +func (f FsFileInfo) Size() int64 { + return f.SizeInternal +} + +func (f FsFileInfo) Mode() os.FileMode { + return f.ModeInternal +} + +func (f FsFileInfo) ModTime() time.Time { + return time.Unix(0, f.ModTimeInternal) +} + +func (f FsFileInfo) IsDir() bool { + return f.IsDirInternal +} + +func (f FsFileInfo) Sys() interface{} { + return nil +} + +var _ fs.FileInfo = FsFileInfo{} + +func ToFsFileInfo(fi *wshrpc.FileInfo) FsFileInfo { + return FsFileInfo{ + NameInternal: fi.Name, + ModeInternal: fi.Mode, + SizeInternal: fi.Size, + ModTimeInternal: fi.ModTime, + IsDirInternal: fi.IsDir, + } +} diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index 893aff9ab..e35f65cdc 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -56,7 +56,7 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func } // WriterChan reads from a channel and writes the data to an io.Writer -func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], callback func(), errCallback func(error)) { +func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], callback func(), cancel context.CancelCauseFunc) { sha256Hash := sha256.New() go func() { defer func() { @@ -72,23 +72,23 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn return } if resp.Error != nil { - errCallback(resp.Error) + cancel(resp.Error) return } if _, err := sha256Hash.Write(resp.Response.Data); err != nil { - errCallback(fmt.Errorf("WriterChan: error writing to sha256 hash: %v", err)) + cancel(fmt.Errorf("WriterChan: error writing to sha256 hash: %v", err)) return } // The checksum is sent as the last packet if resp.Response.Checksum != nil { localChecksum := sha256Hash.Sum(nil) if !bytes.Equal(localChecksum, resp.Response.Checksum) { - errCallback(fmt.Errorf("WriterChan: checksum mismatch")) + cancel(fmt.Errorf("WriterChan: checksum mismatch")) } return } if _, err := w.Write(resp.Response.Data); err != nil { - errCallback(fmt.Errorf("WriterChan: write error: %v", err)) + cancel(fmt.Errorf("WriterChan: write error: %v", err)) return } } From 0c44c24ee79149293487804a7d2e20f8ff592f46 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 18:15:09 -0800 Subject: [PATCH 109/126] add tar stream for wavefile --- pkg/remote/connparse/connparse.go | 4 +++ pkg/remote/fileshare/wavefs/wavefs.go | 48 ++++++++++++++++++++++----- pkg/util/tarcopy/tarcopy.go | 30 +++++++++-------- 3 files changed, 59 insertions(+), 23 deletions(-) diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index e4cadba91..892d9fb02 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -57,6 +57,10 @@ func (c *Connection) GetFullURI() string { return c.Scheme + "://" + c.GetPathWithHost() } +func (c *Connection) GetSchemeAndHost() string { + return c.Scheme + "://" + c.Host +} + func ParseURIAndReplaceCurrentHost(ctx context.Context, uri string) (*Connection, error) { conn, err := ParseURI(uri) if err != nil { diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 8ddea96bf..520e72516 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -9,7 +9,9 @@ import ( "encoding/base64" "errors" "fmt" + "io" "io/fs" + "log" "path" "strings" "time" @@ -101,15 +103,15 @@ func (c WaveClient) Read(ctx context.Context, conn *connparse.Connection, data w } func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { - pathPrefix, err := cleanPath(conn.Path) - if err != nil { - return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error cleaning path: %w", err)) - } + log.Printf("ReadTarStream: conn: %v, opts: %v\n", conn, opts) list, err := c.ListEntries(ctx, conn, nil) if err != nil { return wshutil.SendErrCh[iochantypes.Packet](fmt.Errorf("error listing blockfiles: %w", err)) } + pathPrefix := getPathPrefix(conn) + schemeAndHost := conn.GetSchemeAndHost() + "/" + timeout := time.Millisecond * 100 if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond @@ -127,6 +129,7 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio rtn <- wshutil.RespErr[iochantypes.Packet](readerCtx.Err()) return } + file.Mode = 0644 if err = writeHeader(fileutil.ToFsFileInfo(file), file.Path); err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("error writing tar header: %w", err)) @@ -136,7 +139,11 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio continue } - _, dataBuf, err := filestore.WFS.ReadFile(ctx, conn.Host, file.Path) + log.Printf("ReadTarStream: reading file: %s\n", file.Path) + + internalPath := strings.TrimPrefix(file.Path, schemeAndHost) + + _, dataBuf, err := filestore.WFS.ReadFile(ctx, conn.Host, internalPath) if err != nil { rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("error reading blockfile: %w", err)) return @@ -168,10 +175,14 @@ func (c WaveClient) ListEntriesStream(ctx context.Context, conn *connparse.Conne } func (c WaveClient) ListEntries(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) ([]*wshrpc.FileInfo, error) { + log.Printf("ListEntries: conn: %v, opts: %v\n", conn, opts) zoneId := conn.Host if zoneId == "" { return nil, fmt.Errorf("zoneid not found in connection") } + if opts == nil { + opts = &wshrpc.FileListOpts{} + } prefix, err := cleanPath(conn.Path) if err != nil { return nil, fmt.Errorf("error cleaning path: %w", err) @@ -432,13 +443,19 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } + destPrefix := getPathPrefix(destConn) + destPrefix = strings.TrimPrefix(destPrefix, destConn.GetSchemeAndHost()+"/") + log.Printf("CopyRemote: srcConn: %v, destConn: %v, destPrefix: %s\n", srcConn, destConn, destPrefix) readCtx, cancel := context.WithCancelCause(ctx) ioch := srcClient.ReadTarStream(readCtx, srcConn, opts) err := tarcopy.TarCopyDest(readCtx, cancel, ioch, func(next *tar.Header, reader *tar.Reader) error { if next.Typeflag == tar.TypeDir { return nil } - fileName, err := cleanPath(path.Join(destConn.Path, next.Name)) + fileName, err := cleanPath(path.Join(destPrefix, next.Name)) + if err != nil { + return fmt.Errorf("error cleaning path: %w", err) + } _, err = filestore.WFS.Stat(ctx, zoneId, fileName) if err != nil { if !errors.Is(err, fs.ErrNotExist) { @@ -449,12 +466,15 @@ func (c WaveClient) CopyRemote(ctx context.Context, srcConn, destConn *connparse return fmt.Errorf("error making blockfile: %w", err) } } + log.Printf("CopyRemote: writing file: %s; size: %d\n", fileName, next.Size) dataBuf := make([]byte, next.Size) - n, err := reader.Read(dataBuf) + _, err = reader.Read(dataBuf) if err != nil { - return fmt.Errorf("error reading tar data: %w", err) + if !errors.Is(err, io.EOF) { + return fmt.Errorf("error reading tar data: %w", err) + } } - err = filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf[:n]) + err = filestore.WFS.WriteFile(ctx, zoneId, fileName, dataBuf) if err != nil { return fmt.Errorf("error writing to blockfile: %w", err) } @@ -535,3 +555,13 @@ func cleanPath(path string) (string, error) { func (c WaveClient) GetConnectionType() string { return connparse.ConnectionTypeWave } + +func getPathPrefix(conn *connparse.Connection) string { + fullUri := conn.GetFullURI() + pathPrefix := fullUri + lastSlash := strings.LastIndex(fullUri, "/") + if lastSlash > 10 && lastSlash < len(fullUri)-1 { + pathPrefix = fullUri[:lastSlash+1] + } + return pathPrefix +} diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index b90938777..7748461d5 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -17,14 +17,19 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshrpc" ) +const ( + maxRetries = 5 + retryDelay = 10 * time.Millisecond +) + func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string) error, writer io.Writer, close func()) { pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { - for { + for retries := 0; retries < maxRetries; retries++ { if err := pipeReader.Close(); err != nil { log.Printf("TarCopySrc: error closing pipe reader: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) + time.Sleep(retryDelay) continue } break @@ -52,18 +57,18 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh } return nil }, tarWriter, func() { - for { + for retries := 0; retries < maxRetries; retries++ { if err := tarWriter.Close(); err != nil { log.Printf("TarCopySrc: error closing tar writer: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) + time.Sleep(retryDelay) continue } break } - for { + for retries := 0; retries < maxRetries; retries++ { if err := pipeWriter.Close(); err != nil { log.Printf("TarCopySrc: error closing pipe writer: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) + time.Sleep(retryDelay) continue } break @@ -74,10 +79,10 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader) error) error { pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { - for { + for retries := 0; retries < maxRetries; retries++ { if err := pipeWriter.Close(); err != nil { log.Printf("TarCopyDest: error closing pipe writer: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) + time.Sleep(retryDelay) continue } cancel(nil) @@ -86,10 +91,10 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { - for { + for retries := 0; retries < maxRetries; retries++ { if err := pipeReader.Close(); err != nil { log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) - time.Sleep(time.Millisecond * 10) + time.Sleep(retryDelay) continue } cancel(nil) @@ -99,16 +104,13 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan for { select { case <-ctx.Done(): - if ctx.Err() != nil { - return context.Cause(ctx) - } return nil default: next, err := tarReader.Next() if err != nil { // Do one more check for context error before returning if ctx.Err() != nil { - return context.Cause(ctx) + return nil } if errors.Is(err, io.EOF) { return nil From f1494736a2bbff0d920a8ffb666cff18d704bd8d Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 18:18:55 -0800 Subject: [PATCH 110/126] add unsupported errors for s3 impl until ready --- pkg/remote/fileshare/s3fs/s3fs.go | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/pkg/remote/fileshare/s3fs/s3fs.go b/pkg/remote/fileshare/s3fs/s3fs.go index 057b287e4..b406615d4 100644 --- a/pkg/remote/fileshare/s3fs/s3fs.go +++ b/pkg/remote/fileshare/s3fs/s3fs.go @@ -5,6 +5,7 @@ package s3fs import ( "context" + "errors" "log" "github.com/aws/aws-sdk-go-v2/aws" @@ -30,15 +31,15 @@ func NewS3Client(config *aws.Config) *S3Client { } func (c S3Client) Read(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) (*wshrpc.FileData, error) { - return nil, nil + return nil, errors.ErrUnsupported } func (c S3Client) ReadStream(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) <-chan wshrpc.RespOrErrorUnion[wshrpc.FileData] { - return nil + return wshutil.SendErrCh[wshrpc.FileData](errors.ErrUnsupported) } func (c S3Client) ReadTarStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileCopyOpts) <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { - return nil + return wshutil.SendErrCh[iochantypes.Packet](errors.ErrUnsupported) } func (c S3Client) ListEntriesStream(ctx context.Context, conn *connparse.Connection, opts *wshrpc.FileListOpts) <-chan wshrpc.RespOrErrorUnion[wshrpc.CommandRemoteListEntriesRtnData] { @@ -83,39 +84,39 @@ func (c S3Client) ListEntries(ctx context.Context, conn *connparse.Connection, o } func (c S3Client) Stat(ctx context.Context, conn *connparse.Connection) (*wshrpc.FileInfo, error) { - return nil, nil + return nil, errors.ErrUnsupported } func (c S3Client) PutFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { - return nil + return errors.ErrUnsupported } func (c S3Client) AppendFile(ctx context.Context, conn *connparse.Connection, data wshrpc.FileData) error { - return nil + return errors.ErrUnsupported } func (c S3Client) Mkdir(ctx context.Context, conn *connparse.Connection) error { - return nil + return errors.ErrUnsupported } func (c S3Client) MoveInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return nil + return errors.ErrUnsupported } func (c S3Client) CopyRemote(ctx context.Context, srcConn, destConn *connparse.Connection, srcClient fstype.FileShareClient, opts *wshrpc.FileCopyOpts) error { - return nil + return errors.ErrUnsupported } func (c S3Client) CopyInternal(ctx context.Context, srcConn, destConn *connparse.Connection, opts *wshrpc.FileCopyOpts) error { - return nil + return errors.ErrUnsupported } func (c S3Client) Delete(ctx context.Context, conn *connparse.Connection, recursive bool) error { - return nil + return errors.ErrUnsupported } func (c S3Client) Join(ctx context.Context, conn *connparse.Connection, parts ...string) (string, error) { - return "", nil + return "", errors.ErrUnsupported } func (c S3Client) GetConnectionType() string { From a141d52916dc102bf4939fb5c5276c70bce6bedd Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 18:22:05 -0800 Subject: [PATCH 111/126] add "recursive" delete for wavefs --- pkg/remote/fileshare/wavefs/wavefs.go | 38 +++++++++++++++++---------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 520e72516..03ef137dd 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -500,23 +500,33 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu if zoneId == "" { return fmt.Errorf("zoneid not found in connection") } - fileName, err := cleanPath(conn.Path) + schemeAndHost := conn.GetSchemeAndHost() + "/" + + entries, err := c.ListEntries(ctx, conn, nil) if err != nil { - return fmt.Errorf("error cleaning path: %w", err) + return fmt.Errorf("error listing blockfiles: %w", err) } - err = filestore.WFS.DeleteFile(ctx, zoneId, fileName) - if err != nil { - return fmt.Errorf("error deleting blockfile: %w", err) + if len(entries) > 0 { + if !recursive { + return fmt.Errorf("more than one entry, use recursive flag to delete") + } + for _, entry := range entries { + fileName := strings.TrimPrefix(entry.Path, schemeAndHost) + err = filestore.WFS.DeleteFile(ctx, zoneId, fileName) + if err != nil { + return fmt.Errorf("error deleting blockfile: %w", err) + } + wps.Broker.Publish(wps.WaveEvent{ + Event: wps.Event_BlockFile, + Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, zoneId).String()}, + Data: &wps.WSFileEventData{ + ZoneId: zoneId, + FileName: fileName, + FileOp: wps.FileOp_Delete, + }, + }) + } } - wps.Broker.Publish(wps.WaveEvent{ - Event: wps.Event_BlockFile, - Scopes: []string{waveobj.MakeORef(waveobj.OType_Block, zoneId).String()}, - Data: &wps.WSFileEventData{ - ZoneId: zoneId, - FileName: fileName, - FileOp: wps.FileOp_Delete, - }, - }) return nil } From a4e6d5242e8fe85bbcbac09fbf37c46b39f3ed55 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 18:25:38 -0800 Subject: [PATCH 112/126] add comments to tarcopy --- pkg/util/tarcopy/tarcopy.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 7748461d5..12b13bf92 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -1,3 +1,7 @@ +// Copyright 2025, Command Line Inc. +// SPDX-License-Identifier: Apache-2.0 + +// Package tarcopy provides functions for copying files over a channel via a tar stream. package tarcopy import ( @@ -22,6 +26,10 @@ const ( retryDelay = 10 * time.Millisecond ) +// TarCopySrc creates a tar stream writer and returns a channel to send the tar stream to. +// writeHeader is a function that writes the tar header for the file. +// writer is the tar writer to write the file data to. +// close is a function that closes the tar writer and internal pipe writer. func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string) error, writer io.Writer, close func()) { pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) @@ -76,6 +84,9 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh } } +// TarCopyDest reads a tar stream from a channel and writes the files to the destination. +// readNext is a function that is called for each file in the tar stream to read the file data. It should return an error if the file cannot be read. +// The function returns an error if the tar stream cannot be read. func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader) error) error { pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { From 9dc8ebf1430cfb7d6d1ce658e2bddd03594026b4 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 20:07:22 -0800 Subject: [PATCH 113/126] simplify graceful close logic, add log if fails after n retries --- pkg/util/tarcopy/tarcopy.go | 74 +++++++++++++++---------------------- 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 12b13bf92..e0513267b 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -22,8 +22,13 @@ import ( ) const ( - maxRetries = 5 - retryDelay = 10 * time.Millisecond + maxRetries = 5 + retryDelay = 10 * time.Millisecond + tarCopySrcName = "TarCopySrc" + tarCopyDestName = "TarCopyDest" + pipeReaderName = "pipe reader" + pipeWriterName = "pipe writer" + tarWriterName = "tar writer" ) // TarCopySrc creates a tar stream writer and returns a channel to send the tar stream to. @@ -34,14 +39,7 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { - for retries := 0; retries < maxRetries; retries++ { - if err := pipeReader.Close(); err != nil { - log.Printf("TarCopySrc: error closing pipe reader: %v, trying again in 10ms\n", err) - time.Sleep(retryDelay) - continue - } - break - } + gracefulClose(pipeReader, tarCopySrcName, pipeReaderName) }) return rtnChan, func(fi fs.FileInfo, file string) error { @@ -65,22 +63,8 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh } return nil }, tarWriter, func() { - for retries := 0; retries < maxRetries; retries++ { - if err := tarWriter.Close(); err != nil { - log.Printf("TarCopySrc: error closing tar writer: %v, trying again in 10ms\n", err) - time.Sleep(retryDelay) - continue - } - break - } - for retries := 0; retries < maxRetries; retries++ { - if err := pipeWriter.Close(); err != nil { - log.Printf("TarCopySrc: error closing pipe writer: %v, trying again in 10ms\n", err) - time.Sleep(retryDelay) - continue - } - break - } + gracefulClose(tarWriter, tarCopySrcName, tarWriterName) + gracefulClose(pipeWriter, tarCopySrcName, pipeWriterName) } } @@ -90,27 +74,13 @@ func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputCh func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], readNext func(next *tar.Header, reader *tar.Reader) error) error { pipeReader, pipeWriter := io.Pipe() iochan.WriterChan(ctx, pipeWriter, ch, func() { - for retries := 0; retries < maxRetries; retries++ { - if err := pipeWriter.Close(); err != nil { - log.Printf("TarCopyDest: error closing pipe writer: %v, trying again in 10ms\n", err) - time.Sleep(retryDelay) - continue - } - cancel(nil) - break - } + gracefulClose(pipeWriter, tarCopyDestName, pipeWriterName) + cancel(nil) }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { - for retries := 0; retries < maxRetries; retries++ { - if err := pipeReader.Close(); err != nil { - log.Printf("error closing pipe reader: %v, trying again in 10ms\n", err) - time.Sleep(retryDelay) - continue - } - cancel(nil) - break - } + gracefulClose(pipeReader, tarCopyDestName, pipeReaderName) + cancel(nil) }() for { select { @@ -136,3 +106,19 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan } } } + +func gracefulClose(closer io.Closer, debugName string, closerName string) { + closed := false + for retries := 0; retries < maxRetries; retries++ { + if err := closer.Close(); err != nil { + log.Printf("%s: error closing %s: %v, trying again in %dms\n", debugName, closerName, err, retryDelay.Milliseconds()) + time.Sleep(retryDelay) + continue + } + closed = true + break + } + if !closed { + log.Printf("%s: unable to close %s after %d retries\n", debugName, closerName, maxRetries) + } +} From 8a8225abca05cedd74574e73ba93642def8fde76 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 20:10:37 -0800 Subject: [PATCH 114/126] don't need to cancel the writer context in defer if the pipereader closes successfully --- pkg/util/tarcopy/tarcopy.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index e0513267b..bded4dcbb 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -79,8 +79,10 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan }, cancel) tarReader := tar.NewReader(pipeReader) defer func() { - gracefulClose(pipeReader, tarCopyDestName, pipeReaderName) - cancel(nil) + if !gracefulClose(pipeReader, tarCopyDestName, pipeReaderName) { + // If the pipe reader cannot be closed, cancel the context. This should kill the writer goroutine. + cancel(nil) + } }() for { select { @@ -107,7 +109,7 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan } } -func gracefulClose(closer io.Closer, debugName string, closerName string) { +func gracefulClose(closer io.Closer, debugName string, closerName string) bool { closed := false for retries := 0; retries < maxRetries; retries++ { if err := closer.Close(); err != nil { @@ -121,4 +123,5 @@ func gracefulClose(closer io.Closer, debugName string, closerName string) { if !closed { log.Printf("%s: unable to close %s after %d retries\n", debugName, closerName, maxRetries) } + return closed } From 8211e8212936f5baa09d8ea449f6512072612fc7 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 20:14:43 -0800 Subject: [PATCH 115/126] continue on error for recursive wavefile delete --- pkg/remote/fileshare/wavefs/wavefs.go | 9 +++++++-- pkg/util/tarcopy/tarcopy.go | 2 +- pkg/wshrpc/wshremote/wshremote.go | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index 03ef137dd..e3bab6890 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -117,7 +117,7 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio timeout = time.Duration(opts.Timeout) * time.Millisecond } readerCtx, cancel := context.WithTimeout(context.Background(), timeout) - rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, wshrpc.FileChunkSize, pathPrefix) + rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, pathPrefix) go func() { defer func() { @@ -510,11 +510,13 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu if !recursive { return fmt.Errorf("more than one entry, use recursive flag to delete") } + errs := make([]error, 0) for _, entry := range entries { fileName := strings.TrimPrefix(entry.Path, schemeAndHost) err = filestore.WFS.DeleteFile(ctx, zoneId, fileName) if err != nil { - return fmt.Errorf("error deleting blockfile: %w", err) + errs = append(errs, fmt.Errorf("error deleting blockfile %s/%s: %w", zoneId, fileName, err)) + continue } wps.Broker.Publish(wps.WaveEvent{ Event: wps.Event_BlockFile, @@ -526,6 +528,9 @@ func (c WaveClient) Delete(ctx context.Context, conn *connparse.Connection, recu }, }) } + if len(errs) > 0 { + return fmt.Errorf("error deleting blockfiles: %v", errs) + } } return nil } diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index bded4dcbb..00ecc674c 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -35,7 +35,7 @@ const ( // writeHeader is a function that writes the tar header for the file. // writer is the tar writer to write the file data to. // close is a function that closes the tar writer and internal pipe writer. -func TarCopySrc(ctx context.Context, chunkSize int, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string) error, writer io.Writer, close func()) { +func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc.RespOrErrorUnion[iochantypes.Packet], writeHeader func(fi fs.FileInfo, file string) error, writer io.Writer, close func()) { pipeReader, pipeWriter := io.Pipe() tarWriter := tar.NewWriter(pipeWriter) rtnChan := iochan.ReaderChan(ctx, pipeReader, wshrpc.FileChunkSize, func() { diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index ed49bc858..cf600ca79 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -264,7 +264,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. timeout = time.Duration(opts.Timeout) * time.Millisecond } readerCtx, cancel := context.WithTimeout(context.Background(), timeout) - rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, wshrpc.FileChunkSize, pathPrefix) + rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, pathPrefix) go func() { defer func() { From fa94e2d1021cdcc5e30595ed3c8fb344d355e19f Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 20:19:28 -0800 Subject: [PATCH 116/126] move validate to separate func --- pkg/util/tarcopy/tarcopy.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index 00ecc674c..bb39f0a2f 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -50,11 +50,8 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. } header.Name = filepath.Clean(strings.TrimPrefix(file, pathPrefix)) - if header.Name == "" { - return nil - } - if strings.HasPrefix(header.Name, "/") { - header.Name = header.Name[1:] + if err := validatePath(header.Name); err != nil { + return err } // write header @@ -68,6 +65,16 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. } } +func validatePath(path string) error { + if strings.Contains(path, "..") { + return fmt.Errorf("invalid path containing directory traversal: %s", path) + } + if strings.HasPrefix(path, "/") { + return fmt.Errorf("invalid path starting with /: %s", path) + } + return nil +} + // TarCopyDest reads a tar stream from a channel and writes the files to the destination. // readNext is a function that is called for each file in the tar stream to read the file data. It should return an error if the file cannot be read. // The function returns an error if the tar stream cannot be read. From 7121868d524cc8ab67bbadc734d35a53139d80dc Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 20:27:55 -0800 Subject: [PATCH 117/126] fix relative paths --- pkg/util/fileutil/fileutil.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 72a979eae..764a86e81 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -6,7 +6,6 @@ package fileutil import ( "io" "io/fs" - "log" "mime" "net/http" "os" @@ -20,15 +19,14 @@ import ( ) func FixPath(path string) (string, error) { + var err error if strings.HasPrefix(path, "~") { path = filepath.Join(wavebase.GetHomeDir(), path[1:]) } else if !filepath.IsAbs(path) { - log.Printf("FixPath: path is not absolute: %s", path) - path, err := filepath.Abs(path) + path, err = filepath.Abs(path) if err != nil { return "", err } - log.Printf("FixPath: fixed path: %s", path) } return path, nil } From eadc3dd8c40fbb0bca2a4872245da8d9b8f9a1d4 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 20:58:14 -0800 Subject: [PATCH 118/126] fix bug not showing errors, bug with prefix when copying single file, increase chunk size --- pkg/util/tarcopy/tarcopy.go | 13 +++++++++---- pkg/wshrpc/wshremote/wshremote.go | 29 ++++++++++++++++++++--------- pkg/wshrpc/wshrpctypes.go | 2 +- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index bb39f0a2f..dd3bc17fa 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -67,10 +67,10 @@ func TarCopySrc(ctx context.Context, pathPrefix string) (outputChan chan wshrpc. func validatePath(path string) error { if strings.Contains(path, "..") { - return fmt.Errorf("invalid path containing directory traversal: %s", path) + return fmt.Errorf("invalid tar path containing directory traversal: %s", path) } if strings.HasPrefix(path, "/") { - return fmt.Errorf("invalid path starting with /: %s", path) + return fmt.Errorf("invalid tar path starting with /: %s", path) } return nil } @@ -94,18 +94,23 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan for { select { case <-ctx.Done(): + if ctx.Err() != nil { + return context.Cause(ctx) + } return nil default: next, err := tarReader.Next() if err != nil { // Do one more check for context error before returning if ctx.Err() != nil { - return nil + if ctx.Err() != nil { + return context.Cause(ctx) + } } if errors.Is(err, io.EOF) { return nil } else { - return fmt.Errorf("cannot read tar stream: %w", err) + return err } } err = readNext(next, tarReader) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index cf600ca79..03b96525b 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -251,7 +251,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if finfo.IsDir() && strings.HasSuffix(cleanedPath, "/") { pathPrefix = cleanedPath } else { - pathPrefix = filepath.Dir(cleanedPath) + pathPrefix = filepath.Dir(cleanedPath) + "/" } if finfo.IsDir() { if !recursive { @@ -271,31 +271,37 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. tarClose() cancel() }() - err := filepath.Walk(path, func(file string, fi os.FileInfo, err error) error { + walkFunc := func(path string, info fs.FileInfo, err error) error { if readerCtx.Err() != nil { return readerCtx.Err() } if err != nil { return err } - if err = writeHeader(fi, file); err != nil { + if err = writeHeader(info, path); err != nil { return err } // if not a dir, write file content - if !fi.IsDir() { - data, err := os.Open(file) + if !info.IsDir() { + data, err := os.Open(path) if err != nil { return err } if _, err := io.Copy(fileWriter, data); err != nil { - log.Printf("error copying file %q: %v\n", file, err) return err } } return nil - }) + } + log.Printf("RemoteTarStreamCommand: starting\n") + err = nil + if finfo.IsDir() { + err = filepath.Walk(path, walkFunc) + } else { + err = walkFunc(path, finfo, nil) + } if err != nil { - rtn <- wshutil.RespErr[iochantypes.Packet](fmt.Errorf("cannot create tar stream for %q: %w", path, err)) + rtn <- wshutil.RespErr[iochantypes.Packet](err) } log.Printf("RemoteTarStreamCommand: done\n") }() @@ -435,7 +441,12 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C if err != nil { return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) } - log.Printf("RemoteFileCopyCommand: done; %d files copied in %dms, total of %d bytes, %d files skipped\n", numFiles, time.Since(copyStart).Milliseconds(), totalBytes, numSkipped) + totalTime := time.Since(copyStart).Seconds() + rate := float64(0) + if totalTime > 0 { + rate = float64(totalBytes) / totalTime / 1024 / 1024 + } + log.Printf("RemoteFileCopyCommand: done; %d files copied in %.3fs, total of %d bytes, %.2f MB/s, %d files skipped\n", numFiles, totalTime, totalBytes, rate, numSkipped) } return nil } diff --git a/pkg/wshrpc/wshrpctypes.go b/pkg/wshrpc/wshrpctypes.go index b601ddf42..06de27f5c 100644 --- a/pkg/wshrpc/wshrpctypes.go +++ b/pkg/wshrpc/wshrpctypes.go @@ -26,7 +26,7 @@ const ( // MaxDirSize is the maximum number of entries that can be read in a directory MaxDirSize = 1024 // FileChunkSize is the size of the file chunk to read - FileChunkSize = 16 * 1024 + FileChunkSize = 64 * 1024 // DirChunkSize is the size of the directory chunk to read DirChunkSize = 128 ) From 73afcc10ee83fd5a481ae8771d9180ed759d34c0 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 21:19:42 -0800 Subject: [PATCH 119/126] fix relative subpaths, improve metrics in logs --- pkg/remote/connparse/connparse.go | 2 +- pkg/remote/connparse/connparse_test.go | 48 ++++++++++++++++++++++++++ pkg/util/fileutil/fileutil.go | 3 ++ pkg/wshrpc/wshremote/wshremote.go | 5 +-- 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/pkg/remote/connparse/connparse.go b/pkg/remote/connparse/connparse.go index 892d9fb02..995106561 100644 --- a/pkg/remote/connparse/connparse.go +++ b/pkg/remote/connparse/connparse.go @@ -152,7 +152,7 @@ func ParseURI(uri string) (*Connection, error) { } if strings.HasPrefix(remotePath, "/~") { remotePath = strings.TrimPrefix(remotePath, "/") - } else if len(remotePath) > 1 && !windowsDriveRegex.MatchString(remotePath) && !strings.HasPrefix(remotePath, "/") && !strings.HasPrefix(remotePath, "~") { + } else if len(remotePath) > 1 && !windowsDriveRegex.MatchString(remotePath) && !strings.HasPrefix(remotePath, "/") && !strings.HasPrefix(remotePath, "~") && !strings.HasPrefix(remotePath, "./") && !strings.HasPrefix(remotePath, "../") && !strings.HasPrefix(remotePath, ".\\") && !strings.HasPrefix(remotePath, "..\\") && remotePath != ".." { remotePath = "/" + remotePath } } diff --git a/pkg/remote/connparse/connparse_test.go b/pkg/remote/connparse/connparse_test.go index 5eae1ce88..c530c8e76 100644 --- a/pkg/remote/connparse/connparse_test.go +++ b/pkg/remote/connparse/connparse_test.go @@ -190,6 +190,54 @@ func TestParseURI_WSHCurrentPathShorthand(t *testing.T) { } } +func TestParseURI_WSHCurrentPath(t *testing.T) { + cstr := "./Documents/path/to/file" + c, err := connparse.ParseURI(cstr) + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := "./Documents/path/to/file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "current" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } + expected = "wsh://current/./Documents/path/to/file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } +} + +func TestParseURI_WSHCurrentPathWindows(t *testing.T) { + cstr := ".\\Documents\\path\\to\\file" + c, err := connparse.ParseURI(cstr) + if err != nil { + t.Fatalf("failed to parse URI: %v", err) + } + expected := ".\\Documents\\path\\to\\file" + if c.Path != expected { + t.Fatalf("expected path to be %q, got %q", expected, c.Path) + } + expected = "current" + if c.Host != expected { + t.Fatalf("expected host to be %q, got %q", expected, c.Host) + } + expected = "wsh" + if c.Scheme != expected { + t.Fatalf("expected scheme to be %q, got %q", expected, c.Scheme) + } + expected = "wsh://current/.\\Documents\\path\\to\\file" + if c.GetFullURI() != expected { + t.Fatalf("expected full URI to be %q, got %q", expected, c.GetFullURI()) + } +} + func TestParseURI_WSHLocalShorthand(t *testing.T) { t.Parallel() cstr := "/~/path/to/file" diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 764a86e81..596d2fe25 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -6,6 +6,7 @@ package fileutil import ( "io" "io/fs" + "log" "mime" "net/http" "os" @@ -23,10 +24,12 @@ func FixPath(path string) (string, error) { if strings.HasPrefix(path, "~") { path = filepath.Join(wavebase.GetHomeDir(), path[1:]) } else if !filepath.IsAbs(path) { + log.Printf("FixPath: path is not absolute: %s", path) path, err = filepath.Abs(path) if err != nil { return "", err } + log.Printf("FixPath: fixed path: %s", path) } return path, nil } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 03b96525b..21883495c 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -442,11 +442,12 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot copy %q to %q: %w", srcUri, destUri, err) } totalTime := time.Since(copyStart).Seconds() + totalMegaBytes := float64(totalBytes) / 1024 / 1024 rate := float64(0) if totalTime > 0 { - rate = float64(totalBytes) / totalTime / 1024 / 1024 + rate = totalMegaBytes / totalTime } - log.Printf("RemoteFileCopyCommand: done; %d files copied in %.3fs, total of %d bytes, %.2f MB/s, %d files skipped\n", numFiles, totalTime, totalBytes, rate, numSkipped) + log.Printf("RemoteFileCopyCommand: done; %d files copied in %.3fs, total of %.4f MB, %.2f MB/s, %d files skipped\n", numFiles, totalTime, totalMegaBytes, rate, numSkipped) } return nil } From 82abf36313364563716b2565505dc2d0f781a7b0 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 21:25:37 -0800 Subject: [PATCH 120/126] Update pkg/util/tarcopy/tarcopy.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/util/tarcopy/tarcopy.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/util/tarcopy/tarcopy.go b/pkg/util/tarcopy/tarcopy.go index dd3bc17fa..825ac2b31 100644 --- a/pkg/util/tarcopy/tarcopy.go +++ b/pkg/util/tarcopy/tarcopy.go @@ -103,9 +103,7 @@ func TarCopyDest(ctx context.Context, cancel context.CancelCauseFunc, ch <-chan if err != nil { // Do one more check for context error before returning if ctx.Err() != nil { - if ctx.Err() != nil { - return context.Cause(ctx) - } + return context.Cause(ctx) } if errors.Is(err, io.EOF) { return nil From 57feafbe838522df20360bfa43c739081605c9a4 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Tue, 28 Jan 2025 21:29:33 -0800 Subject: [PATCH 121/126] increase default timeout for copy --- pkg/remote/fileshare/wavefs/wavefs.go | 6 +++++- pkg/wshrpc/wshremote/wshremote.go | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/remote/fileshare/wavefs/wavefs.go b/pkg/remote/fileshare/wavefs/wavefs.go index e3bab6890..63cbe36a1 100644 --- a/pkg/remote/fileshare/wavefs/wavefs.go +++ b/pkg/remote/fileshare/wavefs/wavefs.go @@ -29,6 +29,10 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) +const ( + DefaultTimeout = 30 * time.Second +) + type WaveClient struct{} var _ fstype.FileShareClient = WaveClient{} @@ -112,7 +116,7 @@ func (c WaveClient) ReadTarStream(ctx context.Context, conn *connparse.Connectio pathPrefix := getPathPrefix(conn) schemeAndHost := conn.GetSchemeAndHost() + "/" - timeout := time.Millisecond * 100 + timeout := DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 21883495c..1d1da042b 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -29,6 +29,10 @@ import ( "github.com/wavetermdev/waveterm/pkg/wshutil" ) +const ( + DefaultTimeout = 30 * time.Second +) + type ServerImpl struct { LogWriter io.Writer } @@ -259,7 +263,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. } } - timeout := time.Millisecond * 100 + timeout := DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } @@ -351,7 +355,7 @@ func (impl *ServerImpl) RemoteFileCopyCommand(ctx context.Context, data wshrpc.C return fmt.Errorf("cannot copy file %q to %q: %w", srcPathCleaned, destPathCleaned, err) } } else { - timeout := time.Millisecond * 100 + timeout := DefaultTimeout if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } From 317dae446a81f3753999de88d7001d21595b9c23 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 29 Jan 2025 11:52:16 -0800 Subject: [PATCH 122/126] remove fixpath logs --- pkg/util/fileutil/fileutil.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 596d2fe25..764a86e81 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -6,7 +6,6 @@ package fileutil import ( "io" "io/fs" - "log" "mime" "net/http" "os" @@ -24,12 +23,10 @@ func FixPath(path string) (string, error) { if strings.HasPrefix(path, "~") { path = filepath.Join(wavebase.GetHomeDir(), path[1:]) } else if !filepath.IsAbs(path) { - log.Printf("FixPath: path is not absolute: %s", path) path, err = filepath.Abs(path) if err != nil { return "", err } - log.Printf("FixPath: fixed path: %s", path) } return path, nil } From 9265cd01a47f25a21ad6be71ce4ab63371861d3c Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 29 Jan 2025 12:23:55 -0800 Subject: [PATCH 123/126] fix channel leak in wshutil --- pkg/wshutil/wshrpcio.go | 11 +++++++++++ pkg/wshutil/wshutil.go | 2 ++ 2 files changed, 13 insertions(+) diff --git a/pkg/wshutil/wshrpcio.go b/pkg/wshutil/wshrpcio.go index 67dbe48d6..9aa5f1609 100644 --- a/pkg/wshutil/wshrpcio.go +++ b/pkg/wshutil/wshrpcio.go @@ -22,12 +22,23 @@ func AdaptStreamToMsgCh(input io.Reader, output chan []byte) error { } func AdaptOutputChToStream(outputCh chan []byte, output io.Writer) error { + drain := false + defer func() { + if drain { + go func() { + for range outputCh { + } + }() + } + }() for msg := range outputCh { if _, err := output.Write(msg); err != nil { + drain = true return fmt.Errorf("error writing to output (AdaptOutputChToStream): %w", err) } // write trailing newline if _, err := output.Write([]byte{'\n'}); err != nil { + drain = true return fmt.Errorf("error writing trailing newline to output (AdaptOutputChToStream): %w", err) } } diff --git a/pkg/wshutil/wshutil.go b/pkg/wshutil/wshutil.go index 871fd72d1..17a422eeb 100644 --- a/pkg/wshutil/wshutil.go +++ b/pkg/wshutil/wshutil.go @@ -484,6 +484,8 @@ func handleDomainSocketClient(conn net.Conn) { }() defer func() { conn.Close() + close(proxy.FromRemoteCh) + close(proxy.ToRemoteCh) routeIdPtr := routeIdContainer.Load() if routeIdPtr != nil && *routeIdPtr != "" { DefaultRouter.UnregisterRoute(*routeIdPtr) From fb0f0cb1995475439faee0c695656b134e4505f9 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Wed, 29 Jan 2025 12:50:30 -0800 Subject: [PATCH 124/126] conditionally drain channel in writerchan --- pkg/util/iochan/iochan.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/util/iochan/iochan.go b/pkg/util/iochan/iochan.go index e35f65cdc..98fb94a19 100644 --- a/pkg/util/iochan/iochan.go +++ b/pkg/util/iochan/iochan.go @@ -20,12 +20,12 @@ import ( // ReaderChan reads from an io.Reader and sends the data to a channel func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func()) chan wshrpc.RespOrErrorUnion[iochantypes.Packet] { ch := make(chan wshrpc.RespOrErrorUnion[iochantypes.Packet], 32) - sha256Hash := sha256.New() go func() { defer func() { close(ch) callback() }() + sha256Hash := sha256.New() for { select { case <-ctx.Done(): @@ -57,12 +57,14 @@ func ReaderChan(ctx context.Context, r io.Reader, chunkSize int64, callback func // WriterChan reads from a channel and writes the data to an io.Writer func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet], callback func(), cancel context.CancelCauseFunc) { - sha256Hash := sha256.New() go func() { defer func() { - drainChannel(ch) + if ctx.Err() != nil { + drainChannel(ch) + } callback() }() + sha256Hash := sha256.New() for { select { case <-ctx.Done(): @@ -97,6 +99,8 @@ func WriterChan(ctx context.Context, w io.Writer, ch <-chan wshrpc.RespOrErrorUn } func drainChannel(ch <-chan wshrpc.RespOrErrorUnion[iochantypes.Packet]) { - for range ch { - } + go func() { + for range ch { + } + }() } From 449e6ccb59f09ca8d5fbc9e5edc5717d0eb8fa82 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 31 Jan 2025 10:40:59 -0800 Subject: [PATCH 125/126] Update pkg/wshrpc/wshremote/wshremote.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/wshrpc/wshremote/wshremote.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/wshrpc/wshremote/wshremote.go b/pkg/wshrpc/wshremote/wshremote.go index 1d1da042b..f324f52fe 100644 --- a/pkg/wshrpc/wshremote/wshremote.go +++ b/pkg/wshrpc/wshremote/wshremote.go @@ -267,7 +267,7 @@ func (impl *ServerImpl) RemoteTarStreamCommand(ctx context.Context, data wshrpc. if opts.Timeout > 0 { timeout = time.Duration(opts.Timeout) * time.Millisecond } - readerCtx, cancel := context.WithTimeout(context.Background(), timeout) + readerCtx, cancel := context.WithTimeout(ctx, timeout) rtn, writeHeader, fileWriter, tarClose := tarcopy.TarCopySrc(readerCtx, pathPrefix) go func() { From 0d7faf98ca8be03ca1473314317147918bbc14f3 Mon Sep 17 00:00:00 2001 From: Evan Simkowitz Date: Fri, 31 Jan 2025 10:41:30 -0800 Subject: [PATCH 126/126] Update pkg/util/fileutil/fileutil.go Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- pkg/util/fileutil/fileutil.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/pkg/util/fileutil/fileutil.go b/pkg/util/fileutil/fileutil.go index 764a86e81..18bb538d6 100644 --- a/pkg/util/fileutil/fileutil.go +++ b/pkg/util/fileutil/fileutil.go @@ -199,12 +199,17 @@ func (f FsFileInfo) Sys() interface{} { var _ fs.FileInfo = FsFileInfo{} +// ToFsFileInfo converts wshrpc.FileInfo to FsFileInfo. +// It panics if fi is nil. func ToFsFileInfo(fi *wshrpc.FileInfo) FsFileInfo { - return FsFileInfo{ - NameInternal: fi.Name, - ModeInternal: fi.Mode, - SizeInternal: fi.Size, - ModTimeInternal: fi.ModTime, - IsDirInternal: fi.IsDir, - } + if fi == nil { + panic("ToFsFileInfo: nil FileInfo") + } + return FsFileInfo{ + NameInternal: fi.Name, + ModeInternal: fi.Mode, + SizeInternal: fi.Size, + ModTimeInternal: fi.ModTime, + IsDirInternal: fi.IsDir, + } }