diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml index a5e6b6e..479ea08 100644 --- a/.gitea/workflows/release.yml +++ b/.gitea/workflows/release.yml @@ -49,17 +49,17 @@ jobs: cp build/bin/GoExplore-amd64-installer.exe dist/goexplore-windows-installer.exe " - # - name: Generate Changelog - # id: changelog - # uses: https://github.com/metcalfc/changelog-generator@v4.6.2 - # with: - # myToken: ${{ secrets.GITHUB_TOKEN }} + - name: Generate Changelog + id: changelog + uses: https://github.com/metcalfc/changelog-generator@v4.6.2 + with: + myToken: ${{ secrets.GITHUB_TOKEN }} - name: Create Gitea Release uses: https://gitea.com/actions/gitea-release-action@v1 with: tag_name: ${{ steps.get_version.outputs.VERSION }} - name: GoExplore ${{ steps.get_version.outputs.VERSION }} + name: ${{ steps.get_version.outputs.VERSION }} body: ${{ steps.changelog.outputs.changelog }} draft: false prerelease: false diff --git a/app.go b/app.go index 7788abf..6a03dfa 100644 --- a/app.go +++ b/app.go @@ -6,6 +6,7 @@ import ( "goexplore/internal/config" "goexplore/internal/explorer" "goexplore/internal/keychain" + "goexplore/internal/protocols/ftp" "goexplore/internal/protocols/local" "goexplore/internal/protocols/nfs" "goexplore/internal/protocols/s3" @@ -123,6 +124,8 @@ func (a *App) getExplorerForConnection(id string) (explorer.Explorer, error) { return webdav.New(conn, secret), nil case "nfs": return nfs.New(conn, secret) + case "ftp": + return ftp.New(conn, secret), nil default: return nil, fmt.Errorf("protocol %s not fully implemented", conn.Protocol) } @@ -164,7 +167,7 @@ func (a *App) Delete(connId, path string) error { return exp.Delete(path) } -func (a *App) QueueTransfer(id string, srcConnId, dstConnId, srcPath, dstPath, filename string, size int64) error { +func (a *App) QueueTransfer(id string, srcConnId, dstConnId, srcPath, dstPath, filename string, size int64, verify bool, limitMBps int) error { srcExp, err := a.getExplorerForConnection(srcConnId) if err != nil { return err @@ -182,7 +185,7 @@ func (a *App) QueueTransfer(id string, srcConnId, dstConnId, srcPath, dstPath, f return err } - return a.transferManager.QueueTransfer(id, srcPath, dstPath, filename, size, srcExp, dstExp) + return a.transferManager.QueueTransfer(id, srcPath, dstPath, filename, size, srcExp, dstExp, verify, limitMBps) } func (a *App) GetTransfers() []*transfer.Transfer { @@ -229,7 +232,7 @@ func (a *App) PromptUploadFiles(connId, destPath string) error { remotePath = destPath + fileName } - a.QueueTransfer(id, "local", connId, localPath, remotePath, fileName, stat.Size()) + a.QueueTransfer(id, "local", connId, localPath, remotePath, fileName, stat.Size(), false, 0) } return nil } @@ -269,7 +272,7 @@ func (a *App) PromptUploadDirectory(connId, destPath string) error { } id := uuid.New().String() - a.QueueTransfer(id, "local", connId, path, remotePath, info.Name(), info.Size()) + a.QueueTransfer(id, "local", connId, path, remotePath, info.Name(), info.Size(), false, 0) return nil }) } @@ -301,7 +304,7 @@ func (a *App) PromptDownload(connId, remotePath string) error { size = stat.Size } - return a.QueueTransfer(id, connId, "local", remotePath, localPath, fileName, size) + return a.QueueTransfer(id, connId, "local", remotePath, localPath, fileName, size, false, 0) } type TransferItem struct { @@ -311,7 +314,7 @@ type TransferItem struct { Size int64 `json:"size"` } -func (a *App) TransferItems(srcConnId, dstConnId, dstPath string, items []TransferItem) error { +func (a *App) TransferItems(srcConnId, dstConnId, dstPath string, items []TransferItem, verify bool, limitMBps int) error { for _, item := range items { if !item.IsDir { id := uuid.New().String() @@ -325,18 +328,18 @@ func (a *App) TransferItems(srcConnId, dstConnId, dstPath string, items []Transf remotePath = dstPath + item.Name } - if err := a.QueueTransfer(id, srcConnId, dstConnId, item.Path, remotePath, item.Name, item.Size); err != nil { + if err := a.QueueTransfer(id, srcConnId, dstConnId, item.Path, remotePath, item.Name, item.Size, verify, limitMBps); err != nil { return err } } else { // Run remote walk in goroutine to not block UI - go a.transferRemoteDirectory(srcConnId, dstConnId, item.Path, dstPath) + go a.transferRemoteDirectory(srcConnId, dstConnId, item.Path, dstPath, verify, limitMBps) } } return nil } -func (a *App) transferRemoteDirectory(srcConnId, dstConnId, srcDirPath, dstBasePath string) { +func (a *App) transferRemoteDirectory(srcConnId, dstConnId, srcDirPath, dstBasePath string, verify bool, limitMBps int) { baseName := filepath.Base(srcDirPath) newDstPath := dstBasePath @@ -357,11 +360,11 @@ func (a *App) transferRemoteDirectory(srcConnId, dstConnId, srcDirPath, dstBaseP for _, entry := range entries { if entry.IsDir { - a.transferRemoteDirectory(srcConnId, dstConnId, entry.Path, newDstPath) + a.transferRemoteDirectory(srcConnId, dstConnId, entry.Path, newDstPath, verify, limitMBps) } else { id := uuid.New().String() itemDstPath := newDstPath + "/" + entry.Name - a.QueueTransfer(id, srcConnId, dstConnId, entry.Path, itemDstPath, entry.Name, entry.Size) + a.QueueTransfer(id, srcConnId, dstConnId, entry.Path, itemDstPath, entry.Name, entry.Size, verify, limitMBps) } } } diff --git a/frontend/index.html b/frontend/index.html index 0e2d7dd..fd034e3 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -83,6 +83,7 @@ + + +
- +
+ + +
+
@@ -163,6 +176,21 @@ + +
+ + +
+ +
+ + + (0 = Unlimited) +
+
diff --git a/frontend/src/main.js b/frontend/src/main.js index 5f5bd0e..50b03d1 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -46,9 +46,13 @@ async function init() { bucket: document.getElementById('conn-bucket').value, region: document.getElementById('conn-region').value, path_style: document.getElementById('conn-pathstyle').checked, + secure: document.getElementById('conn-secure').checked, username: document.getElementById('conn-username').value }; - const secret = document.getElementById('conn-secret').value; + let secret = document.getElementById('conn-secret').value; + if (conn.protocol === 'sftp' && document.getElementById('conn-sftp-auth-type').value === 'key') { + secret = document.getElementById('conn-secret-key').value; + } try { await SaveConnection(conn, secret); @@ -102,8 +106,11 @@ window.editConnection = (id) => { document.getElementById('conn-bucket').value = c.bucket || ''; document.getElementById('conn-region').value = c.region || ''; document.getElementById('conn-pathstyle').checked = c.path_style || false; + document.getElementById('conn-secure').checked = c.secure || false; document.getElementById('conn-username').value = c.username || ''; document.getElementById('conn-secret').value = ''; + document.getElementById('conn-secret-key').value = ''; + document.getElementById('conn-sftp-auth-type').value = 'password'; document.getElementById('conn-delete-btn').style.display = 'block'; updateProtocolFields(); @@ -114,6 +121,10 @@ window.openConnModal = () => { document.getElementById('modal-title').innerText = "Add Connection"; document.getElementById('conn-form').reset(); document.getElementById('conn-id').value = uuidv4(); + document.getElementById('conn-secure').checked = false; + document.getElementById('conn-secret').value = ''; + document.getElementById('conn-secret-key').value = ''; + document.getElementById('conn-sftp-auth-type').value = 'password'; document.getElementById('conn-delete-btn').style.display = 'none'; updateProtocolFields(); document.getElementById('conn-modal').style.display = 'flex'; @@ -133,6 +144,34 @@ window.updateProtocolFields = () => { const showS3Specific = protocol === 's3'; document.getElementById('region-group').style.display = showS3Specific ? 'flex' : 'none'; document.getElementById('pathstyle-group').style.display = showS3Specific ? 'flex' : 'none'; + + // FTP specific fields + const showFTP = protocol === 'ftp'; + document.getElementById('ftp-secure-group').style.display = showFTP ? 'flex' : 'none'; + + // SFTP Auth Type toggle + const authTypeSelect = document.getElementById('conn-sftp-auth-type'); + const secretInput = document.getElementById('conn-secret'); + const secretKeyArea = document.getElementById('conn-secret-key'); + const secretLabel = document.getElementById('secret-label'); + + if (protocol === 'sftp') { + authTypeSelect.style.display = 'block'; + if (authTypeSelect.value === 'key') { + secretInput.style.display = 'none'; + secretKeyArea.style.display = 'block'; + secretLabel.innerText = 'SSH Private Key'; + } else { + secretInput.style.display = 'block'; + secretKeyArea.style.display = 'none'; + secretLabel.innerText = 'Password'; + } + } else { + authTypeSelect.style.display = 'none'; + secretInput.style.display = 'block'; + secretKeyArea.style.display = 'none'; + secretLabel.innerText = 'Password / Secret Key'; + } }; window.deleteConnection = async () => { @@ -489,6 +528,8 @@ window.openTransferModal = () => { window.closeTransferModal = () => { document.getElementById('transfer-modal').style.display = 'none'; + document.getElementById('transfer-verify').checked = false; + document.getElementById('transfer-limit').value = "0"; }; window.loadTransferDestRoot = () => { @@ -543,7 +584,9 @@ window.loadTransferDestDirectory = async (connId, path) => { window.executeTransfer = async () => { if (selectedItems.length === 0) return; try { - await TransferItems(currentConn, transferDestConn, transferDestPath, selectedItems); + const verify = document.getElementById('transfer-verify').checked; + const limit = parseInt(document.getElementById('transfer-limit').value, 10) || 0; + await TransferItems(currentConn, transferDestConn, transferDestPath, selectedItems, verify, limit); closeTransferModal(); showTransfers(); selectedItems = []; diff --git a/go.mod b/go.mod index 6f8b7e5..0cd290c 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/godbus/dbus/v5 v5.2.2 // indirect github.com/gorilla/websocket v1.5.3 // indirect github.com/jchv/go-winloader v0.0.0-20210711035445-715c2860da7e // indirect + github.com/jlaffaye/ftp v0.2.1 // indirect github.com/kr/fs v0.1.0 // indirect github.com/labstack/echo/v4 v4.13.3 // indirect github.com/labstack/gommon v0.4.2 // indirect @@ -63,6 +64,7 @@ require ( golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.15.0 // indirect ) // replace github.com/wailsapp/wails/v2 v2.12.0 => /home/jamie/go/pkg/mod diff --git a/go.sum b/go.sum index 3017996..1b3ce69 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRK github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/jchv/go-winloader v0.0.0-20210711035445-715c2860da7e h1:Q3+PugElBCf4PFpxhErSzU3/PY5sFL5Z6rfv4AbGAck= github.com/jchv/go-winloader v0.0.0-20210711035445-715c2860da7e/go.mod h1:alcuEEnZsY1WQsagKhZDsoPCRoOijYqhZvPwLG0kzVs= +github.com/jlaffaye/ftp v0.2.1 h1:AICcTYPMkaXlmjLMm9I+lB36f6jXCsCvBqVQc6EfC1Y= +github.com/jlaffaye/ftp v0.2.1/go.mod h1:gXSIr1pA9NhynDNigiFHs4+yL7o7I6bGF9Za9wi9tcE= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -146,6 +148,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= diff --git a/internal/config/config.go b/internal/config/config.go index e67a603..63b00b9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,6 +16,7 @@ type ConnectionConfig struct { Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty"` Region string `yaml:"region,omitempty" json:"region,omitempty"` PathStyle bool `yaml:"path_style,omitempty" json:"path_style,omitempty"` + Secure bool `yaml:"secure,omitempty" json:"secure,omitempty"` Username string `yaml:"username,omitempty" json:"username,omitempty"` KeychainKey string `yaml:"keychain_key,omitempty" json:"keychain_key,omitempty"` } diff --git a/internal/explorer/explorer.go b/internal/explorer/explorer.go index 0224ed3..9fbb28b 100644 --- a/internal/explorer/explorer.go +++ b/internal/explorer/explorer.go @@ -21,4 +21,5 @@ type Explorer interface { Rename(src, dst string) error ReadFile(path string) (io.ReadCloser, error) WriteFile(path string, r io.Reader, size int64) error + Checksum(path string) (string, error) } diff --git a/internal/protocols/ftp/ftp.go b/internal/protocols/ftp/ftp.go new file mode 100644 index 0000000..ce1dfec --- /dev/null +++ b/internal/protocols/ftp/ftp.go @@ -0,0 +1,194 @@ +package ftp + +import ( + "crypto/md5" + "crypto/tls" + "encoding/hex" + "fmt" + "io" + "path/filepath" + "time" + + "github.com/jlaffaye/ftp" + "goexplore/internal/config" + "goexplore/internal/explorer" +) + +type FTPExplorer struct { + cfg *config.ConnectionConfig + secret string + client *ftp.ServerConn +} + +func New(c *config.ConnectionConfig, secret string) *FTPExplorer { + return &FTPExplorer{cfg: c, secret: secret} +} + +func (e *FTPExplorer) Connect() error { + port := e.cfg.Port + if port == 0 { + port = 21 + } + + addr := fmt.Sprintf("%s:%d", e.cfg.Host, port) + var c *ftp.ServerConn + var err error + + if e.cfg.Secure { + c, err = ftp.Dial(addr, ftp.DialWithExplicitTLS(&tls.Config{ + InsecureSkipVerify: true, + })) + } else { + c, err = ftp.Dial(addr) + } + + if err != nil { + return err + } + + if err := c.Login(e.cfg.Username, e.secret); err != nil { + c.Quit() + return err + } + + e.client = c + return nil +} + +func (e *FTPExplorer) Disconnect() error { + if e.client != nil { + return e.client.Quit() + } + return nil +} + +func (e *FTPExplorer) ListDir(path string) ([]explorer.FileEntry, error) { + if path == "" { + path = "." + } + + entries, err := e.client.List(path) + if err != nil { + return nil, err + } + + var res []explorer.FileEntry + for _, f := range entries { + if f.Name == "." || f.Name == ".." { + continue + } + + isDir := f.Type == ftp.EntryTypeFolder + res = append(res, explorer.FileEntry{ + Name: f.Name, + Path: filepath.ToSlash(filepath.Join(path, f.Name)), + Size: int64(f.Size), + Modified: f.Time.Format(time.RFC3339), + IsDir: isDir, + Permissions: "", + }) + } + return res, nil +} + +func (e *FTPExplorer) Stat(path string) (explorer.FileEntry, error) { + if path == "" { + path = "." + } + + dir := filepath.Dir(path) + base := filepath.Base(path) + + entries, err := e.client.List(dir) + if err != nil { + return explorer.FileEntry{}, err + } + + for _, f := range entries { + if f.Name == base { + return explorer.FileEntry{ + Name: f.Name, + Path: filepath.ToSlash(path), + Size: int64(f.Size), + Modified: f.Time.Format(time.RFC3339), + IsDir: f.Type == ftp.EntryTypeFolder, + Permissions: "", + }, nil + } + } + + return explorer.FileEntry{}, fmt.Errorf("file not found: %s", path) +} + +func (e *FTPExplorer) MkDir(path string) error { + return e.client.MakeDir(path) +} + +func (e *FTPExplorer) removeAll(path string) error { + entries, err := e.client.List(path) + if err != nil { + // If it fails to list, it might be a file + return e.client.Delete(path) + } + + // Try treating it as a directory to remove contents + isDir := false + for _, f := range entries { + if f.Name == filepath.Base(path) && f.Type == ftp.EntryTypeFolder { + isDir = true + break + } + } + + if !isDir && len(entries) == 1 && entries[0].Name == filepath.Base(path) { + return e.client.Delete(path) + } + + // Remove contents + for _, f := range entries { + if f.Name == "." || f.Name == ".." { + continue + } + subPath := filepath.ToSlash(filepath.Join(path, f.Name)) + if f.Type == ftp.EntryTypeFolder { + if err := e.removeAll(subPath); err != nil { + return err + } + } else { + if err := e.client.Delete(subPath); err != nil { + return err + } + } + } + + return e.client.RemoveDir(path) +} + +func (e *FTPExplorer) Delete(path string) error { + return e.removeAll(path) +} + +func (e *FTPExplorer) Rename(src, dst string) error { + return e.client.Rename(src, dst) +} + +func (e *FTPExplorer) ReadFile(path string) (io.ReadCloser, error) { + return e.client.Retr(path) +} + +func (e *FTPExplorer) WriteFile(path string, r io.Reader, size int64) error { + return e.client.Stor(path, r) +} + +func (e *FTPExplorer) Checksum(path string) (string, error) { + r, err := e.ReadFile(path) + if err != nil { + return "", err + } + defer r.Close() + h := md5.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/protocols/local/local.go b/internal/protocols/local/local.go index b0c73ab..1d353bb 100644 --- a/internal/protocols/local/local.go +++ b/internal/protocols/local/local.go @@ -1,6 +1,8 @@ package local import ( + "crypto/md5" + "encoding/hex" "io" "os" "path/filepath" @@ -82,6 +84,19 @@ func (e *LocalExplorer) Rename(src, dst string) error { return os.Rename(src, dst) } +func (e *LocalExplorer) Checksum(path string) (string, error) { + r, err := e.ReadFile(path) + if err != nil { + return "", err + } + defer r.Close() + h := md5.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + func (e *LocalExplorer) ReadFile(path string) (io.ReadCloser, error) { return os.Open(path) } diff --git a/internal/protocols/nfs/nfs_unix.go b/internal/protocols/nfs/nfs_unix.go index 8f74ee2..30cae20 100644 --- a/internal/protocols/nfs/nfs_unix.go +++ b/internal/protocols/nfs/nfs_unix.go @@ -3,6 +3,8 @@ package nfs import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "path/filepath" @@ -125,3 +127,16 @@ func (e *NFSExplorer) WriteFile(path string, r io.Reader, size int64) error { _, err = io.Copy(f, r) return err } + +func (e *NFSExplorer) Checksum(path string) (string, error) { + r, err := e.ReadFile(path) + if err != nil { + return "", err + } + defer r.Close() + h := md5.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/protocols/nfs/nfs_windows.go b/internal/protocols/nfs/nfs_windows.go index 24b005d..59501a6 100644 --- a/internal/protocols/nfs/nfs_windows.go +++ b/internal/protocols/nfs/nfs_windows.go @@ -25,3 +25,4 @@ func (e *NFSExplorer) Delete(path string) error { return nil } func (e *NFSExplorer) Rename(src, dst string) error { return nil } func (e *NFSExplorer) ReadFile(path string) (io.ReadCloser, error) { return nil, nil } func (e *NFSExplorer) WriteFile(path string, r io.Reader, size int64) error { return nil } +func (e *NFSExplorer) Checksum(path string) (string, error) { return "", nil } diff --git a/internal/protocols/s3/s3.go b/internal/protocols/s3/s3.go index 33b609f..cbded41 100644 --- a/internal/protocols/s3/s3.go +++ b/internal/protocols/s3/s3.go @@ -307,3 +307,28 @@ func (e *S3Explorer) WriteFile(path string, r io.Reader, size int64) error { }) return err } + +func (e *S3Explorer) Checksum(path string) (string, error) { + bucket, subpath, _, err := e.getBucketAndPath(path) + if err != nil { + return "", err + } + + if subpath == "" { + return "", fmt.Errorf("cannot calculate checksum of bucket") + } + + input := &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(subpath), + } + head, err := e.client.HeadObject(context.TODO(), input) + if err != nil { + return "", err + } + + if head.ETag != nil { + return strings.Trim(*head.ETag, "\""), nil + } + return "", fmt.Errorf("no etag found") +} diff --git a/internal/protocols/sftp/sftp.go b/internal/protocols/sftp/sftp.go index 1918ee9..4e37775 100644 --- a/internal/protocols/sftp/sftp.go +++ b/internal/protocols/sftp/sftp.go @@ -1,6 +1,8 @@ package sftp import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "path/filepath" @@ -162,3 +164,16 @@ func (e *SFTPExplorer) WriteFile(path string, r io.Reader, size int64) error { _, err = io.Copy(f, r) return err } + +func (e *SFTPExplorer) Checksum(path string) (string, error) { + r, err := e.ReadFile(path) + if err != nil { + return "", err + } + defer r.Close() + h := md5.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/protocols/smb/smb.go b/internal/protocols/smb/smb.go index 22c7fd3..fcd44db 100644 --- a/internal/protocols/smb/smb.go +++ b/internal/protocols/smb/smb.go @@ -1,6 +1,8 @@ package smb import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "net" @@ -243,3 +245,16 @@ func (e *SMBExplorer) WriteFile(path string, r io.Reader, size int64) error { _, err = io.Copy(f, r) return err } + +func (e *SMBExplorer) Checksum(path string) (string, error) { + r, err := e.ReadFile(path) + if err != nil { + return "", err + } + defer r.Close() + h := md5.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/protocols/webdav/webdav.go b/internal/protocols/webdav/webdav.go index 5ecb3bf..9203a7b 100644 --- a/internal/protocols/webdav/webdav.go +++ b/internal/protocols/webdav/webdav.go @@ -1,6 +1,8 @@ package webdav import ( + "crypto/md5" + "encoding/hex" "fmt" "io" "strings" @@ -102,3 +104,16 @@ func (e *WebDAVExplorer) ReadFile(path string) (io.ReadCloser, error) { func (e *WebDAVExplorer) WriteFile(path string, r io.Reader, size int64) error { return e.client.WriteStream(path, r, 0644) } + +func (e *WebDAVExplorer) Checksum(path string) (string, error) { + r, err := e.ReadFile(path) + if err != nil { + return "", err + } + defer r.Close() + h := md5.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/transfer/transfer.go b/internal/transfer/transfer.go index 4b9b6d8..2a09aa2 100644 --- a/internal/transfer/transfer.go +++ b/internal/transfer/transfer.go @@ -1,11 +1,18 @@ package transfer import ( + "context" + "crypto/md5" + "encoding/hex" "errors" + "fmt" "io" + "strings" "sync" "time" + "golang.org/x/time/rate" + "goexplore/internal/explorer" ) @@ -29,6 +36,8 @@ type Transfer struct { ETA int `json:"eta_seconds"` Status Status `json:"status"` Error string `json:"error,omitempty"` + Verify bool `json:"verify"` + LimitMBps int `json:"limit_mbps"` srcExp explorer.Explorer dstExp explorer.Explorer @@ -81,12 +90,62 @@ func (m *Manager) doTransfer(t *Transfer) error { } defer r.Close() + hash := md5.New() + tr := io.TeeReader(r, hash) + + var streamReader io.Reader = tr + if t.LimitMBps > 0 { + limitBytes := t.LimitMBps * 1024 * 1024 + // Set burst to limitBytes to allow typical io.Copy chunk sizes without erroring + limiter := rate.NewLimiter(rate.Limit(limitBytes), limitBytes) + streamReader = &throttledReader{ + r: tr, + limiter: limiter, + } + } + pr := &progressReader{ - r: r, + r: streamReader, t: t, } - return t.dstExp.WriteFile(t.Destination, pr, t.BytesTotal) + if err := t.dstExp.WriteFile(t.Destination, pr, t.BytesTotal); err != nil { + return err + } + + if t.Verify { + expectedHash := hex.EncodeToString(hash.Sum(nil)) + t.Status = StatusActive // Still active during verification + + actualHash, err := t.dstExp.Checksum(t.Destination) + if err != nil { + return fmt.Errorf("verification failed: could not calculate destination checksum: %w", err) + } + + if expectedHash != actualHash { + // S3 Multi-part ETags end with -N, we can't easily verify them with local md5 + if !strings.Contains(actualHash, "-") { + return fmt.Errorf("verification failed: checksum mismatch (expected %s, got %s)", expectedHash, actualHash) + } + } + } + + return nil +} + +type throttledReader struct { + r io.Reader + limiter *rate.Limiter +} + +func (tr *throttledReader) Read(p []byte) (int, error) { + n, err := tr.r.Read(p) + if n > 0 && tr.limiter != nil { + if errWait := tr.limiter.WaitN(context.Background(), n); errWait != nil { + return n, errWait + } + } + return n, err } type progressReader struct { @@ -109,7 +168,7 @@ func (pr *progressReader) Read(p []byte) (int, error) { return n, err } -func (m *Manager) QueueTransfer(id, srcPath, dstPath, filename string, size int64, srcExp, dstExp explorer.Explorer) error { +func (m *Manager) QueueTransfer(id, srcPath, dstPath, filename string, size int64, srcExp, dstExp explorer.Explorer, verify bool, limitMBps int) error { if srcExp == nil || dstExp == nil { return errors.New("invalid explorers") } @@ -121,6 +180,8 @@ func (m *Manager) QueueTransfer(id, srcPath, dstPath, filename string, size int6 Filename: filename, BytesTotal: size, Status: StatusQueued, + Verify: verify, + LimitMBps: limitMBps, srcExp: srcExp, dstExp: dstExp, }