jamie/goexplore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

feat: enable changelog generation in release workflow #2

Merged
jamie merged 5 commits from v1.1.0 into main 2026-06-09 14:53:28 +01:00
Conversation 0 Commits 5 Files Changed 17
+461 -23
17 changed files with 461 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/release.yml
+6 -6
View File
@@ -49,17 +49,17 @@ jobs:
cp build/bin/GoExplore-amd64-installer.exe dist/goexplore-windows-installer.exe
"
# - name: Generate Changelog
# id: changelog
# uses: https://github.com/metcalfc/changelog-generator@v4.6.2
# with:
# myToken: ${{ secrets.GITHUB_TOKEN }}
- name: Generate Changelog
id: changelog
uses: https://github.com/metcalfc/changelog-generator@v4.6.2
with:
myToken: ${{ secrets.GITHUB_TOKEN }}
- name: Create Gitea Release
uses: https://gitea.com/actions/gitea-release-action@v1
with:
tag_name: ${{ steps.get_version.outputs.VERSION }}
name: GoExplore ${{ steps.get_version.outputs.VERSION }}
name: ${{ steps.get_version.outputs.VERSION }}
body: ${{ steps.changelog.outputs.changelog }}
draft: false
prerelease: false
app.go
+14 -11
View File
@@ -6,6 +6,7 @@ import (
"goexplore/internal/config"
"goexplore/internal/explorer"
"goexplore/internal/keychain"
"goexplore/internal/protocols/ftp"
"goexplore/internal/protocols/local"
"goexplore/internal/protocols/nfs"
"goexplore/internal/protocols/s3"
@@ -123,6 +124,8 @@ func (a *App) getExplorerForConnection(id string) (explorer.Explorer, error) {
return webdav.New(conn, secret), nil
case "nfs":
return nfs.New(conn, secret)
case "ftp":
return ftp.New(conn, secret), nil
default:
return nil, fmt.Errorf("protocol %s not fully implemented", conn.Protocol)
}
@@ -164,7 +167,7 @@ func (a *App) Delete(connId, path string) error {
return exp.Delete(path)
}
func (a *App) QueueTransfer(id string, srcConnId, dstConnId, srcPath, dstPath, filename string, size int64) error {
func (a *App) QueueTransfer(id string, srcConnId, dstConnId, srcPath, dstPath, filename string, size int64, verify bool, limitMBps int) error {
srcExp, err := a.getExplorerForConnection(srcConnId)
if err != nil {
return err
@@ -182,7 +185,7 @@ func (a *App) QueueTransfer(id string, srcConnId, dstConnId, srcPath, dstPath, f
return err
}
return a.transferManager.QueueTransfer(id, srcPath, dstPath, filename, size, srcExp, dstExp)
return a.transferManager.QueueTransfer(id, srcPath, dstPath, filename, size, srcExp, dstExp, verify, limitMBps)
}
func (a *App) GetTransfers() []*transfer.Transfer {
@@ -229,7 +232,7 @@ func (a *App) PromptUploadFiles(connId, destPath string) error {
remotePath = destPath + fileName
}
a.QueueTransfer(id, "local", connId, localPath, remotePath, fileName, stat.Size())
a.QueueTransfer(id, "local", connId, localPath, remotePath, fileName, stat.Size(), false, 0)
}
return nil
}
@@ -269,7 +272,7 @@ func (a *App) PromptUploadDirectory(connId, destPath string) error {
}
id := uuid.New().String()
a.QueueTransfer(id, "local", connId, path, remotePath, info.Name(), info.Size())
a.QueueTransfer(id, "local", connId, path, remotePath, info.Name(), info.Size(), false, 0)
return nil
})
}
@@ -301,7 +304,7 @@ func (a *App) PromptDownload(connId, remotePath string) error {
size = stat.Size
}
return a.QueueTransfer(id, connId, "local", remotePath, localPath, fileName, size)
return a.QueueTransfer(id, connId, "local", remotePath, localPath, fileName, size, false, 0)
}
type TransferItem struct {
@@ -311,7 +314,7 @@ type TransferItem struct {
Size int64 `json:"size"`
}
func (a *App) TransferItems(srcConnId, dstConnId, dstPath string, items []TransferItem) error {
func (a *App) TransferItems(srcConnId, dstConnId, dstPath string, items []TransferItem, verify bool, limitMBps int) error {
for _, item := range items {
if !item.IsDir {
id := uuid.New().String()
@@ -325,18 +328,18 @@ func (a *App) TransferItems(srcConnId, dstConnId, dstPath string, items []Transf
remotePath = dstPath + item.Name
}
if err := a.QueueTransfer(id, srcConnId, dstConnId, item.Path, remotePath, item.Name, item.Size); err != nil {
if err := a.QueueTransfer(id, srcConnId, dstConnId, item.Path, remotePath, item.Name, item.Size, verify, limitMBps); err != nil {
return err
}
} else {
// Run remote walk in goroutine to not block UI
go a.transferRemoteDirectory(srcConnId, dstConnId, item.Path, dstPath)
go a.transferRemoteDirectory(srcConnId, dstConnId, item.Path, dstPath, verify, limitMBps)
}
}
return nil
}
func (a *App) transferRemoteDirectory(srcConnId, dstConnId, srcDirPath, dstBasePath string) {
func (a *App) transferRemoteDirectory(srcConnId, dstConnId, srcDirPath, dstBasePath string, verify bool, limitMBps int) {
baseName := filepath.Base(srcDirPath)
newDstPath := dstBasePath
@@ -357,11 +360,11 @@ func (a *App) transferRemoteDirectory(srcConnId, dstConnId, srcDirPath, dstBaseP
for _, entry := range entries {
if entry.IsDir {
a.transferRemoteDirectory(srcConnId, dstConnId, entry.Path, newDstPath)
a.transferRemoteDirectory(srcConnId, dstConnId, entry.Path, newDstPath, verify, limitMBps)
} else {
id := uuid.New().String()
itemDstPath := newDstPath + "/" + entry.Name
a.QueueTransfer(id, srcConnId, dstConnId, entry.Path, itemDstPath, entry.Name, entry.Size)
a.QueueTransfer(id, srcConnId, dstConnId, entry.Path, itemDstPath, entry.Name, entry.Size, verify, limitMBps)
}
}
}
frontend/index.html
+29 -1
View File
@@ -83,6 +83,7 @@
<select id="conn-protocol" onchange="updateProtocolFields()">
<option value="s3">S3 / S3-Compatible</option>
<option value="sftp">SFTP</option>
<option value="ftp">FTP</option>
<option value="webdav">WebDAV</option>
<option value="smb">SMB</option>
<option value="nfs">NFS</option>
@@ -117,14 +118,26 @@
</div>
</div>
<div class="form-group" id="ftp-secure-group" style="display: none; align-items: center; gap: 0.5rem; flex-direction: row; margin-bottom: 1rem;">
<input type="checkbox" id="conn-secure" style="width: auto;">
<label style="margin: 0;">Use FTPS (Secure Explicit TLS)</label>
</div>
<div class="form-row">
<div class="form-group" style="flex: 1;">
<label>Username / Access Key</label>
<input type="text" id="conn-username" placeholder="user">
</div>
<div class="form-group" style="flex: 1;">
<label>Password / Secret Key</label>
<div style="display: flex; justify-content: space-between; align-items: center;">
<label id="secret-label">Password / Secret Key</label>
<select id="conn-sftp-auth-type" style="display: none; width: auto; font-size: 0.75rem; padding: 0.1rem 0.3rem;" onchange="updateProtocolFields()">
<option value="password">Password</option>
<option value="key">Private Key</option>
</select>
</div>
<input type="password" id="conn-secret" placeholder="Stored in OS Keychain">
<textarea id="conn-secret-key" style="display: none; width: 100%; box-sizing: border-box; background: var(--bg-alt); color: var(--text-primary); border: 1px solid var(--border); border-radius: 4px; padding: 0.5rem; font-family: monospace;" rows="4" placeholder="-----BEGIN OPENSSH PRIVATE KEY-----..."></textarea>
</div>
</div>
@@ -163,6 +176,21 @@
<tbody id="transfer-dest-list"></tbody>
</table>
</div>
<div class="form-group" style="display: flex; align-items: center; gap: 0.5rem; flex-direction: row; margin-bottom: 1rem;">
<input type="checkbox" id="transfer-verify" style="width: auto;">
<label style="margin: 0; display: flex; flex-direction: column;">
Verify Checksum (MD5)
<span style="font-size: 0.75rem; color: var(--text-secondary); font-weight: normal;">Slower for non-S3 destinations as it requires re-reading the file over the network.</span>
</label>
</div>
<div class="form-group" style="display: flex; align-items: center; gap: 0.5rem; flex-direction: row; margin-bottom: 1.5rem;">
<label style="margin: 0; width: 120px;">Max Speed (MB/s)</label>
<input type="number" id="transfer-limit" value="0" min="0" style="width: 80px;">
<span style="font-size: 0.75rem; color: var(--text-secondary);">(0 = Unlimited)</span>
</div>
<div style="display: flex; justify-content: flex-end; gap: 1rem;">
<button class="btn" style="background: transparent; color: var(--text-primary); border-color: var(--border);" onclick="closeTransferModal()">Cancel</button>
<button class="btn btn-primary" onclick="executeTransfer()">Transfer Here</button>
frontend/src/main.js
+45 -2
View File
@@ -46,9 +46,13 @@ async function init() {
bucket: document.getElementById('conn-bucket').value,
region: document.getElementById('conn-region').value,
path_style: document.getElementById('conn-pathstyle').checked,
secure: document.getElementById('conn-secure').checked,
username: document.getElementById('conn-username').value
};
const secret = document.getElementById('conn-secret').value;
let secret = document.getElementById('conn-secret').value;
if (conn.protocol === 'sftp' && document.getElementById('conn-sftp-auth-type').value === 'key') {
secret = document.getElementById('conn-secret-key').value;
}
try {
await SaveConnection(conn, secret);
@@ -102,8 +106,11 @@ window.editConnection = (id) => {
document.getElementById('conn-bucket').value = c.bucket || '';
document.getElementById('conn-region').value = c.region || '';
document.getElementById('conn-pathstyle').checked = c.path_style || false;
document.getElementById('conn-secure').checked = c.secure || false;
document.getElementById('conn-username').value = c.username || '';
document.getElementById('conn-secret').value = '';
document.getElementById('conn-secret-key').value = '';
document.getElementById('conn-sftp-auth-type').value = 'password';
document.getElementById('conn-delete-btn').style.display = 'block';
updateProtocolFields();
@@ -114,6 +121,10 @@ window.openConnModal = () => {
document.getElementById('modal-title').innerText = "Add Connection";
document.getElementById('conn-form').reset();
document.getElementById('conn-id').value = uuidv4();
document.getElementById('conn-secure').checked = false;
document.getElementById('conn-secret').value = '';
document.getElementById('conn-secret-key').value = '';
document.getElementById('conn-sftp-auth-type').value = 'password';
document.getElementById('conn-delete-btn').style.display = 'none';
updateProtocolFields();
document.getElementById('conn-modal').style.display = 'flex';
@@ -133,6 +144,34 @@ window.updateProtocolFields = () => {
const showS3Specific = protocol === 's3';
document.getElementById('region-group').style.display = showS3Specific ? 'flex' : 'none';
document.getElementById('pathstyle-group').style.display = showS3Specific ? 'flex' : 'none';
// FTP specific fields
const showFTP = protocol === 'ftp';
document.getElementById('ftp-secure-group').style.display = showFTP ? 'flex' : 'none';
// SFTP Auth Type toggle
const authTypeSelect = document.getElementById('conn-sftp-auth-type');
const secretInput = document.getElementById('conn-secret');
const secretKeyArea = document.getElementById('conn-secret-key');
const secretLabel = document.getElementById('secret-label');
if (protocol === 'sftp') {
authTypeSelect.style.display = 'block';
if (authTypeSelect.value === 'key') {
secretInput.style.display = 'none';
secretKeyArea.style.display = 'block';
secretLabel.innerText = 'SSH Private Key';
} else {
secretInput.style.display = 'block';
secretKeyArea.style.display = 'none';
secretLabel.innerText = 'Password';
}
} else {
authTypeSelect.style.display = 'none';
secretInput.style.display = 'block';
secretKeyArea.style.display = 'none';
secretLabel.innerText = 'Password / Secret Key';
}
};
window.deleteConnection = async () => {
@@ -489,6 +528,8 @@ window.openTransferModal = () => {
window.closeTransferModal = () => {
document.getElementById('transfer-modal').style.display = 'none';
document.getElementById('transfer-verify').checked = false;
document.getElementById('transfer-limit').value = "0";
};
window.loadTransferDestRoot = () => {
@@ -543,7 +584,9 @@ window.loadTransferDestDirectory = async (connId, path) => {
window.executeTransfer = async () => {
if (selectedItems.length === 0) return;
try {
await TransferItems(currentConn, transferDestConn, transferDestPath, selectedItems);
const verify = document.getElementById('transfer-verify').checked;
const limit = parseInt(document.getElementById('transfer-limit').value, 10) || 0;
await TransferItems(currentConn, transferDestConn, transferDestPath, selectedItems, verify, limit);
closeTransferModal();
showTransfers();
selectedItems = [];
go.mod
+2
View File
@@ -41,6 +41,7 @@ require (
github.com/godbus/dbus/v5 v5.2.2 // indirect
github.com/gorilla/websocket v1.5.3 // indirect
github.com/jchv/go-winloader v0.0.0-20210711035445-715c2860da7e // indirect
github.com/jlaffaye/ftp v0.2.1 // indirect
github.com/kr/fs v0.1.0 // indirect
github.com/labstack/echo/v4 v4.13.3 // indirect
github.com/labstack/gommon v0.4.2 // indirect
@@ -63,6 +64,7 @@ require (
golang.org/x/net v0.55.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/text v0.38.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
// replace github.com/wailsapp/wails/v2 v2.12.0 => /home/jamie/go/pkg/mod
go.sum
+4
View File
@@ -56,6 +56,8 @@ github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRK
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
github.com/jchv/go-winloader v0.0.0-20210711035445-715c2860da7e h1:Q3+PugElBCf4PFpxhErSzU3/PY5sFL5Z6rfv4AbGAck=
github.com/jchv/go-winloader v0.0.0-20210711035445-715c2860da7e/go.mod h1:alcuEEnZsY1WQsagKhZDsoPCRoOijYqhZvPwLG0kzVs=
github.com/jlaffaye/ftp v0.2.1 h1:AICcTYPMkaXlmjLMm9I+lB36f6jXCsCvBqVQc6EfC1Y=
github.com/jlaffaye/ftp v0.2.1/go.mod h1:gXSIr1pA9NhynDNigiFHs4+yL7o7I6bGF9Za9wi9tcE=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -146,6 +148,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
internal/config/config.go
+1
View File
@@ -16,6 +16,7 @@ type ConnectionConfig struct {
Bucket string `yaml:"bucket,omitempty" json:"bucket,omitempty"`
Region string `yaml:"region,omitempty" json:"region,omitempty"`
PathStyle bool `yaml:"path_style,omitempty" json:"path_style,omitempty"`
Secure bool `yaml:"secure,omitempty" json:"secure,omitempty"`
Username string `yaml:"username,omitempty" json:"username,omitempty"`
KeychainKey string `yaml:"keychain_key,omitempty" json:"keychain_key,omitempty"`
}
internal/explorer/explorer.go
+1
View File
@@ -21,4 +21,5 @@ type Explorer interface {
Rename(src, dst string) error
ReadFile(path string) (io.ReadCloser, error)
WriteFile(path string, r io.Reader, size int64) error
Checksum(path string) (string, error)
}
internal/protocols/ftp/ftp.go
+194
View File
@@ -0,0 +1,194 @@
package ftp
import (
"crypto/md5"
"crypto/tls"
"encoding/hex"
"fmt"
"io"
"path/filepath"
"time"
"github.com/jlaffaye/ftp"
"goexplore/internal/config"
"goexplore/internal/explorer"
)
type FTPExplorer struct {
cfg *config.ConnectionConfig
secret string
client *ftp.ServerConn
}
func New(c *config.ConnectionConfig, secret string) *FTPExplorer {
return &FTPExplorer{cfg: c, secret: secret}
}
func (e *FTPExplorer) Connect() error {
port := e.cfg.Port
if port == 0 {
port = 21
}
addr := fmt.Sprintf("%s:%d", e.cfg.Host, port)
var c *ftp.ServerConn
var err error
if e.cfg.Secure {
c, err = ftp.Dial(addr, ftp.DialWithExplicitTLS(&tls.Config{
InsecureSkipVerify: true,
}))
} else {
c, err = ftp.Dial(addr)
}
if err != nil {
return err
}
if err := c.Login(e.cfg.Username, e.secret); err != nil {
c.Quit()
return err
}
e.client = c
return nil
}
func (e *FTPExplorer) Disconnect() error {
if e.client != nil {
return e.client.Quit()
}
return nil
}
func (e *FTPExplorer) ListDir(path string) ([]explorer.FileEntry, error) {
if path == "" {
path = "."
}
entries, err := e.client.List(path)
if err != nil {
return nil, err
}
var res []explorer.FileEntry
for _, f := range entries {
if f.Name == "." || f.Name == ".." {
continue
}
isDir := f.Type == ftp.EntryTypeFolder
res = append(res, explorer.FileEntry{
Name: f.Name,
Path: filepath.ToSlash(filepath.Join(path, f.Name)),
Size: int64(f.Size),
Modified: f.Time.Format(time.RFC3339),
IsDir: isDir,
Permissions: "",
})
}
return res, nil
}
func (e *FTPExplorer) Stat(path string) (explorer.FileEntry, error) {
if path == "" {
path = "."
}
dir := filepath.Dir(path)
base := filepath.Base(path)
entries, err := e.client.List(dir)
if err != nil {
return explorer.FileEntry{}, err
}
for _, f := range entries {
if f.Name == base {
return explorer.FileEntry{
Name: f.Name,
Path: filepath.ToSlash(path),
Size: int64(f.Size),
Modified: f.Time.Format(time.RFC3339),
IsDir: f.Type == ftp.EntryTypeFolder,
Permissions: "",
}, nil
}
}
return explorer.FileEntry{}, fmt.Errorf("file not found: %s", path)
}
func (e *FTPExplorer) MkDir(path string) error {
return e.client.MakeDir(path)
}
func (e *FTPExplorer) removeAll(path string) error {
entries, err := e.client.List(path)
if err != nil {
// If it fails to list, it might be a file
return e.client.Delete(path)
}
// Try treating it as a directory to remove contents
isDir := false
for _, f := range entries {
if f.Name == filepath.Base(path) && f.Type == ftp.EntryTypeFolder {
isDir = true
break
}
}
if !isDir && len(entries) == 1 && entries[0].Name == filepath.Base(path) {
return e.client.Delete(path)
}
// Remove contents
for _, f := range entries {
if f.Name == "." || f.Name == ".." {
continue
}
subPath := filepath.ToSlash(filepath.Join(path, f.Name))
if f.Type == ftp.EntryTypeFolder {
if err := e.removeAll(subPath); err != nil {
return err
}
} else {
if err := e.client.Delete(subPath); err != nil {
return err
}
}
}
return e.client.RemoveDir(path)
}
func (e *FTPExplorer) Delete(path string) error {
return e.removeAll(path)
}
func (e *FTPExplorer) Rename(src, dst string) error {
return e.client.Rename(src, dst)
}
func (e *FTPExplorer) ReadFile(path string) (io.ReadCloser, error) {
return e.client.Retr(path)
}
func (e *FTPExplorer) WriteFile(path string, r io.Reader, size int64) error {
return e.client.Stor(path, r)
}
func (e *FTPExplorer) Checksum(path string) (string, error) {
r, err := e.ReadFile(path)
if err != nil {
return "", err
}
defer r.Close()
h := md5.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
internal/protocols/local/local.go
+15
View File
@@ -1,6 +1,8 @@
package local
import (
"crypto/md5"
"encoding/hex"
"io"
"os"
"path/filepath"
@@ -82,6 +84,19 @@ func (e *LocalExplorer) Rename(src, dst string) error {
return os.Rename(src, dst)
}
func (e *LocalExplorer) Checksum(path string) (string, error) {
r, err := e.ReadFile(path)
if err != nil {
return "", err
}
defer r.Close()
h := md5.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func (e *LocalExplorer) ReadFile(path string) (io.ReadCloser, error) {
return os.Open(path)
}
internal/protocols/nfs/nfs_unix.go
+15
View File
@@ -3,6 +3,8 @@
package nfs
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"path/filepath"
@@ -125,3 +127,16 @@ func (e *NFSExplorer) WriteFile(path string, r io.Reader, size int64) error {
_, err = io.Copy(f, r)
return err
}
func (e *NFSExplorer) Checksum(path string) (string, error) {
r, err := e.ReadFile(path)
if err != nil {
return "", err
}
defer r.Close()
h := md5.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
internal/protocols/nfs/nfs_windows.go
+1
View File
@@ -25,3 +25,4 @@ func (e *NFSExplorer) Delete(path string) error { return nil }
func (e *NFSExplorer) Rename(src, dst string) error { return nil }
func (e *NFSExplorer) ReadFile(path string) (io.ReadCloser, error) { return nil, nil }
func (e *NFSExplorer) WriteFile(path string, r io.Reader, size int64) error { return nil }
func (e *NFSExplorer) Checksum(path string) (string, error) { return "", nil }
internal/protocols/s3/s3.go
+25
View File
@@ -307,3 +307,28 @@ func (e *S3Explorer) WriteFile(path string, r io.Reader, size int64) error {
})
return err
}
func (e *S3Explorer) Checksum(path string) (string, error) {
bucket, subpath, _, err := e.getBucketAndPath(path)
if err != nil {
return "", err
}
if subpath == "" {
return "", fmt.Errorf("cannot calculate checksum of bucket")
}
input := &s3.HeadObjectInput{
Bucket: aws.String(bucket),
Key: aws.String(subpath),
}
head, err := e.client.HeadObject(context.TODO(), input)
if err != nil {
return "", err
}
if head.ETag != nil {
return strings.Trim(*head.ETag, "\""), nil
}
return "", fmt.Errorf("no etag found")
}
internal/protocols/sftp/sftp.go
+15
View File
@@ -1,6 +1,8 @@
package sftp
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"path/filepath"
@@ -162,3 +164,16 @@ func (e *SFTPExplorer) WriteFile(path string, r io.Reader, size int64) error {
_, err = io.Copy(f, r)
return err
}
func (e *SFTPExplorer) Checksum(path string) (string, error) {
r, err := e.ReadFile(path)
if err != nil {
return "", err
}
defer r.Close()
h := md5.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
internal/protocols/smb/smb.go
+15
View File
@@ -1,6 +1,8 @@
package smb
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"net"
@@ -243,3 +245,16 @@ func (e *SMBExplorer) WriteFile(path string, r io.Reader, size int64) error {
_, err = io.Copy(f, r)
return err
}
func (e *SMBExplorer) Checksum(path string) (string, error) {
r, err := e.ReadFile(path)
if err != nil {
return "", err
}
defer r.Close()
h := md5.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
internal/protocols/webdav/webdav.go
+15
View File
@@ -1,6 +1,8 @@
package webdav
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"strings"
@@ -102,3 +104,16 @@ func (e *WebDAVExplorer) ReadFile(path string) (io.ReadCloser, error) {
func (e *WebDAVExplorer) WriteFile(path string, r io.Reader, size int64) error {
return e.client.WriteStream(path, r, 0644)
}
func (e *WebDAVExplorer) Checksum(path string) (string, error) {
r, err := e.ReadFile(path)
if err != nil {
return "", err
}
defer r.Close()
h := md5.New()
if _, err := io.Copy(h, r); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
internal/transfer/transfer.go
+64 -3
View File
@@ -1,11 +1,18 @@
package transfer
import (
"context"
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"strings"
"sync"
"time"
"golang.org/x/time/rate"
"goexplore/internal/explorer"
)
@@ -29,6 +36,8 @@ type Transfer struct {
ETA int `json:"eta_seconds"`
Status Status `json:"status"`
Error string `json:"error,omitempty"`
Verify bool `json:"verify"`
LimitMBps int `json:"limit_mbps"`
srcExp explorer.Explorer
dstExp explorer.Explorer
@@ -81,12 +90,62 @@ func (m *Manager) doTransfer(t *Transfer) error {
}
defer r.Close()
hash := md5.New()
tr := io.TeeReader(r, hash)
var streamReader io.Reader = tr
if t.LimitMBps > 0 {
limitBytes := t.LimitMBps * 1024 * 1024
// Set burst to limitBytes to allow typical io.Copy chunk sizes without erroring
limiter := rate.NewLimiter(rate.Limit(limitBytes), limitBytes)
streamReader = &throttledReader{
r: tr,
limiter: limiter,
}
}
pr := &progressReader{
r: r,
r: streamReader,
t: t,
}
return t.dstExp.WriteFile(t.Destination, pr, t.BytesTotal)
if err := t.dstExp.WriteFile(t.Destination, pr, t.BytesTotal); err != nil {
return err
}
if t.Verify {
expectedHash := hex.EncodeToString(hash.Sum(nil))
t.Status = StatusActive // Still active during verification
actualHash, err := t.dstExp.Checksum(t.Destination)
if err != nil {
return fmt.Errorf("verification failed: could not calculate destination checksum: %w", err)
}
if expectedHash != actualHash {
// S3 Multi-part ETags end with -N, we can't easily verify them with local md5
if !strings.Contains(actualHash, "-") {
return fmt.Errorf("verification failed: checksum mismatch (expected %s, got %s)", expectedHash, actualHash)
}
}
}
return nil
}
type throttledReader struct {
r io.Reader
limiter *rate.Limiter
}
func (tr *throttledReader) Read(p []byte) (int, error) {
n, err := tr.r.Read(p)
if n > 0 && tr.limiter != nil {
if errWait := tr.limiter.WaitN(context.Background(), n); errWait != nil {
return n, errWait
}
}
return n, err
}
type progressReader struct {
@@ -109,7 +168,7 @@ func (pr *progressReader) Read(p []byte) (int, error) {
return n, err
}
func (m *Manager) QueueTransfer(id, srcPath, dstPath, filename string, size int64, srcExp, dstExp explorer.Explorer) error {
func (m *Manager) QueueTransfer(id, srcPath, dstPath, filename string, size int64, srcExp, dstExp explorer.Explorer, verify bool, limitMBps int) error {
if srcExp == nil || dstExp == nil {
return errors.New("invalid explorers")
}
@@ -121,6 +180,8 @@ func (m *Manager) QueueTransfer(id, srcPath, dstPath, filename string, size int6
Filename: filename,
BytesTotal: size,
Status: StatusQueued,
Verify: verify,
LimitMBps: limitMBps,
srcExp: srcExp,
dstExp: dstExp,
}