From e47d4103ae43c1f43e8300f4e793a97df2877ad7 Mon Sep 17 00:00:00 2001 From: Jamie Banks Date: Sat, 13 Jun 2026 11:41:02 +0100 Subject: [PATCH] feat: implement S3 manager uploader and enable checksum validation for S3 operations --- go.mod | 7 ++++--- go.sum | 8 ++++++++ internal/protocols/s3/s3.go | 7 ++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f685caa..e23aef9 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26 require ( github.com/aws/aws-sdk-go-v2 v1.42.0 - github.com/aws/aws-sdk-go-v2/config v1.32.24 - github.com/aws/aws-sdk-go-v2/credentials v1.19.23 + github.com/aws/aws-sdk-go-v2/config v1.32.25 + github.com/aws/aws-sdk-go-v2/credentials v1.19.24 github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3 github.com/google/uuid v1.6.0 github.com/hirochachacha/go-smb2 v1.1.0 @@ -24,6 +24,7 @@ require ( git.sr.ht/~jackmordaunt/go-toast/v2 v2.0.3 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.27 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect @@ -31,7 +32,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.29 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.1.5 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 // indirect diff --git a/go.sum b/go.sum index 1b3ce69..97401d3 100644 --- a/go.sum +++ b/go.sum @@ -6,10 +6,16 @@ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13 h1:p1BBrg/Hhp6uK7z github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.13/go.mod h1:8cIfkE9MDhkRZGpQ22aV6/lkYeYSozpz16Smrs5x4Ls= github.com/aws/aws-sdk-go-v2/config v1.32.24 h1:aEDEj533yGdVvEHfkCY0D/1FbDrjnZr4pIulxRjqpHs= github.com/aws/aws-sdk-go-v2/config v1.32.24/go.mod h1:yZtrGKJGlqfEW+/m2uTsJK+Jz7xF5R0eZfgcIG9m1ss= +github.com/aws/aws-sdk-go-v2/config v1.32.25 h1:ACCejvStYoilgwrfegSt5ZntCbPrk52qfwyNcnl3omM= +github.com/aws/aws-sdk-go-v2/config v1.32.25/go.mod h1:LJyU8sDRbXUxFn8xMJIGP+v9QYYwveNLI8a/giAOiAs= github.com/aws/aws-sdk-go-v2/credentials v1.19.23 h1:Zhu3GOpRCkNjtE/gJpuPDsytSnaCCTQk8neAGsgzG5Y= github.com/aws/aws-sdk-go-v2/credentials v1.19.23/go.mod h1:VsJF2ropPB37gDr7M2rLSpCE8IQWdpl62uae7qxZmqU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I= +github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 h1:r6qZHbT+wxgWO/e9vYNUEtg7lv5+UN3pRqKhLXvnArg= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29/go.mod h1:QRnaRcTVGKPGRy8w78HMQtKUGRYcnMZAANATkeVA6Mo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.27 h1:gb+HtIZdwcIoLxv/xwGumQr1DmGmGGCQnjKKVVSMYsU= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.27/go.mod h1:2b/8jZl/qwUMBZpSAcxX+IdM3zj6RUyfnB2IdLt9I+I= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29/go.mod h1:MzoLFUArKGpGD+ukmPiTPG1X5x4o6M2kq4v2dr1FiEc= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 h1:RdwIf/CuUsvJX3RgJagbOyotl/cxoLY4xviKuE7p2GY= @@ -28,6 +34,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3 h1:JRseEu/vIDMaWis4bSw0QbXL+cvI github.com/aws/aws-sdk-go-v2/service/s3 v1.103.3/go.mod h1:77ZAgynvx1txMvDG8gGWoWkO1augYDxkp9JElWFgjQU= github.com/aws/aws-sdk-go-v2/service/signin v1.1.5 h1:6Xt6Ztjkwdia/7EtEaG7ki/qZUYlCcd7tGUotQed1QE= github.com/aws/aws-sdk-go-v2/service/signin v1.1.5/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 h1:3nXpRcFwRCW8n7HgO2QGy0Dc20eQNfBuUemGQhpF8m8= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 h1:ey1XLTYXb9PcLt4535632o5kCGXNXEhNb620Dqwuylo= github.com/aws/aws-sdk-go-v2/service/sso v1.31.3/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMbLMbSDF2YXsigqXngs6g= diff --git a/internal/protocols/s3/s3.go b/internal/protocols/s3/s3.go index 3bcd7e2..e49688d 100644 --- a/internal/protocols/s3/s3.go +++ b/internal/protocols/s3/s3.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" appcfg "goexplore/internal/config" @@ -57,6 +58,8 @@ func (e *S3Explorer) Connect() error { o.BaseEndpoint = aws.String(host) } o.UsePathStyle = e.cfg.PathStyle + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired } e.client = s3.NewFromConfig(awsCfg, clientOpts) @@ -351,7 +354,9 @@ func (e *S3Explorer) WriteFile(path string, r io.Reader, size int64) error { if err != nil { return err } - _, err = e.client.PutObject(context.TODO(), &s3.PutObjectInput{ + + uploader := manager.NewUploader(e.client) + _, err = uploader.Upload(context.TODO(), &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(subpath), Body: r, -- 2.47.3