Mirrors/sftpgo
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2025-12-07 06:40:54 +03:00
Code Issues Packages Projects Releases Wiki Activity

s3: upload concurrency is now configurable

Browse Source 
Please note that if the upload bandwidth between the SFTP client and
SFTPGo is greater than the upload bandwidth between SFTPGo and S3 then
the SFTP client have to wait for the upload of the last parts to S3
after it ends the file upload to SFTPGo, and it may time out.
Keep this in mind if you customize parts size and upload concurrency
This commit is contained in:
Nicola Murino
2020-03-13 19:13:58 +01:00
parent de3e69f846
commit 1770da545d
13 changed files with 87 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
vfs/s3fs.go
View File

@@ -39,10 +39,13 @@ type S3FsConfig struct {
// The buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB,
// and if this value is set to zero, the default value (5MB) for the AWS SDK will be used.
// The minimum allowed value is 5.
// Please note that if the upload bandwidth between the SFTP client and SFTPGo is greater than the
// upload bandwidth between SFTPGo and S3 then the SFTP client will idle wait for the uploads of
// the last parts, and it could timeout. Keep this in mind if you customize these parameters.
// Please note that if the upload bandwidth between the SFTP client and SFTPGo is greater than
// the upload bandwidth between SFTPGo and S3 then the SFTP client have to wait for the upload
// of the last parts to S3 after it ends the file upload to SFTPGo, and it may time out.
// Keep this in mind if you customize these parameters.
UploadPartSize int64 `json:"upload_part_size,omitempty"`
// How many parts are uploaded in parallel
UploadConcurrency int `json:"upload_concurrency,omitempty"`
}
// S3Fs is a Fs implementation for Amazon S3 compatible object storage.
@@ -93,6 +96,9 @@ func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) {
} else {
fs.config.UploadPartSize *= 1024 * 1024
}
if fs.config.UploadConcurrency == 0 {
fs.config.UploadConcurrency = 2
}
sessOpts := session.Options{
Config: *awsConfig,
@@ -213,7 +219,7 @@ func (fs S3Fs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, fu
Body: r,
StorageClass: utils.NilIfEmpty(fs.config.StorageClass),
}, func(u *s3manager.Uploader) {
u.Concurrency = 2
u.Concurrency = fs.config.UploadConcurrency
u.PartSize = fs.config.UploadPartSize
})
r.CloseWithError(err)

3
vfs/vfs.go
View File

@@ -106,6 +106,9 @@ func ValidateS3FsConfig(config *S3FsConfig) error {
if config.UploadPartSize != 0 && config.UploadPartSize < 5 {
return errors.New("upload_part_size cannot be != 0 and lower than 5 (MB)")
}
if config.UploadConcurrency < 0 {
return fmt.Errorf("invalid upload concurrency: %v", config.UploadConcurrency)
}
return nil
}