s3: export PartSize parameter

By default AWS SDK use part_size of 5 MB. For big files,
it is not ideal case. For Hadoop, it is not uncommon to
use 512 MB.
This commit is contained in:
Michael Bonfils
2020-03-13 08:54:36 +01:00
committed by drakkan
parent 6b70f0b25f
commit cdf1233065
5 changed files with 27 additions and 1 deletions

View File

@@ -103,6 +103,9 @@ func ValidateS3FsConfig(config *S3FsConfig) error {
config.KeyPrefix += "/"
}
}
if config.PartSize != 0 && config.PartSize < 5 {
return errors.New("part_size ret cannot be lower than 5MB")
}
return nil
}