s3: allow to configure the chunk download timeout

This commit is contained in:
Nicola Murino
2021-07-11 18:39:45 +02:00
parent 7344366ce8
commit e1a2451c22
8 changed files with 66 additions and 16 deletions

View File

@@ -16,6 +16,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awserr"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
@@ -178,10 +179,17 @@ func (fs *S3Fs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, fun
}
ctx, cancelFn := context.WithCancel(context.Background())
downloader := s3manager.NewDownloaderWithClient(fs.svc)
/*downloader.RequestOptions = append(downloader.RequestOptions, func(r *request.Request) {
newCtx, _ := context.WithTimeout(r.Context(), time.Minute)
r.SetContext(newCtx)
})*/
if offset == 0 && fs.config.DownloadPartMaxTime > 0 {
downloader.RequestOptions = append(downloader.RequestOptions, func(r *request.Request) {
chunkCtx, cancel := context.WithTimeout(r.Context(), time.Duration(fs.config.DownloadPartMaxTime)*time.Second)
r.SetContext(chunkCtx)
go func() {
<-ctx.Done()
cancel()
}()
})
}
var streamRange *string
if offset > 0 {
streamRange = aws.String(fmt.Sprintf("bytes=%v-", offset))