Mirrors/sftpgo
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2025-12-06 14:20:55 +03:00
Code Issues Packages Projects Releases Wiki Activity

S3: don't use manager for uploads

Browse Source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino
2025-06-16 18:48:04 +02:00
parent cea5dd665e
commit ff5ea7cd40
3 changed files with 255 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
internal/vfs/azblobfs.go
View File

@@ -946,6 +946,8 @@ func (fs *AzureBlobFs) handleMultipartDownload(ctx context.Context, blockBlob *b
guard := make(chan struct{}, fs.config.DownloadConcurrency)
blockCtxTimeout := time.Duration(fs.config.DownloadPartSize/(1024*1024)) * time.Minute
pool := newBufferAllocator(int(partSize))
defer pool.free()
finished := false
var wg sync.WaitGroup
var errOnce sync.Once
@@ -999,7 +1001,6 @@ func (fs *AzureBlobFs) handleMultipartDownload(ctx context.Context, blockBlob *b
wg.Wait()
close(guard)
pool.free()
return poolError
}
@@ -1014,6 +1015,8 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
// sync.Pool seems to use a lot of memory so prefer our own, very simple, allocator
// we only need to recycle few byte slices
pool := newBufferAllocator(int(partSize))
defer pool.free()
finished := false
var blocks []string
var wg sync.WaitGroup
@@ -1027,7 +1030,7 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
for part := 0; !finished; part++ {
buf := pool.getBuffer()
n, err := fs.readFill(reader, buf)
n, err := readFill(reader, buf)
if err == io.EOF {
// read finished, if n > 0 we need to process the last data chunck
if n == 0 {
@@ -1037,7 +1040,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
finished = true
} else if err != nil {
pool.releaseBuffer(buf)
pool.free()
return err
}
@@ -1046,7 +1048,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
generatedUUID, err := uuid.NewRandom()
if err != nil {
pool.releaseBuffer(buf)
pool.free()
return fmt.Errorf("unable to generate block ID: %w", err)
}
blockID := base64.StdEncoding.EncodeToString([]byte(generatedUUID.String()))
@@ -1087,7 +1088,6 @@ func (fs *AzureBlobFs) handleMultipartUpload(ctx context.Context, reader io.Read
wg.Wait()
close(guard)
pool.free()
if poolError != nil {
return poolError
@@ -1117,16 +1117,6 @@ func (*AzureBlobFs) writeAtFull(w io.WriterAt, buf []byte, offset int64, count i
return written, nil
}
// copied from rclone
func (*AzureBlobFs) readFill(r io.Reader, buf []byte) (n int, err error) {
var nn int
for n < len(buf) && err == nil {
nn, err = r.Read(buf[n:])
n += nn
}
return n, err
}
func (fs *AzureBlobFs) getCopyOptions(srcInfo os.FileInfo, updateModTime bool) *blob.StartCopyFromURLOptions {
copyOptions := &blob.StartCopyFromURLOptions{}
if fs.config.AccessTier != "" {
@@ -1187,66 +1177,6 @@ func getAzContainerClientOptions() *container.ClientOptions {
}
}
type bytesReaderWrapper struct {
*bytes.Reader
}
func (b *bytesReaderWrapper) Close() error {
return nil
}
type bufferAllocator struct {
sync.Mutex
available [][]byte
bufferSize int
finalized bool
}
func newBufferAllocator(size int) *bufferAllocator {
return &bufferAllocator{
bufferSize: size,
finalized: false,
}
}
func (b *bufferAllocator) getBuffer() []byte {
b.Lock()
defer b.Unlock()
if len(b.available) > 0 {
var result []byte
truncLength := len(b.available) - 1
result = b.available[truncLength]
b.available[truncLength] = nil
b.available = b.available[:truncLength]
return result
}
return make([]byte, b.bufferSize)
}
func (b *bufferAllocator) releaseBuffer(buf []byte) {
b.Lock()
defer b.Unlock()
if b.finalized || len(buf) != b.bufferSize {
return
}
b.available = append(b.available, buf)
}
func (b *bufferAllocator) free() {
b.Lock()
defer b.Unlock()
b.available = nil
b.finalized = true
}
type azureBlobDirLister struct {
baseDirLister
paginator *runtime.Pager[container.ListBlobsHierarchyResponse]