From c63b923ec3ba51253fc39115049fe031dd36ebe0 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Mon, 31 May 2021 21:45:29 +0200 Subject: [PATCH] cryptfs: add support for atomic uploads --- common/common.go | 1 + common/common_test.go | 6 +-- common/transfer.go | 80 ++++++++++++++++++------------------ common/transfer_test.go | 20 ++++----- dataprovider/dataprovider.go | 17 +++++++- docs/dare.md | 1 - ftpd/handler.go | 6 +-- ftpd/internal_test.go | 8 ++-- httpd/handler.go | 2 +- httpd/internal_test.go | 2 +- sftpd/handler.go | 6 +-- sftpd/internal_test.go | 18 ++++---- sftpd/scp.go | 4 +- sftpd/ssh_cmd.go | 6 +-- vfs/cryptfs.go | 5 --- webdavd/handler.go | 6 +-- webdavd/internal_test.go | 26 ++++++------ 17 files changed, 112 insertions(+), 102 deletions(-) diff --git a/common/common.go b/common/common.go index d702cc7c..270f6d98 100644 --- a/common/common.go +++ b/common/common.go @@ -151,6 +151,7 @@ func Initialize(c Configuration) error { } } vfs.SetTempPath(c.TempPath) + dataprovider.SetTempPath(c.TempPath) return nil } diff --git a/common/common_test.go b/common/common_test.go index 0c3e47cb..aa0453ce 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -435,9 +435,9 @@ func TestConnectionStatus(t *testing.T) { fakeConn1 := &fakeConnection{ BaseConnection: c1, } - t1 := NewBaseTransfer(nil, c1, nil, "/p1", "/r1", TransferUpload, 0, 0, 0, true, fs) + t1 := NewBaseTransfer(nil, c1, nil, "/p1", "/p1", "/r1", TransferUpload, 0, 0, 0, true, fs) t1.BytesReceived = 123 - t2 := NewBaseTransfer(nil, c1, nil, "/p2", "/r2", TransferDownload, 0, 0, 0, true, fs) + t2 := NewBaseTransfer(nil, c1, nil, "/p2", "/p2", "/r2", TransferDownload, 0, 0, 0, true, fs) t2.BytesSent = 456 c2 := NewBaseConnection("id2", ProtocolSSH, user) fakeConn2 := &fakeConnection{ @@ -449,7 +449,7 @@ func TestConnectionStatus(t *testing.T) { BaseConnection: c3, command: "PROPFIND", } - t3 := NewBaseTransfer(nil, c3, nil, "/p2", "/r2", TransferDownload, 0, 0, 0, true, fs) + t3 := NewBaseTransfer(nil, c3, nil, "/p2", "/p2", "/r2", TransferDownload, 0, 0, 0, true, fs) Connections.Add(fakeConn1) Connections.Add(fakeConn2) Connections.Add(fakeConn3) diff --git a/common/transfer.go b/common/transfer.go index 7263f2b4..3f2002c5 100644 --- a/common/transfer.go +++ b/common/transfer.go @@ -20,46 +20,48 @@ var ( // BaseTransfer contains protocols common transfer details for an upload or a download. type BaseTransfer struct { //nolint:maligned - ID uint64 - BytesSent int64 - BytesReceived int64 - Fs vfs.Fs - File vfs.File - Connection *BaseConnection - cancelFn func() - fsPath string - requestPath string - start time.Time - MaxWriteSize int64 - MinWriteOffset int64 - InitialSize int64 - isNewFile bool - transferType int - AbortTransfer int32 + ID uint64 + BytesSent int64 + BytesReceived int64 + Fs vfs.Fs + File vfs.File + Connection *BaseConnection + cancelFn func() + fsPath string + effectiveFsPath string + requestPath string + start time.Time + MaxWriteSize int64 + MinWriteOffset int64 + InitialSize int64 + isNewFile bool + transferType int + AbortTransfer int32 sync.Mutex ErrTransfer error } // NewBaseTransfer returns a new BaseTransfer and adds it to the given connection -func NewBaseTransfer(file vfs.File, conn *BaseConnection, cancelFn func(), fsPath, requestPath string, transferType int, - minWriteOffset, initialSize, maxWriteSize int64, isNewFile bool, fs vfs.Fs) *BaseTransfer { +func NewBaseTransfer(file vfs.File, conn *BaseConnection, cancelFn func(), fsPath, effectiveFsPath, requestPath string, + transferType int, minWriteOffset, initialSize, maxWriteSize int64, isNewFile bool, fs vfs.Fs) *BaseTransfer { t := &BaseTransfer{ - ID: conn.GetTransferID(), - File: file, - Connection: conn, - cancelFn: cancelFn, - fsPath: fsPath, - start: time.Now(), - transferType: transferType, - MinWriteOffset: minWriteOffset, - InitialSize: initialSize, - isNewFile: isNewFile, - requestPath: requestPath, - BytesSent: 0, - BytesReceived: 0, - MaxWriteSize: maxWriteSize, - AbortTransfer: 0, - Fs: fs, + ID: conn.GetTransferID(), + File: file, + Connection: conn, + cancelFn: cancelFn, + fsPath: fsPath, + effectiveFsPath: effectiveFsPath, + start: time.Now(), + transferType: transferType, + MinWriteOffset: minWriteOffset, + InitialSize: initialSize, + isNewFile: isNewFile, + requestPath: requestPath, + BytesSent: 0, + BytesReceived: 0, + MaxWriteSize: maxWriteSize, + AbortTransfer: 0, + Fs: fs, } conn.AddTransfer(t) @@ -215,15 +217,15 @@ func (t *BaseTransfer) Close() error { } t.Connection.Log(logger.LevelWarn, "upload denied due to space limit, delete temporary file: %#v, deletion error: %v", t.File.Name(), err) - } else if t.transferType == TransferUpload && t.File != nil && t.File.Name() != t.fsPath { + } else if t.transferType == TransferUpload && t.effectiveFsPath != t.fsPath { if t.ErrTransfer == nil || Config.UploadMode == UploadModeAtomicWithResume { - err = t.Fs.Rename(t.File.Name(), t.fsPath) + err = t.Fs.Rename(t.effectiveFsPath, t.fsPath) t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %#v -> %#v, error: %v", - t.File.Name(), t.fsPath, err) + t.effectiveFsPath, t.fsPath, err) } else { - err = t.Fs.Remove(t.File.Name(), false) + err = t.Fs.Remove(t.effectiveFsPath, false) t.Connection.Log(logger.LevelWarn, "atomic upload completed with error: \"%v\", delete temporary file: %#v, "+ - "deletion error: %v", t.ErrTransfer, t.File.Name(), err) + "deletion error: %v", t.ErrTransfer, t.effectiveFsPath, err) if err == nil { numFiles-- atomic.StoreInt64(&t.BytesReceived, 0) diff --git a/common/transfer_test.go b/common/transfer_test.go index 082b2f43..66abce3d 100644 --- a/common/transfer_test.go +++ b/common/transfer_test.go @@ -62,7 +62,7 @@ func TestTransferThrottling(t *testing.T) { wantedUploadElapsed -= wantedDownloadElapsed / 10 wantedDownloadElapsed -= wantedDownloadElapsed / 10 conn := NewBaseConnection("id", ProtocolSCP, u) - transfer := NewBaseTransfer(nil, conn, nil, "", "", TransferUpload, 0, 0, 0, true, fs) + transfer := NewBaseTransfer(nil, conn, nil, "", "", "", TransferUpload, 0, 0, 0, true, fs) transfer.BytesReceived = testFileSize transfer.Connection.UpdateLastActivity() startTime := transfer.Connection.GetLastActivity() @@ -72,7 +72,7 @@ func TestTransferThrottling(t *testing.T) { err := transfer.Close() assert.NoError(t, err) - transfer = NewBaseTransfer(nil, conn, nil, "", "", TransferDownload, 0, 0, 0, true, fs) + transfer = NewBaseTransfer(nil, conn, nil, "", "", "", TransferDownload, 0, 0, 0, true, fs) transfer.BytesSent = testFileSize transfer.Connection.UpdateLastActivity() startTime = transfer.Connection.GetLastActivity() @@ -96,7 +96,7 @@ func TestRealPath(t *testing.T) { file, err := os.Create(testFile) require.NoError(t, err) conn := NewBaseConnection(fs.ConnectionID(), ProtocolSFTP, u) - transfer := NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs) + transfer := NewBaseTransfer(file, conn, nil, testFile, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs) rPath := transfer.GetRealFsPath(testFile) assert.Equal(t, testFile, rPath) rPath = conn.getRealFsPath(testFile) @@ -131,7 +131,7 @@ func TestTruncate(t *testing.T) { _, err = file.Write([]byte("hello")) assert.NoError(t, err) conn := NewBaseConnection(fs.ConnectionID(), ProtocolSFTP, u) - transfer := NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 5, 100, false, fs) + transfer := NewBaseTransfer(file, conn, nil, testFile, testFile, "/transfer_test_file", TransferUpload, 0, 5, 100, false, fs) err = conn.SetStat("/transfer_test_file", &StatAttributes{ Size: 2, @@ -148,7 +148,7 @@ func TestTruncate(t *testing.T) { assert.Equal(t, int64(2), fi.Size()) } - transfer = NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, 100, true, fs) + transfer = NewBaseTransfer(file, conn, nil, testFile, testFile, "/transfer_test_file", TransferUpload, 0, 0, 100, true, fs) // file.Stat will fail on a closed file err = conn.SetStat("/transfer_test_file", &StatAttributes{ Size: 2, @@ -158,7 +158,7 @@ func TestTruncate(t *testing.T) { err = transfer.Close() assert.NoError(t, err) - transfer = NewBaseTransfer(nil, conn, nil, testFile, "", TransferUpload, 0, 0, 0, true, fs) + transfer = NewBaseTransfer(nil, conn, nil, testFile, testFile, "", TransferUpload, 0, 0, 0, true, fs) _, err = transfer.Truncate("mismatch", 0) assert.EqualError(t, err, errTransferMismatch.Error()) _, err = transfer.Truncate(testFile, 0) @@ -193,7 +193,7 @@ func TestTransferErrors(t *testing.T) { assert.FailNow(t, "unable to open test file") } conn := NewBaseConnection("id", ProtocolSFTP, u) - transfer := NewBaseTransfer(file, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs) + transfer := NewBaseTransfer(file, conn, nil, testFile, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs) assert.Nil(t, transfer.cancelFn) assert.Equal(t, testFile, transfer.GetFsPath()) transfer.SetCancelFn(cancelFn) @@ -219,7 +219,7 @@ func TestTransferErrors(t *testing.T) { assert.FailNow(t, "unable to open test file") } fsPath := filepath.Join(os.TempDir(), "test_file") - transfer = NewBaseTransfer(file, conn, nil, fsPath, "/test_file", TransferUpload, 0, 0, 0, true, fs) + transfer = NewBaseTransfer(file, conn, nil, fsPath, file.Name(), "/test_file", TransferUpload, 0, 0, 0, true, fs) transfer.BytesReceived = 9 transfer.TransferError(errFake) assert.Error(t, transfer.ErrTransfer, errFake.Error()) @@ -238,7 +238,7 @@ func TestTransferErrors(t *testing.T) { if !assert.NoError(t, err) { assert.FailNow(t, "unable to open test file") } - transfer = NewBaseTransfer(file, conn, nil, fsPath, "/test_file", TransferUpload, 0, 0, 0, true, fs) + transfer = NewBaseTransfer(file, conn, nil, fsPath, file.Name(), "/test_file", TransferUpload, 0, 0, 0, true, fs) transfer.BytesReceived = 9 // the file is closed from the embedding struct before to call close err = file.Close() @@ -262,7 +262,7 @@ func TestRemovePartialCryptoFile(t *testing.T) { HomeDir: os.TempDir(), } conn := NewBaseConnection(fs.ConnectionID(), ProtocolSFTP, u) - transfer := NewBaseTransfer(nil, conn, nil, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs) + transfer := NewBaseTransfer(nil, conn, nil, testFile, testFile, "/transfer_test_file", TransferUpload, 0, 0, 0, true, fs) transfer.ErrTransfer = errors.New("test error") _, err = transfer.getUploadFileSize() assert.Error(t, err) diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 097f405c..4c2a6fcc 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -141,6 +141,7 @@ var ( argon2Params *argon2id.Params lastLoginMinDelay = 10 * time.Minute usernameRegex = regexp.MustCompile("^[a-zA-Z0-9-_.~]+$") + tempPath string ) type schemaVersion struct { @@ -449,6 +450,11 @@ type fsValidatorHelper interface { GetEncrytionAdditionalData() string } +// SetTempPath sets the path for temporary files +func SetTempPath(fsPath string) { + tempPath = fsPath +} + // Initialize the data provider. // An error is returned if the configured driver is invalid or if the data provider cannot be initialized func Initialize(cnf Config, basePath string, checkAdmins bool) error { @@ -1097,8 +1103,15 @@ func buildUserHomeDir(user *User) { if user.HomeDir == "" { if config.UsersBaseDir != "" { user.HomeDir = filepath.Join(config.UsersBaseDir, user.Username) - } else if user.FsConfig.Provider == vfs.SFTPFilesystemProvider { - user.HomeDir = filepath.Join(os.TempDir(), user.Username) + return + } + switch user.FsConfig.Provider { + case vfs.SFTPFilesystemProvider, vfs.S3FilesystemProvider, vfs.AzureBlobFilesystemProvider, vfs.GCSFilesystemProvider: + if tempPath != "" { + user.HomeDir = filepath.Join(tempPath, user.Username) + } else { + user.HomeDir = filepath.Join(os.TempDir(), user.Username) + } } } } diff --git a/docs/dare.md b/docs/dare.md index 23d6febc..12abc117 100644 --- a/docs/dare.md +++ b/docs/dare.md @@ -15,5 +15,4 @@ The encrypted filesystem has some limitations compared to the local, unencrypted - Resuming uploads is not supported. - Opening a file for both reading and writing at the same time is not supported and so clients that require advanced filesystem-like features such as `sshfs` are not supported too. - Truncate is not supported. -- Atomic uploads are not supported. - System commands such as `git` or `rsync` are not supported: they will store data unencrypted. diff --git a/ftpd/handler.go b/ftpd/handler.go index 00d2636f..4407a090 100644 --- a/ftpd/handler.go +++ b/ftpd/handler.go @@ -308,7 +308,7 @@ func (c *Connection) downloadFile(fs vfs.Fs, fsPath, ftpPath string, offset int6 return nil, c.GetFsError(fs, err) } - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, fsPath, ftpPath, common.TransferDownload, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, fsPath, fsPath, ftpPath, common.TransferDownload, 0, 0, 0, false, fs) t := newTransfer(baseTransfer, nil, r, offset) @@ -373,7 +373,7 @@ func (c *Connection) handleFTPUploadToNewFile(fs vfs.Fs, resolvedPath, filePath, // we can get an error only for resume maxWriteSize, _ := c.GetMaxWriteSize(quotaResult, false, 0, fs.IsUploadResumeSupported()) - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, common.TransferUpload, 0, 0, maxWriteSize, true, fs) t := newTransfer(baseTransfer, w, nil, 0) @@ -449,7 +449,7 @@ func (c *Connection) handleFTPUploadToExistingFile(fs vfs.Fs, flags int, resolve vfs.SetPathPermissions(fs, filePath, c.User.GetUID(), c.User.GetGID()) - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, common.TransferUpload, minWriteOffset, initialSize, maxWriteSize, false, fs) t := newTransfer(baseTransfer, w, nil, 0) diff --git a/ftpd/internal_test.go b/ftpd/internal_test.go index 9a8a3d1a..afd1a547 100644 --- a/ftpd/internal_test.go +++ b/ftpd/internal_test.go @@ -724,8 +724,8 @@ func TestTransferErrors(t *testing.T) { BaseConnection: common.NewBaseConnection(connID, common.ProtocolFTP, user), clientContext: mockCC, } - baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, file.Name(), testfile, common.TransferDownload, - 0, 0, 0, false, fs) + baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, file.Name(), file.Name(), testfile, + common.TransferDownload, 0, 0, 0, false, fs) tr := newTransfer(baseTransfer, nil, nil, 0) err = tr.Close() assert.NoError(t, err) @@ -742,7 +742,7 @@ func TestTransferErrors(t *testing.T) { r, _, err := pipeat.Pipe() assert.NoError(t, err) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile, testfile, common.TransferUpload, 0, 0, 0, false, fs) tr = newTransfer(baseTransfer, nil, r, 10) pos, err := tr.Seek(10, 0) @@ -754,7 +754,7 @@ func TestTransferErrors(t *testing.T) { r, w, err := pipeat.Pipe() assert.NoError(t, err) pipeWriter := vfs.NewPipeWriter(w) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testfile, testfile, testfile, common.TransferUpload, 0, 0, 0, false, fs) tr = newTransfer(baseTransfer, pipeWriter, nil, 0) diff --git a/httpd/handler.go b/httpd/handler.go index c933da3d..abb864f3 100644 --- a/httpd/handler.go +++ b/httpd/handler.go @@ -105,7 +105,7 @@ func (c *Connection) getFileReader(name string, offset int64, method string) (io return nil, c.GetFsError(fs, err) } - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, p, name, common.TransferDownload, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, p, p, name, common.TransferDownload, 0, 0, 0, false, fs) return newHTTPDFile(baseTransfer, r), nil } diff --git a/httpd/internal_test.go b/httpd/internal_test.go index d4b3403e..e98087de 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -1428,7 +1428,7 @@ func TestHTTPDFile(t *testing.T) { err = file.Close() assert.NoError(t, err) - baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, p, name, common.TransferDownload, + baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, p, p, name, common.TransferDownload, 0, 0, 0, false, fs) httpdFile := newHTTPDFile(baseTransfer, nil) // the file is closed, read should fail diff --git a/sftpd/handler.go b/sftpd/handler.go index b92a6f4a..fc9309cb 100644 --- a/sftpd/handler.go +++ b/sftpd/handler.go @@ -70,7 +70,7 @@ func (c *Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) { return nil, c.GetFsError(fs, err) } - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, p, request.Filepath, common.TransferDownload, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, p, p, request.Filepath, common.TransferDownload, 0, 0, 0, false, fs) t := newTransfer(baseTransfer, nil, r, nil) @@ -346,7 +346,7 @@ func (c *Connection) handleSFTPUploadToNewFile(fs vfs.Fs, resolvedPath, filePath // we can get an error only for resume maxWriteSize, _ := c.GetMaxWriteSize(quotaResult, false, 0, fs.IsUploadResumeSupported()) - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, common.TransferUpload, 0, 0, maxWriteSize, true, fs) t := newTransfer(baseTransfer, w, nil, errForRead) @@ -423,7 +423,7 @@ func (c *Connection) handleSFTPUploadToExistingFile(fs vfs.Fs, pflags sftp.FileO vfs.SetPathPermissions(fs, filePath, c.User.GetUID(), c.User.GetGID()) - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, common.TransferUpload, minWriteOffset, initialSize, maxWriteSize, false, fs) t := newTransfer(baseTransfer, w, nil, errForRead) diff --git a/sftpd/internal_test.go b/sftpd/internal_test.go index 15c4b659..9ea5d652 100644 --- a/sftpd/internal_test.go +++ b/sftpd/internal_test.go @@ -159,7 +159,7 @@ func TestUploadResumeInvalidOffset(t *testing.T) { } fs := vfs.NewOsFs("", os.TempDir(), "") conn := common.NewBaseConnection("", common.ProtocolSFTP, user) - baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), testfile, common.TransferUpload, 10, 0, 0, false, fs) + baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), file.Name(), testfile, common.TransferUpload, 10, 0, 0, false, fs) transfer := newTransfer(baseTransfer, nil, nil, nil) _, err = transfer.WriteAt([]byte("test"), 0) assert.Error(t, err, "upload with invalid offset must fail") @@ -187,7 +187,7 @@ func TestReadWriteErrors(t *testing.T) { } fs := vfs.NewOsFs("", os.TempDir(), "") conn := common.NewBaseConnection("", common.ProtocolSFTP, user) - baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) + baseTransfer := common.NewBaseTransfer(file, conn, nil, file.Name(), file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) transfer := newTransfer(baseTransfer, nil, nil, nil) err = file.Close() assert.NoError(t, err) @@ -201,7 +201,7 @@ func TestReadWriteErrors(t *testing.T) { r, _, err := pipeat.Pipe() assert.NoError(t, err) - baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) + baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) transfer = newTransfer(baseTransfer, nil, r, nil) err = transfer.Close() assert.NoError(t, err) @@ -211,7 +211,7 @@ func TestReadWriteErrors(t *testing.T) { r, w, err := pipeat.Pipe() assert.NoError(t, err) pipeWriter := vfs.NewPipeWriter(w) - baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) + baseTransfer = common.NewBaseTransfer(nil, conn, nil, file.Name(), file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) transfer = newTransfer(baseTransfer, pipeWriter, nil, nil) err = r.Close() @@ -256,7 +256,7 @@ func TestTransferCancelFn(t *testing.T) { } fs := vfs.NewOsFs("", os.TempDir(), "") conn := common.NewBaseConnection("", common.ProtocolSFTP, user) - baseTransfer := common.NewBaseTransfer(file, conn, cancelFn, file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) + baseTransfer := common.NewBaseTransfer(file, conn, cancelFn, file.Name(), file.Name(), testfile, common.TransferDownload, 0, 0, 0, false, fs) transfer := newTransfer(baseTransfer, nil, nil, nil) errFake := errors.New("fake error, this will trigger cancelFn") @@ -948,7 +948,7 @@ func TestSystemCommandErrors(t *testing.T) { WriteError: nil, } sshCmd.connection.channel = &mockSSHChannel - baseTransfer := common.NewBaseTransfer(nil, sshCmd.connection.BaseConnection, nil, "", "", common.TransferDownload, + baseTransfer := common.NewBaseTransfer(nil, sshCmd.connection.BaseConnection, nil, "", "", "", common.TransferDownload, 0, 0, 0, false, fs) transfer := newTransfer(baseTransfer, nil, nil, nil) destBuff := make([]byte, 65535) @@ -1605,7 +1605,7 @@ func TestSCPUploadFiledata(t *testing.T) { file, err := os.Create(testfile) assert.NoError(t, err) - baseTransfer := common.NewBaseTransfer(file, scpCommand.connection.BaseConnection, nil, file.Name(), + baseTransfer := common.NewBaseTransfer(file, scpCommand.connection.BaseConnection, nil, file.Name(), file.Name(), "/"+testfile, common.TransferDownload, 0, 0, 0, true, fs) transfer := newTransfer(baseTransfer, nil, nil, nil) @@ -1688,7 +1688,7 @@ func TestUploadError(t *testing.T) { fileTempName := "temptestfile" file, err := os.Create(fileTempName) assert.NoError(t, err) - baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, testfile, + baseTransfer := common.NewBaseTransfer(file, connection.BaseConnection, nil, testfile, file.Name(), testfile, common.TransferUpload, 0, 0, 0, true, fs) transfer := newTransfer(baseTransfer, nil, nil, nil) @@ -1745,7 +1745,7 @@ func TestTransferFailingReader(t *testing.T) { r, _, err := pipeat.Pipe() assert.NoError(t, err) - baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, fsPath, filepath.Base(fsPath), common.TransferUpload, 0, 0, 0, false, fs) + baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, fsPath, fsPath, filepath.Base(fsPath), common.TransferUpload, 0, 0, 0, false, fs) errRead := errors.New("read is not allowed") tr := newTransfer(baseTransfer, nil, r, errRead) _, err = tr.ReadAt(buf, 0) diff --git a/sftpd/scp.go b/sftpd/scp.go index 456140ae..660af757 100644 --- a/sftpd/scp.go +++ b/sftpd/scp.go @@ -258,7 +258,7 @@ func (c *scpCommand) handleUploadFile(fs vfs.Fs, resolvedPath, filePath string, vfs.SetPathPermissions(fs, filePath, c.connection.User.GetUID(), c.connection.User.GetGID()) - baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, resolvedPath, requestPath, + baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, common.TransferUpload, 0, initialSize, maxWriteSize, isNewFile, fs) t := newTransfer(baseTransfer, w, nil, nil) @@ -527,7 +527,7 @@ func (c *scpCommand) handleDownload(filePath string) error { return err } - baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, p, filePath, + baseTransfer := common.NewBaseTransfer(file, c.connection.BaseConnection, cancelFn, p, p, filePath, common.TransferDownload, 0, 0, 0, false, fs) t := newTransfer(baseTransfer, nil, r, nil) diff --git a/sftpd/ssh_cmd.go b/sftpd/ssh_cmd.go index 6cfe4beb..466b96dd 100644 --- a/sftpd/ssh_cmd.go +++ b/sftpd/ssh_cmd.go @@ -354,7 +354,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error { go func() { defer stdin.Close() - baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath, + baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, command.fsPath, sshDestPath, common.TransferUpload, 0, 0, remainingQuotaSize, false, command.fs) transfer := newTransfer(baseTransfer, nil, nil, nil) @@ -367,7 +367,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error { }() go func() { - baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath, + baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, command.fsPath, sshDestPath, common.TransferDownload, 0, 0, 0, false, command.fs) transfer := newTransfer(baseTransfer, nil, nil, nil) @@ -381,7 +381,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error { }() go func() { - baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, sshDestPath, + baseTransfer := common.NewBaseTransfer(nil, c.connection.BaseConnection, nil, command.fsPath, command.fsPath, sshDestPath, common.TransferDownload, 0, 0, 0, false, command.fs) transfer := newTransfer(baseTransfer, nil, nil, nil) diff --git a/vfs/cryptfs.go b/vfs/cryptfs.go index 140d7b7f..57897bdc 100644 --- a/vfs/cryptfs.go +++ b/vfs/cryptfs.go @@ -220,11 +220,6 @@ func (*CryptFs) IsUploadResumeSupported() bool { return false } -// IsAtomicUploadSupported returns true if atomic upload is supported -func (*CryptFs) IsAtomicUploadSupported() bool { - return false -} - // GetMimeType returns the content type func (fs *CryptFs) GetMimeType(name string) (string, error) { f, key, err := fs.getFileAndEncryptionKey(name) diff --git a/webdavd/handler.go b/webdavd/handler.go index 0db4ba99..d73cf09a 100644 --- a/webdavd/handler.go +++ b/webdavd/handler.go @@ -145,7 +145,7 @@ func (c *Connection) getFile(fs vfs.Fs, fsPath, virtualPath string) (webdav.File } } - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, fsPath, virtualPath, common.TransferDownload, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, fsPath, fsPath, virtualPath, common.TransferDownload, 0, 0, 0, false, fs) return newWebDavFile(baseTransfer, nil, r), nil @@ -209,7 +209,7 @@ func (c *Connection) handleUploadToNewFile(fs vfs.Fs, resolvedPath, filePath, re // we can get an error only for resume maxWriteSize, _ := c.GetMaxWriteSize(quotaResult, false, 0, fs.IsUploadResumeSupported()) - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, common.TransferUpload, 0, 0, maxWriteSize, true, fs) return newWebDavFile(baseTransfer, w, nil), nil @@ -263,7 +263,7 @@ func (c *Connection) handleUploadToExistingFile(fs vfs.Fs, resolvedPath, filePat vfs.SetPathPermissions(fs, filePath, c.User.GetUID(), c.User.GetGID()) - baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, requestPath, + baseTransfer := common.NewBaseTransfer(file, c.BaseConnection, cancelFn, resolvedPath, filePath, requestPath, common.TransferUpload, 0, initialSize, maxWriteSize, false, fs) return newWebDavFile(baseTransfer, w, nil), nil diff --git a/webdavd/internal_test.go b/webdavd/internal_test.go index 443007cb..b8b263b3 100644 --- a/webdavd/internal_test.go +++ b/webdavd/internal_test.go @@ -683,7 +683,7 @@ func TestContentType(t *testing.T) { } testFilePath := filepath.Join(user.HomeDir, testFile) ctx := context.Background() - baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) fs = newMockOsFs(nil, false, fs.ConnectionID(), user.GetHomeDir(), nil) err := os.WriteFile(testFilePath, []byte(""), os.ModePerm) @@ -731,7 +731,7 @@ func TestTransferReadWriteErrors(t *testing.T) { BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolWebDAV, user), } testFilePath := filepath.Join(user.HomeDir, testFile) - baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferUpload, 0, 0, 0, false, fs) davFile := newWebDavFile(baseTransfer, nil, nil) p := make([]byte, 1) @@ -749,7 +749,7 @@ func TestTransferReadWriteErrors(t *testing.T) { err = w.Close() assert.NoError(t, err) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil) _, err = davFile.Read(p) @@ -757,7 +757,7 @@ func TestTransferReadWriteErrors(t *testing.T) { _, err = davFile.Stat() assert.True(t, os.IsNotExist(err)) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) err = os.WriteFile(testFilePath, []byte(""), os.ModePerm) assert.NoError(t, err) @@ -782,7 +782,7 @@ func TestTransferReadWriteErrors(t *testing.T) { r, w, err = pipeat.Pipe() assert.NoError(t, err) mockFs := newMockOsFs(nil, false, fs.ConnectionID(), user.HomeDir, r) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, mockFs) davFile = newWebDavFile(baseTransfer, nil, nil) @@ -802,7 +802,7 @@ func TestTransferReadWriteErrors(t *testing.T) { err = davFile.Close() assert.NoError(t, err) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil) davFile.writer = f @@ -825,7 +825,7 @@ func TestTransferSeek(t *testing.T) { } testFilePath := filepath.Join(user.HomeDir, testFile) testFileContents := []byte("content") - baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer := common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferUpload, 0, 0, 0, false, fs) davFile := newWebDavFile(baseTransfer, nil, nil) _, err := davFile.Seek(0, io.SeekStart) @@ -833,7 +833,7 @@ func TestTransferSeek(t *testing.T) { err = davFile.Close() assert.NoError(t, err) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil) _, err = davFile.Seek(0, io.SeekCurrent) @@ -847,14 +847,14 @@ func TestTransferSeek(t *testing.T) { err = f.Close() assert.NoError(t, err) } - baseTransfer = common.NewBaseTransfer(f, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(f, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil) _, err = davFile.Seek(0, io.SeekStart) assert.Error(t, err) davFile.Connection.RemoveTransfer(davFile.BaseTransfer) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil) res, err := davFile.Seek(0, io.SeekStart) @@ -869,14 +869,14 @@ func TestTransferSeek(t *testing.T) { err = davFile.updateStatInfo() assert.Nil(t, err) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFilePath+"1", testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil) _, err = davFile.Seek(0, io.SeekEnd) assert.True(t, os.IsNotExist(err)) davFile.Connection.RemoveTransfer(davFile.BaseTransfer) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath, testFilePath, testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil) davFile.reader = f @@ -891,7 +891,7 @@ func TestTransferSeek(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(5), res) - baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFile, + baseTransfer = common.NewBaseTransfer(nil, connection.BaseConnection, nil, testFilePath+"1", testFilePath+"1", testFile, common.TransferDownload, 0, 0, 0, false, fs) davFile = newWebDavFile(baseTransfer, nil, nil)