From 77f3400161caae154ee15e4aa41c0345d04867a8 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 2 Apr 2022 18:32:46 +0200 Subject: [PATCH] allow to mount virtual folders on root (/) path Fixes #783 Signed-off-by: Nicola Murino --- README.md | 2 +- common/common_test.go | 10 ++-- common/protocol_test.go | 101 +++++++++++++++++++++++++++++++++++ common/transfer.go | 23 ++++++-- common/transfer_test.go | 31 +++++++++++ dataprovider/dataprovider.go | 53 ++++-------------- dataprovider/user.go | 30 ++++++----- docs/virtual-folders.md | 17 ++++-- go.mod | 6 +-- go.sum | 16 +++--- httpd/httpd_test.go | 56 ++----------------- openapi/openapi.yaml | 6 +-- sftpd/internal_test.go | 16 ++++-- util/util.go | 2 +- vfs/azblobfs.go | 12 +++-- vfs/cryptfs.go | 2 +- vfs/gcsfs.go | 2 +- vfs/osfs.go | 2 +- vfs/s3fs.go | 2 +- vfs/sftpfs.go | 2 +- vfs/vfs.go | 17 ++++++ webdavd/server.go | 2 +- 22 files changed, 257 insertions(+), 153 deletions(-) diff --git a/README.md b/README.md index 1a9b146a..12f2cf74 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Several storage backends are supported: local filesystem, encrypted local filesy - Support for serving local filesystem, encrypted local filesystem, S3 Compatible Object Storage, Google Cloud Storage, Azure Blob Storage or other SFTP accounts over SFTP/SCP/FTP/WebDAV. - Virtual folders are supported: a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. -- Configurable [custom commands and/or HTTP hooks](./docs/custom-actions.md) on file upload, pre-upload, download, pre-download, delete, pre-delete, rename, mkdir, rmdir on SSH commands and on user add, update and delete. +- Configurable [custom commands and/or HTTP hooks](./docs/custom-actions.md) on upload, pre-upload, download, pre-download, delete, pre-delete, rename, mkdir, rmdir on SSH commands and on user add, update and delete. - Virtual accounts stored within a "data provider". - SQLite, MySQL, PostgreSQL, CockroachDB, Bolt (key/value store in pure Go) and in-memory data providers are supported. - Chroot isolation for local accounts. Cloud-based accounts can be restricted to a certain base path. diff --git a/common/common_test.go b/common/common_test.go index 35602d32..4f8c12a5 100644 --- a/common/common_test.go +++ b/common/common_test.go @@ -886,13 +886,17 @@ func TestCachedFs(t *testing.T) { _, p, err = conn.GetFsAndResolvedPath("/") assert.NoError(t, err) assert.Equal(t, filepath.Clean(os.TempDir()), p) - user.FsConfig.Provider = sdk.S3FilesystemProvider - _, err = user.GetFilesystem("") - assert.Error(t, err) + // the filesystem is cached changing the provider will not affect the connection conn.User.FsConfig.Provider = sdk.S3FilesystemProvider _, p, err = conn.GetFsAndResolvedPath("/") assert.NoError(t, err) assert.Equal(t, filepath.Clean(os.TempDir()), p) + user = dataprovider.User{} + user.HomeDir = filepath.Join(os.TempDir(), "temp") + user.FsConfig.Provider = sdk.S3FilesystemProvider + _, err = user.GetFilesystem("") + assert.Error(t, err) + err = os.Remove(user.HomeDir) assert.NoError(t, err) } diff --git a/common/protocol_test.go b/common/protocol_test.go index be285953..ef39bb8b 100644 --- a/common/protocol_test.go +++ b/common/protocol_test.go @@ -660,6 +660,107 @@ func TestFileNotAllowedErrors(t *testing.T) { assert.NoError(t, err) } +func TestRootDirVirtualFolder(t *testing.T) { + u := getTestUser() + u.QuotaFiles = 1000 + u.UploadDataTransfer = 1000 + u.DownloadDataTransfer = 5000 + mappedPath1 := filepath.Join(os.TempDir(), "mapped1") + folderName1 := filepath.Base(mappedPath1) + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: folderName1, + MappedPath: mappedPath1, + FsConfig: vfs.Filesystem{ + Provider: sdk.CryptedFilesystemProvider, + CryptConfig: vfs.CryptFsConfig{ + Passphrase: kms.NewPlainSecret("cryptsecret"), + }, + }, + }, + VirtualPath: "/", + QuotaFiles: 1000, + }) + mappedPath2 := filepath.Join(os.TempDir(), "mapped2") + folderName2 := filepath.Base(mappedPath2) + vdirPath2 := "/vmapped" + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + BaseVirtualFolder: vfs.BaseVirtualFolder{ + Name: folderName2, + MappedPath: mappedPath2, + }, + VirtualPath: vdirPath2, + QuotaFiles: -1, + QuotaSize: -1, + }) + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + conn, client, err := getSftpClient(user) + if assert.NoError(t, err) { + defer conn.Close() + defer client.Close() + + err = checkBasicSFTP(client) + assert.NoError(t, err) + f, err := client.Create(testFileName) + if assert.NoError(t, err) { + _, err = f.Write(testFileContent) + assert.NoError(t, err) + err = f.Close() + assert.NoError(t, err) + } + assert.NoFileExists(t, filepath.Join(user.HomeDir, testFileName)) + assert.FileExists(t, filepath.Join(mappedPath1, testFileName)) + entries, err := client.ReadDir(".") + if assert.NoError(t, err) { + assert.Len(t, entries, 2) + } + + user, _, err := httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 0, user.UsedQuotaFiles) + folder, _, err := httpdtest.GetFolderByName(folderName1, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 1, folder.UsedQuotaFiles) + + f, err = client.Create(path.Join(vdirPath2, testFileName)) + if assert.NoError(t, err) { + _, err = f.Write(testFileContent) + assert.NoError(t, err) + err = f.Close() + assert.NoError(t, err) + } + user, _, err = httpdtest.GetUserByUsername(user.Username, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 1, user.UsedQuotaFiles) + folder, _, err = httpdtest.GetFolderByName(folderName1, http.StatusOK) + assert.NoError(t, err) + assert.Equal(t, 1, folder.UsedQuotaFiles) + + err = client.Rename(testFileName, path.Join(vdirPath2, testFileName+"_rename")) + assert.Error(t, err) + err = client.Rename(path.Join(vdirPath2, testFileName), testFileName+"_rename") + assert.Error(t, err) + err = client.Rename(testFileName, testFileName+"_rename") + assert.NoError(t, err) + err = client.Rename(path.Join(vdirPath2, testFileName), path.Join(vdirPath2, testFileName+"_rename")) + assert.NoError(t, err) + } + + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName1}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath1) + assert.NoError(t, err) + _, err = httpdtest.RemoveFolder(vfs.BaseVirtualFolder{Name: folderName2}, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(mappedPath2) + assert.NoError(t, err) +} + func TestTruncateQuotaLimits(t *testing.T) { u := getTestUser() u.QuotaSize = 20 diff --git a/common/transfer.go b/common/transfer.go index af20b611..b636972f 100644 --- a/common/transfer.go +++ b/common/transfer.go @@ -306,6 +306,21 @@ func (t *BaseTransfer) getUploadFileSize() (int64, error) { return fileSize, err } +// return 1 if the file is deleted +func (t *BaseTransfer) checkUploadOutsideHomeDir(err error) int { + if Config.TempPath != "" && err != nil { + errRm := t.Fs.Remove(t.effectiveFsPath, false) + t.Connection.Log(logger.LevelWarn, "atomic upload in temp path cannot be renamed, delete temporary file: %#v, deletion error: %v", + t.effectiveFsPath, errRm) + if errRm == nil { + atomic.StoreInt64(&t.BytesReceived, 0) + t.MinWriteOffset = 0 + return 1 + } + } + return 0 +} + // Close it is called when the transfer is completed. // It logs the transfer info, updates the user quota (for uploads) // and executes any defined action. @@ -340,10 +355,12 @@ func (t *BaseTransfer) Close() error { err = t.Fs.Rename(t.effectiveFsPath, t.fsPath) t.Connection.Log(logger.LevelDebug, "atomic upload completed, rename: %#v -> %#v, error: %v", t.effectiveFsPath, t.fsPath, err) + // the file must be removed if it is uploaded to a path outside the home dir and cannot be renamed + numFiles -= t.checkUploadOutsideHomeDir(err) } else { err = t.Fs.Remove(t.effectiveFsPath, false) - t.Connection.Log(logger.LevelWarn, "atomic upload completed with error: \"%v\", delete temporary file: %#v, "+ - "deletion error: %v", t.ErrTransfer, t.effectiveFsPath, err) + t.Connection.Log(logger.LevelWarn, "atomic upload completed with error: \"%v\", delete temporary file: %#v, deletion error: %v", + t.ErrTransfer, t.effectiveFsPath, err) if err == nil { numFiles-- atomic.StoreInt64(&t.BytesReceived, 0) @@ -359,7 +376,7 @@ func (t *BaseTransfer) Close() error { atomic.LoadInt64(&t.BytesSent), t.ErrTransfer) } else { fileSize := atomic.LoadInt64(&t.BytesReceived) + t.MinWriteOffset - if statSize, err := t.getUploadFileSize(); err == nil { + if statSize, errStat := t.getUploadFileSize(); errStat == nil { fileSize = statSize } t.Connection.Log(logger.LevelDebug, "uploaded file size %v", fileSize) diff --git a/common/transfer_test.go b/common/transfer_test.go index 8fcd1b9d..5964b205 100644 --- a/common/transfer_test.go +++ b/common/transfer_test.go @@ -422,3 +422,34 @@ func TestTransferQuota(t *testing.T) { err = transfer.CheckWrite() assert.True(t, conn.IsQuotaExceededError(err)) } + +func TestUploadOutsideHomeRenameError(t *testing.T) { + oldTempPath := Config.TempPath + + conn := NewBaseConnection("", ProtocolSFTP, "", "", dataprovider.User{}) + transfer := BaseTransfer{ + Connection: conn, + transferType: TransferUpload, + BytesReceived: 123, + Fs: vfs.NewOsFs("", filepath.Join(os.TempDir(), "home"), ""), + } + + fileName := filepath.Join(os.TempDir(), "_temp") + err := os.WriteFile(fileName, []byte(`data`), 0644) + assert.NoError(t, err) + + transfer.effectiveFsPath = fileName + res := transfer.checkUploadOutsideHomeDir(os.ErrPermission) + assert.Equal(t, 0, res) + + Config.TempPath = filepath.Clean(os.TempDir()) + res = transfer.checkUploadOutsideHomeDir(nil) + assert.Equal(t, 0, res) + assert.Greater(t, transfer.BytesReceived, int64(0)) + res = transfer.checkUploadOutsideHomeDir(os.ErrPermission) + assert.Equal(t, 1, res) + assert.Equal(t, int64(0), transfer.BytesReceived) + assert.NoFileExists(t, fileName) + + Config.TempPath = oldTempPath +} diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 57e03130..8fb6a553 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -1715,25 +1715,6 @@ func isVirtualDirOverlapped(dir1, dir2 string, fullCheck bool) bool { return false } -func isMappedDirOverlapped(dir1, dir2 string, fullCheck bool) bool { - if dir1 == dir2 { - return true - } - if fullCheck { - if len(dir1) > len(dir2) { - if strings.HasPrefix(dir1, dir2+string(os.PathSeparator)) { - return true - } - } - if len(dir2) > len(dir1) { - if strings.HasPrefix(dir2, dir1+string(os.PathSeparator)) { - return true - } - } - } - return false -} - func validateFolderQuotaLimits(folder vfs.VirtualFolder) error { if folder.QuotaSize < -1 { return util.NewValidationError(fmt.Sprintf("invalid quota_size: %v folder path %#v", folder.QuotaSize, folder.MappedPath)) @@ -1774,13 +1755,10 @@ func validateUserVirtualFolders(user *User) error { return nil } var virtualFolders []vfs.VirtualFolder - mappedPaths := make(map[string]bool) - virtualPaths := make(map[string]bool) + folderNames := make(map[string]bool) + for _, v := range user.VirtualFolders { - cleanedVPath := filepath.ToSlash(path.Clean(v.VirtualPath)) - if !path.IsAbs(cleanedVPath) || cleanedVPath == "/" { - return util.NewValidationError(fmt.Sprintf("invalid virtual folder %#v", v.VirtualPath)) - } + cleanedVPath := util.CleanPath(v.VirtualPath) if err := validateFolderQuotaLimits(v); err != nil { return err } @@ -1788,33 +1766,22 @@ func validateUserVirtualFolders(user *User) error { if err := ValidateFolder(folder); err != nil { return err } - cleanedMPath := folder.MappedPath - if folder.IsLocalOrLocalCrypted() { - if isMappedDirOverlapped(cleanedMPath, user.GetHomeDir(), true) { - return util.NewValidationError(fmt.Sprintf("invalid mapped folder %#v cannot be inside or contain the user home dir %#v", - folder.MappedPath, user.GetHomeDir())) - } - for mPath := range mappedPaths { - if folder.IsLocalOrLocalCrypted() && isMappedDirOverlapped(mPath, cleanedMPath, false) { - return util.NewValidationError(fmt.Sprintf("invalid mapped folder %#v overlaps with mapped folder %#v", - v.MappedPath, mPath)) - } - } - mappedPaths[cleanedMPath] = true + if folderNames[folder.Name] { + return util.NewValidationError(fmt.Sprintf("the folder %#v is duplicated", folder.Name)) } - for vPath := range virtualPaths { - if isVirtualDirOverlapped(vPath, cleanedVPath, false) { - return util.NewValidationError(fmt.Sprintf("invalid virtual folder %#v overlaps with virtual folder %#v", - v.VirtualPath, vPath)) + for _, vFolder := range virtualFolders { + if isVirtualDirOverlapped(vFolder.VirtualPath, cleanedVPath, false) { + return util.NewValidationError(fmt.Sprintf("invalid virtual folder %#v, it overlaps with virtual folder %#v", + v.VirtualPath, vFolder.VirtualPath)) } } - virtualPaths[cleanedVPath] = true virtualFolders = append(virtualFolders, vfs.VirtualFolder{ BaseVirtualFolder: *folder, VirtualPath: cleanedVPath, QuotaSize: v.QuotaSize, QuotaFiles: v.QuotaFiles, }) + folderNames[folder.Name] = true } user.VirtualFolders = virtualFolders return nil diff --git a/dataprovider/user.go b/dataprovider/user.go index d3ae61cf..6a0dacea 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -129,13 +129,7 @@ type User struct { // GetFilesystem returns the base filesystem for this user func (u *User) GetFilesystem(connectionID string) (fs vfs.Fs, err error) { - fs, err = u.getRootFs(connectionID) - if err != nil { - return fs, err - } - u.fsCache = make(map[string]vfs.Fs) - u.fsCache["/"] = fs - return fs, err + return u.GetFilesystemForPath("/", connectionID) } func (u *User) getRootFs(connectionID string) (fs vfs.Fs, err error) { @@ -499,7 +493,8 @@ func (u *User) GetFilesystemForPath(virtualPath, connectionID string) (vfs.Fs, e if u.fsCache == nil { u.fsCache = make(map[string]vfs.Fs) } - if virtualPath != "" && virtualPath != "/" && len(u.VirtualFolders) > 0 { + // allow to override the `/` path with a virtual folder + if len(u.VirtualFolders) > 0 { folder, err := u.GetVirtualFolderForPath(virtualPath) if err == nil { if fs, ok := u.fsCache[folder.VirtualPath]; ok { @@ -524,15 +519,19 @@ func (u *User) GetFilesystemForPath(virtualPath, connectionID string) (vfs.Fs, e if val, ok := u.fsCache["/"]; ok { return val, nil } - - return u.GetFilesystem(connectionID) + fs, err := u.getRootFs(connectionID) + if err != nil { + return fs, err + } + u.fsCache["/"] = fs + return fs, err } // GetVirtualFolderForPath returns the virtual folder containing the specified virtual path. // If the path is not inside a virtual folder an error is returned func (u *User) GetVirtualFolderForPath(virtualPath string) (vfs.VirtualFolder, error) { var folder vfs.VirtualFolder - if virtualPath == "/" || len(u.VirtualFolders) == 0 { + if len(u.VirtualFolders) == 0 { return folder, errNoMatchingVirtualFolder } dirsForPath := util.GetDirsForVirtualPath(virtualPath) @@ -633,7 +632,14 @@ func (u *User) GetVirtualFoldersInPath(virtualPath string) map[string]bool { } func (u *User) hasVirtualDirs() bool { - return len(u.VirtualFolders) > 0 || u.Filters.StartDirectory != "" + if u.Filters.StartDirectory != "" { + return true + } + numFolders := len(u.VirtualFolders) + if numFolders == 1 { + return u.VirtualFolders[0].VirtualPath != "/" + } + return numFolders > 0 } // FilterListDir adds virtual folders and remove hidden items from the given files list diff --git a/docs/virtual-folders.md b/docs/virtual-folders.md index a8b9477d..d9ee3ebc 100644 --- a/docs/virtual-folders.md +++ b/docs/virtual-folders.md @@ -1,11 +1,9 @@ # Virtual Folders -A virtual folder is a mapping between a SFTPGo virtual path and a filesystem path outside the user home directory or a different storage provider. +A virtual folder is a mapping between a SFTPGo virtual path and a filesystem path outside the user home directory or on a different storage provider. For example, you can have a local user with an S3-based virtual folder or vice versa. -The specified local paths must be absolute and the virtual path cannot be "/", it must be a sub directory. - SFTPGo will try to automatically create any missing parent directory for the configured virtual folders at user login. For each virtual folder, the following properties can be configured: @@ -22,7 +20,16 @@ Nested SFTP folders using the same SFTPGo instance (identified using the host ke The same virtual folder can be shared among users, different folder quota limits for each user are supported. Folder quota limits can also be included inside the user quota but in this case the folder is considered "private" and sharing it with other users will break user quota calculation. -The calculation of the quota for a given user is obtained as the sum of the files contained in his home directory and those within each defined virtual folder. +The calculation of the quota for a given user is obtained as the sum of the files contained in his home directory and those within each defined virtual folder included in its quota. + +If you define folders that point to nested paths or to the same path, the quota calculation will be incorrect. Example: + +- `folder1` uses `/srv/data/mapped` or `C:\mapped` as mapped path +- `folder2` uses `/srv/data/mapped/subdir` or `C:\mapped\subdir` as mapped path + +If you upload a file to `folder2` its quota will be updated but the quota of `folder1` will not. We allow this for more flexibility, but if you want to enforce disk quotas using SFTPGo, avoid folders with nested paths. + +It is allowed to mount a virtual folder in the user's root path (`/`). This might be useful if you want to share the same virtual folder between different users. In this case the user's root filesystem is hidden from the virtual folder. Using the REST API you can: @@ -31,4 +38,4 @@ Using the REST API you can: - inspect the relationships among users and folders - delete a virtual folder. SFTPGo removes folders from the data provider, no files deletion will occur -If you remove a folder, from the data provider, any users relationships will be cleared up. If the deleted folder is included inside the user quota you need to do a user quota scan to update its quota. An orphan virtual folder will not be automatically deleted since if you add it again later then a quota scan is needed and it could be quite expensive, anyway you can easily list the orphan folders using the REST API and delete them if they are not needed anymore. +If you remove a folder, from the data provider, any users relationships will be cleared up. If the deleted folder is mounted on the user's root (`/`) path, the user is still valid and its root filesystem will no longer be hidden. If the deleted folder is included inside the user quota you need to do a user quota scan to update its quota. An orphan virtual folder will not be automatically deleted since if you add it again later then a quota scan is needed and it could be quite expensive, anyway you can easily list the orphan folders using the REST API and delete them if they are not needed anymore. diff --git a/go.mod b/go.mod index f17ba2a0..7ad0ab19 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/rs/xid v1.4.0 github.com/rs/zerolog v1.26.2-0.20220312163309-e9344a8c507b github.com/sftpgo/sdk v0.1.1-0.20220327080604-3c0f878c8c37 - github.com/shirou/gopsutil/v3 v3.22.2 + github.com/shirou/gopsutil/v3 v3.22.3 github.com/spf13/afero v1.8.2 github.com/spf13/cobra v1.4.0 github.com/spf13/viper v1.10.1 @@ -120,7 +120,7 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/miekg/dns v1.1.47 // indirect + github.com/miekg/dns v1.1.48 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect @@ -147,7 +147,7 @@ require ( golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7 // indirect + google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de // indirect google.golang.org/grpc v1.45.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/ini.v1 v1.66.4 // indirect diff --git a/go.sum b/go.sum index 1c5afa92..0faef0d3 100644 --- a/go.sum +++ b/go.sum @@ -576,8 +576,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/mhale/smtpd v0.8.0 h1:5JvdsehCg33PQrZBvFyDMMUDQmvbzVpZgKob7eYBJc0= github.com/mhale/smtpd v0.8.0/go.mod h1:MQl+y2hwIEQCXtNhe5+55n0GZOjSmeqORDIXbqUL3x4= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/miekg/dns v1.1.47 h1:J9bWiXbqMbnZPcY8Qi2E3EWIBsIm6MZzzJB9VRg5gL8= -github.com/miekg/dns v1.1.47/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= +github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= @@ -669,8 +669,8 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/sftpgo/sdk v0.1.1-0.20220327080604-3c0f878c8c37 h1:ESruo35Pb9cCgaGslAmw6leGhzeL0pLzD6o+z9gsZeQ= github.com/sftpgo/sdk v0.1.1-0.20220327080604-3c0f878c8c37/go.mod h1:m5J7DH8unhD5RUsREFRiidP8zgBjup0+iQaxQnYHJOM= -github.com/shirou/gopsutil/v3 v3.22.2 h1:wCrArWFkHYIdDxx/FSfF5RB4dpJYW6t7rcp3+zL8uks= -github.com/shirou/gopsutil/v3 v3.22.2/go.mod h1:WapW1AOOPlHyXr+yOyw3uYx36enocrtSoSBy0L5vUHY= +github.com/shirou/gopsutil/v3 v3.22.3 h1:UebRzEomgMpv61e3hgD1tGooqX5trFbdU/ehphbHd00= +github.com/shirou/gopsutil/v3 v3.22.3/go.mod h1:D01hZJ4pVHPpCTZ3m3T2+wDF2YAGfd+H4ifUguaQzHM= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -706,10 +706,8 @@ github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62 h1:b2nJXyPCa9H github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= -github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM= @@ -899,7 +897,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -907,7 +904,6 @@ golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1137,8 +1133,8 @@ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7 h1:HOL66YCI20JvN2hVk6o2YIp9i/3RvzVUz82PqNr7fXw= -google.golang.org/genproto v0.0.0-20220329172620-7be39ac1afc7/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de h1:9Ti5SG2U4cAcluryUo/sFay3TQKoxiFMfaT0pbizU7k= +google.golang.org/genproto v0.0.0-20220401170504-314d38edb7de/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index e5e1906c..c3f69427 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -2119,56 +2119,6 @@ func TestRetentionAPI(t *testing.T) { func TestAddUserInvalidVirtualFolders(t *testing.T) { u := getTestUser() folderName := "fname" - u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ - BaseVirtualFolder: vfs.BaseVirtualFolder{ - MappedPath: filepath.Join(os.TempDir(), "mapped_dir"), - Name: folderName, - }, - VirtualPath: "vdir", // invalid - }) - _, _, err := httpdtest.AddUser(u, http.StatusBadRequest) - assert.NoError(t, err) - u.VirtualFolders = nil - u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ - BaseVirtualFolder: vfs.BaseVirtualFolder{ - MappedPath: filepath.Join(os.TempDir(), "mapped_dir"), - Name: folderName, - }, - VirtualPath: "/", // invalid - }) - _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) - assert.NoError(t, err) - u.VirtualFolders = nil - u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ - BaseVirtualFolder: vfs.BaseVirtualFolder{ - MappedPath: filepath.Join(u.GetHomeDir(), "mapped_dir"), // invalid, inside home dir - Name: folderName, - }, - VirtualPath: "/vdir", - }) - _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) - assert.NoError(t, err) - u.VirtualFolders = nil - u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ - BaseVirtualFolder: vfs.BaseVirtualFolder{ - MappedPath: u.GetHomeDir(), // invalid - Name: folderName, - }, - VirtualPath: "/vdir", - }) - _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) - assert.NoError(t, err) - u.VirtualFolders = nil - u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ - BaseVirtualFolder: vfs.BaseVirtualFolder{ - MappedPath: filepath.Join(u.GetHomeDir(), ".."), // invalid, contains home dir - Name: "tmp", - }, - VirtualPath: "/vdir", - }) - _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) - assert.NoError(t, err) - u.VirtualFolders = nil u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ BaseVirtualFolder: vfs.BaseVirtualFolder{ MappedPath: filepath.Join(os.TempDir(), "mapped_dir"), @@ -2183,7 +2133,7 @@ func TestAddUserInvalidVirtualFolders(t *testing.T) { }, VirtualPath: "/vdir", // invalid, already defined }) - _, _, err = httpdtest.AddUser(u, http.StatusBadRequest) + _, _, err := httpdtest.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) u.VirtualFolders = nil u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ @@ -2195,8 +2145,8 @@ func TestAddUserInvalidVirtualFolders(t *testing.T) { }) u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ BaseVirtualFolder: vfs.BaseVirtualFolder{ - MappedPath: filepath.Join(os.TempDir(), "mapped_dir"), // invalid, already defined - Name: folderName, + MappedPath: filepath.Join(os.TempDir(), "mapped_dir"), + Name: folderName, // invalid, unique constraint (user.id, folder.id) violated }, VirtualPath: "/vdir2", }) diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 677a08a9..8de9d089 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -18,9 +18,9 @@ tags: info: title: SFTPGo description: | - SFTPGo allows to securely share your files over SFTP, HTTP and optionally FTP/S and WebDAV as well. - Several storage backends are supported and they are configurable per user, so you can serve a local directory for a user and an S3 bucket (or part of it) for another one. - SFTPGo also supports virtual folders, a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a GCS bucket (or part of it) on a specified path and an encrypted local filesystem on another one. + SFTPGo allows you to securely share your files over SFTP and optionally over HTTP/S, FTP/S and WebDAV as well. + Several storage backends are supported and they are configurable per-user, so you can serve a local directory for a user and an S3 bucket (or part of it) for another one. + SFTPGo also supports virtual folders, a virtual folder can use any of the supported storage backends. So you can have, for example, an S3 user that exposes a Google Cloud Storage bucket (or part of it) on a specified path and an encrypted local filesystem on another one. Virtual folders can be private or shared among multiple users, for shared virtual folders you can define different quota limits for each user. The SFTPGo WebClient allows end users to change their credentials, browse and manage their files in the browser and setup two-factor authentication which works with Authy, Google Authenticator and other compatible apps. From the WebClient each authorized user can also create HTTP/S links to externally share files and folders securely, by setting limits to the number of downloads/uploads, protecting the share with a password, limiting access by source IP address, setting an automatic expiration date. diff --git a/sftpd/internal_test.go b/sftpd/internal_test.go index bcf30148..e210472c 100644 --- a/sftpd/internal_test.go +++ b/sftpd/internal_test.go @@ -475,8 +475,9 @@ func TestSSHCommandErrors(t *testing.T) { assert.NoError(t, err) }() user := dataprovider.User{} - user.Permissions = make(map[string][]string) - user.Permissions["/"] = []string{dataprovider.PermAny} + user.Permissions = map[string][]string{ + "/": {dataprovider.PermAny}, + } connection := Connection{ BaseConnection: common.NewBaseConnection("", common.ProtocolSSH, "", "", user), channel: &mockSSHChannel, @@ -505,9 +506,14 @@ func TestSSHCommandErrors(t *testing.T) { err = cmd.handle() assert.Error(t, err, "ssh command must fail, we are requesting an invalid path") - cmd.connection.User.HomeDir = filepath.Clean(os.TempDir()) - cmd.connection.User.QuotaFiles = 1 - cmd.connection.User.UsedQuotaFiles = 2 + user = dataprovider.User{} + user.Permissions = map[string][]string{ + "/": {dataprovider.PermAny}, + } + user.HomeDir = filepath.Clean(os.TempDir()) + user.QuotaFiles = 1 + user.UsedQuotaFiles = 2 + cmd.connection.User = user fs, err := cmd.connection.User.GetFilesystem("123") assert.NoError(t, err) err = cmd.handle() diff --git a/util/util.go b/util/util.go index 274e72c5..9732f483 100644 --- a/util/util.go +++ b/util/util.go @@ -283,7 +283,7 @@ func GenerateEd25519Keys(file string) error { // for example if the path is: /1/2/3/4 it returns: // [ "/1/2/3/4", "/1/2/3", "/1/2", "/1", "/" ] func GetDirsForVirtualPath(virtualPath string) []string { - if virtualPath == "." { + if virtualPath == "" || virtualPath == "." { virtualPath = "/" } else { if !path.IsAbs(virtualPath) { diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index e96068bf..78eb25fc 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -12,6 +12,7 @@ import ( "io" "mime" "net/http" + "net/url" "os" "path" "path/filepath" @@ -65,7 +66,7 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo fs := &AzureBlobFs{ connectionID: connectionID, localTempDir: localTempDir, - mountPath: mountPath, + mountPath: getMountPath(mountPath), config: &config, ctxTimeout: 30 * time.Second, ctxLongTimeout: 90 * time.Second, @@ -74,12 +75,10 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo return fs, err } - if err := fs.config.AccountKey.TryDecrypt(); err != nil { - return fs, err - } - if err := fs.config.SASURL.TryDecrypt(); err != nil { + if err := fs.config.tryDecrypt(); err != nil { return fs, err } + fs.setConfigDefaults() version := version.Get() @@ -90,6 +89,9 @@ func NewAzBlobFs(connectionID, localTempDir, mountPath string, config AzBlobFsCo } if fs.config.SASURL.GetPayload() != "" { + if _, err := url.Parse(fs.config.SASURL.GetPayload()); err != nil { + return fs, fmt.Errorf("invalid SAS URL: %w", err) + } parts := azblob.NewBlobURLParts(fs.config.SASURL.GetPayload()) if parts.ContainerName != "" { if fs.config.Container != "" && fs.config.Container != parts.ContainerName { diff --git a/vfs/cryptfs.go b/vfs/cryptfs.go index 472d8df7..5c69c3df 100644 --- a/vfs/cryptfs.go +++ b/vfs/cryptfs.go @@ -44,7 +44,7 @@ func NewCryptFs(connectionID, rootDir, mountPath string, config CryptFsConfig) ( name: cryptFsName, connectionID: connectionID, rootDir: rootDir, - mountPath: mountPath, + mountPath: getMountPath(mountPath), }, masterKey: []byte(config.Passphrase.GetPayload()), } diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go index 33939145..bb090176 100644 --- a/vfs/gcsfs.go +++ b/vfs/gcsfs.go @@ -69,7 +69,7 @@ func NewGCSFs(connectionID, localTempDir, mountPath string, config GCSFsConfig) fs := &GCSFs{ connectionID: connectionID, localTempDir: localTempDir, - mountPath: mountPath, + mountPath: getMountPath(mountPath), config: &config, ctxTimeout: 30 * time.Second, ctxLongTimeout: 300 * time.Second, diff --git a/vfs/osfs.go b/vfs/osfs.go index 43183327..bc7d2323 100644 --- a/vfs/osfs.go +++ b/vfs/osfs.go @@ -46,7 +46,7 @@ func NewOsFs(connectionID, rootDir, mountPath string) Fs { name: osFsName, connectionID: connectionID, rootDir: rootDir, - mountPath: mountPath, + mountPath: getMountPath(mountPath), } } diff --git a/vfs/s3fs.go b/vfs/s3fs.go index a36b6fe9..396b4a14 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -70,7 +70,7 @@ func NewS3Fs(connectionID, localTempDir, mountPath string, s3Config S3FsConfig) fs := &S3Fs{ connectionID: connectionID, localTempDir: localTempDir, - mountPath: mountPath, + mountPath: getMountPath(mountPath), config: &s3Config, ctxTimeout: 30 * time.Second, } diff --git a/vfs/sftpfs.go b/vfs/sftpfs.go index 43be2d0b..86c00976 100644 --- a/vfs/sftpfs.go +++ b/vfs/sftpfs.go @@ -194,7 +194,7 @@ func NewSFTPFs(connectionID, mountPath, localTempDir string, forbiddenSelfUserna config.forbiddenSelfUsernames = forbiddenSelfUsernames sftpFs := &SFTPFs{ connectionID: connectionID, - mountPath: mountPath, + mountPath: getMountPath(mountPath), localTempDir: localTempDir, config: &config, err: make(chan error, 1), diff --git a/vfs/vfs.go b/vfs/vfs.go index a05d0b64..7017d972 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -497,6 +497,16 @@ func (c *AzBlobFsConfig) checkPartSizeAndConcurrency() error { return nil } +func (c *AzBlobFsConfig) tryDecrypt() error { + if err := c.AccountKey.TryDecrypt(); err != nil { + return fmt.Errorf("unable to decrypt account key: %w", err) + } + if err := c.SASURL.TryDecrypt(); err != nil { + return fmt.Errorf("unable to decrypt SAS URL: %w", err) + } + return nil +} + // Validate returns an error if the configuration is not valid func (c *AzBlobFsConfig) Validate() error { if c.AccountKey == nil { @@ -794,6 +804,13 @@ func fsMetadataCheck(fs fsMetadataChecker, storageID, keyPrefix string) error { } } +func getMountPath(mountPath string) string { + if mountPath == "/" { + return "" + } + return mountPath +} + func fsLog(fs Fs, level logger.LogLevel, format string, v ...interface{}) { logger.Log(level, fs.Name(), fs.ConnectionID(), format, v...) } diff --git a/webdavd/server.go b/webdavd/server.go index b594280b..d9a981b1 100644 --- a/webdavd/server.go +++ b/webdavd/server.go @@ -186,7 +186,7 @@ func (s *webDavServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !isCached { err = user.CheckFsRoot(connectionID) } else { - _, err = user.GetFilesystem(connectionID) + _, err = user.GetFilesystemForPath("/", connectionID) } if err != nil { errClose := user.CloseFs()