diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 915e91df..2202d1d8 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -652,8 +652,9 @@ func validateVirtualFolders(user *User) error { v.MappedPath, user.GetHomeDir())} } virtualFolders = append(virtualFolders, vfs.VirtualFolder{ - VirtualPath: cleanedVPath, - MappedPath: cleanedMPath, + VirtualPath: cleanedVPath, + MappedPath: cleanedMPath, + ExcludeFromQuota: v.ExcludeFromQuota, }) for k, virtual := range mappedPaths { if isMappedDirOverlapped(k, cleanedMPath) { diff --git a/dataprovider/user.go b/dataprovider/user.go index 9d773ffb..6d90ea4d 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -191,6 +191,22 @@ func (u *User) GetPermissionsForPath(p string) []string { return permissions } +// IsFileExcludedFromQuota returns true if the file must be excluded from quota usage +func (u *User) IsFileExcludedFromQuota(sftpPath string) bool { + if len(u.VirtualFolders) == 0 || u.FsConfig.Provider != 0 { + return false + } + dirsForPath := utils.GetDirsForSFTPPath(path.Dir(sftpPath)) + for _, val := range dirsForPath { + for _, v := range u.VirtualFolders { + if v.VirtualPath == val { + return v.ExcludeFromQuota + } + } + } + return false +} + // AddVirtualDirs adds virtual folders, if defined, to the given files list func (u *User) AddVirtualDirs(list []os.FileInfo, sftpPath string) []os.FileInfo { if len(u.VirtualFolders) == 0 { diff --git a/docs/account.md b/docs/account.md index 42ec2988..612c7549 100644 --- a/docs/account.md +++ b/docs/account.md @@ -8,7 +8,7 @@ For each account, the following properties can be configured: - `status` 1 means "active", 0 "inactive". An inactive account cannot login. - `expiration_date` expiration date as unix timestamp in milliseconds. An expired account cannot login. 0 means no expiration. - `home_dir` the user cannot upload or download files outside this directory. Must be an absolute path. A local home directory is required for Cloud Storage Backends too: in this case it will store temporary files. -- `virtual_folders` list of mappings between virtual SFTP/SCP paths and local filesystem paths outside the user home directory. The specified paths must be absolute and the virtual path cannot be "/", it must be a sub directory. The parent directory for the specified virtual path must exist. SFTPGo will try to automatically create any missing parent directory for the configured virtual folders at user login +- `virtual_folders` list of mappings between virtual SFTP/SCP paths and local filesystem paths outside the user home directory. The specified paths must be absolute and the virtual path cannot be "/", it must be a sub directory. The parent directory for the specified virtual path must exist. SFTPGo will try to automatically create any missing parent directory for the configured virtual folders at user login. For each mapping you can configure if the folder will be included or not in user quota limit. - `uid`, `gid`. If SFTPGo runs as root system user then the created files and directories will be assigned to this system uid/gid. Ignored on windows or if SFTPGo runs as non root user: in this case files and directories for all SFTP users will be owned by the system user that runs SFTPGo. - `max_sessions` maximum concurrent sessions. 0 means unlimited. - `quota_size` maximum size allowed as bytes. 0 means unlimited. diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 3605b64a..f847a1b2 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -183,7 +183,7 @@ func TestInitialization(t *testing.T) { } err = httpd.ReloadTLSCertificate() if err != nil { - t.Error("realoding TLS Certificate must return nil error if no certificate is configured") + t.Error("reloading TLS Certificate must return nil error if no certificate is configured") } } @@ -628,8 +628,9 @@ func TestUpdateUser(t *testing.T) { MappedPath: filepath.Join(os.TempDir(), "mapped_dir1"), }) user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{ - VirtualPath: "/vdir12/subdir", - MappedPath: filepath.Join(os.TempDir(), "mapped_dir2"), + VirtualPath: "/vdir12/subdir", + MappedPath: filepath.Join(os.TempDir(), "mapped_dir2"), + ExcludeFromQuota: true, }) user, _, err = httpd.UpdateUser(user, http.StatusOK) if err != nil { @@ -1793,7 +1794,7 @@ func TestWebUserAddMock(t *testing.T) { form.Set("expiration_date", "") form.Set("permissions", "*") form.Set("sub_dirs_permissions", " /subdir::list ,download ") - form.Set("virtual_folders", fmt.Sprintf(" /vdir:: %v ", mappedDir)) + form.Set("virtual_folders", fmt.Sprintf(" /vdir:: %v ::1", mappedDir)) form.Set("allowed_extensions", "/dir1::.jpg,.png") form.Set("denied_extensions", "/dir1::.zip") b, contentType, _ := getMultipartFormData(form, "", "") @@ -1899,10 +1900,7 @@ func TestWebUserAddMock(t *testing.T) { rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr.Code) var users []dataprovider.User - err := render.DecodeJSON(rr.Body, &users) - if err != nil { - t.Errorf("Error decoding users: %v", err) - } + render.DecodeJSON(rr.Body, &users) if len(users) != 1 { t.Errorf("1 user is expected, actual: %v", len(users)) } @@ -1928,7 +1926,7 @@ func TestWebUserAddMock(t *testing.T) { } vfolderFoumd := false for _, v := range newUser.VirtualFolders { - if v.VirtualPath == "/vdir" && v.MappedPath == mappedDir { + if v.VirtualPath == "/vdir" && v.MappedPath == mappedDir && v.ExcludeFromQuota == true { vfolderFoumd = true } } diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index 227cc3b4..d8a02e61 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.0.1 info: title: SFTPGo description: 'SFTPGo REST API' - version: 1.8.4 + version: 1.8.5 servers: - url: /api/v1 @@ -1077,6 +1077,10 @@ components: type: string mapped_path: type: string + exclude_from_quota: + type: boolean + nullable: true + description: This folder will be excluded from user quota required: - virtual_path - mapped_path diff --git a/httpd/web.go b/httpd/web.go index d60d46f4..aedea77b 100644 --- a/httpd/web.go +++ b/httpd/web.go @@ -196,10 +196,17 @@ func getVirtualFoldersFromPostFields(r *http.Request) []vfs.VirtualFolder { if strings.Contains(cleaned, "::") { mapping := strings.Split(cleaned, "::") if len(mapping) > 1 { - virtualFolders = append(virtualFolders, vfs.VirtualFolder{ + vfolder := vfs.VirtualFolder{ VirtualPath: strings.TrimSpace(mapping[0]), MappedPath: strings.TrimSpace(mapping[1]), - }) + } + if len(mapping) > 2 { + excludeFromQuota, err := strconv.Atoi(strings.TrimSpace(mapping[2])) + if err == nil { + vfolder.ExcludeFromQuota = (excludeFromQuota > 0) + } + } + virtualFolders = append(virtualFolders, vfolder) } } } diff --git a/scripts/README.md b/scripts/README.md index e4b86dee..bc1e9f48 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -140,7 +140,7 @@ Output: Command: ``` -python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1::list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --denied-login-methods "" --fs local --virtual-folders "/vdir1::/tmp/mapped1" "/vdir2::/tmp/mapped2" --allowed-extensions "" --denied-extensions "" +python sftpgo_api_cli.py update-user 9576 test_username --password "test_pwd" --home-dir="/tmp/test_home_dir" --uid 0 --gid 33 --max-sessions 3 --quota-size 0 --quota-files 4 --permissions "*" --subdirs-permissions "/dir1::list,download,create_symlinks" --upload-bandwidth 90 --download-bandwidth 80 --status 1 --expiration-date "" --allowed-ip "" --denied-ip "192.168.1.0/24" --denied-login-methods "" --fs local --virtual-folders "/vdir1::/tmp/mapped1" "/vdir2::/tmp/mapped2::1" --allowed-extensions "" --denied-extensions "" ``` Output: @@ -203,10 +203,12 @@ Output: "username": "test_username", "virtual_folders": [ { + "exclude_from_quota": false, "mapped_path": "/tmp/mapped1", "virtual_path": "/vdir1" }, { + "exclude_from_quota": true, "mapped_path": "/tmp/mapped2", "virtual_path": "/vdir2" } @@ -265,10 +267,12 @@ Output: "username": "test_username", "virtual_folders": [ { + "exclude_from_quota": false, "mapped_path": "/tmp/mapped1", "virtual_path": "/vdir1" }, { + "exclude_from_quota": true, "mapped_path": "/tmp/mapped2", "virtual_path": "/vdir2" } diff --git a/scripts/sftpgo_api_cli.py b/scripts/sftpgo_api_cli.py index f2273038..dc5522da 100755 --- a/scripts/sftpgo_api_cli.py +++ b/scripts/sftpgo_api_cli.py @@ -110,12 +110,19 @@ class SFTPGoApiRequests: if '::' in f: vpath = '' mapped_path = '' + exclude_from_quota = False values = f.split('::') if len(values) > 1: vpath = values[0] mapped_path = values[1] + if len(values) > 2: + try: + exclude_from_quota = int(values[2]) > 0 + except: + pass if vpath and mapped_path: - result.append({"virtual_path":vpath, "mapped_path":mapped_path}) + result.append({"virtual_path":vpath, "mapped_path":mapped_path, + "exclude_from_quota":exclude_from_quota}) return result def buildPermissions(self, root_perms, subdirs_perms): @@ -508,7 +515,8 @@ def addCommonUserArguments(parser): parser.add_argument('--subdirs-permissions', type=str, nargs='*', default=[], help='Permissions for subdirs. ' +'For example: "/somedir::list,download" "/otherdir/subdir::*" Default: %(default)s') parser.add_argument('--virtual-folders', type=str, nargs='*', default=[], help='Virtual folder mapping. For example: ' - +'"/vpath::/home/adir" "/vpath::C:\adir", ignored for non local filesystems. Default: %(default)s') + +'"/vpath::/home/adir" "/vpath::C:\adir::1". If the optional third argument is > 0 the virtual ' + +'folder will be excluded from user quota. Ignored for non local filesystems. Default: %(default)s') parser.add_argument('-U', '--upload-bandwidth', type=int, default=0, help='Maximum upload bandwidth as KB/s, 0 means unlimited. Default: %(default)s') parser.add_argument('-D', '--download-bandwidth', type=int, default=0, diff --git a/sftpd/handler.go b/sftpd/handler.go index 74046a66..9865f781 100644 --- a/sftpd/handler.go +++ b/sftpd/handler.go @@ -75,25 +75,26 @@ func (c Connection) Fileread(request *sftp.Request) (io.ReaderAt, error) { c.Log(logger.LevelDebug, logSender, "fileread requested for path: %#v", p) transfer := Transfer{ - file: file, - readerAt: r, - writerAt: nil, - cancelFn: cancelFn, - path: p, - start: time.Now(), - bytesSent: 0, - bytesReceived: 0, - user: c.User, - connectionID: c.ID, - transferType: transferDownload, - lastActivity: time.Now(), - isNewFile: false, - protocol: c.protocol, - transferError: nil, - isFinished: false, - minWriteOffset: 0, - expectedSize: fi.Size(), - lock: new(sync.Mutex), + file: file, + readerAt: r, + writerAt: nil, + cancelFn: cancelFn, + path: p, + start: time.Now(), + bytesSent: 0, + bytesReceived: 0, + user: c.User, + connectionID: c.ID, + transferType: transferDownload, + lastActivity: time.Now(), + isNewFile: false, + protocol: c.protocol, + transferError: nil, + isFinished: false, + minWriteOffset: 0, + expectedSize: fi.Size(), + isExcludedFromQuota: c.User.IsFileExcludedFromQuota(request.Filepath), + lock: new(sync.Mutex), } addTransfer(&transfer) return &transfer, nil @@ -123,7 +124,7 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) { if !c.User.HasPerm(dataprovider.PermUpload, path.Dir(request.Filepath)) { return nil, sftp.ErrSSHFxPermissionDenied } - return c.handleSFTPUploadToNewFile(p, filePath) + return c.handleSFTPUploadToNewFile(p, filePath, c.User.IsFileExcludedFromQuota(request.Filepath)) } if statErr != nil { @@ -141,7 +142,8 @@ func (c Connection) Filewrite(request *sftp.Request) (io.WriterAt, error) { return nil, sftp.ErrSSHFxPermissionDenied } - return c.handleSFTPUploadToExistingFile(request.Pflags(), p, filePath, stat.Size()) + return c.handleSFTPUploadToExistingFile(request.Pflags(), p, filePath, stat.Size(), + c.User.IsFileExcludedFromQuota(request.Filepath)) } // Filecmd hander for basic SFTP system calls related to files, but not anything to do with reading @@ -437,14 +439,16 @@ func (c Connection) handleSFTPRemove(filePath string, request *sftp.Request) err logger.CommandLog(removeLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "") if fi.Mode()&os.ModeSymlink != os.ModeSymlink { - dataprovider.UpdateUserQuota(dataProvider, c.User, -1, -size, false) //nolint:errcheck + if !c.User.IsFileExcludedFromQuota(request.Filepath) { + dataprovider.UpdateUserQuota(dataProvider, c.User, -1, -size, false) //nolint:errcheck + } } go executeAction(newActionNotification(c.User, operationDelete, filePath, "", "", fi.Size(), nil)) //nolint:errcheck return sftp.ErrSSHFxOk } -func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string) (io.WriterAt, error) { +func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string, isExcludedFromQuota bool) (io.WriterAt, error) { if !c.hasSpace(true) { c.Log(logger.LevelInfo, logSender, "denying file write due to space limit") return nil, sftp.ErrSSHFxFailure @@ -459,31 +463,32 @@ func (c Connection) handleSFTPUploadToNewFile(requestPath, filePath string) (io. vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID()) transfer := Transfer{ - file: file, - writerAt: w, - readerAt: nil, - cancelFn: cancelFn, - path: requestPath, - start: time.Now(), - bytesSent: 0, - bytesReceived: 0, - user: c.User, - connectionID: c.ID, - transferType: transferUpload, - lastActivity: time.Now(), - isNewFile: true, - protocol: c.protocol, - transferError: nil, - isFinished: false, - minWriteOffset: 0, - lock: new(sync.Mutex), + file: file, + writerAt: w, + readerAt: nil, + cancelFn: cancelFn, + path: requestPath, + start: time.Now(), + bytesSent: 0, + bytesReceived: 0, + user: c.User, + connectionID: c.ID, + transferType: transferUpload, + lastActivity: time.Now(), + isNewFile: true, + protocol: c.protocol, + transferError: nil, + isFinished: false, + minWriteOffset: 0, + isExcludedFromQuota: isExcludedFromQuota, + lock: new(sync.Mutex), } addTransfer(&transfer) return &transfer, nil } func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, requestPath, filePath string, - fileSize int64) (io.WriterAt, error) { + fileSize int64, isExcludedFromQuota bool) (io.WriterAt, error) { var err error if !c.hasSpace(false) { c.Log(logger.LevelInfo, logSender, "denying file write due to space limit") @@ -520,7 +525,9 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re minWriteOffset = fileSize } else { if vfs.IsLocalOsFs(c.fs) { - dataprovider.UpdateUserQuota(dataProvider, c.User, 0, -fileSize, false) //nolint:errcheck + if !isExcludedFromQuota { + dataprovider.UpdateUserQuota(dataProvider, c.User, 0, -fileSize, false) //nolint:errcheck + } } else { initialSize = fileSize } @@ -529,25 +536,26 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re vfs.SetPathPermissions(c.fs, filePath, c.User.GetUID(), c.User.GetGID()) transfer := Transfer{ - file: file, - writerAt: w, - readerAt: nil, - cancelFn: cancelFn, - path: requestPath, - start: time.Now(), - bytesSent: 0, - bytesReceived: 0, - user: c.User, - connectionID: c.ID, - transferType: transferUpload, - lastActivity: time.Now(), - isNewFile: false, - protocol: c.protocol, - transferError: nil, - isFinished: false, - minWriteOffset: minWriteOffset, - initialSize: initialSize, - lock: new(sync.Mutex), + file: file, + writerAt: w, + readerAt: nil, + cancelFn: cancelFn, + path: requestPath, + start: time.Now(), + bytesSent: 0, + bytesReceived: 0, + user: c.User, + connectionID: c.ID, + transferType: transferUpload, + lastActivity: time.Now(), + isNewFile: false, + protocol: c.protocol, + transferError: nil, + isFinished: false, + minWriteOffset: minWriteOffset, + initialSize: initialSize, + isExcludedFromQuota: isExcludedFromQuota, + lock: new(sync.Mutex), } addTransfer(&transfer) return &transfer, nil diff --git a/sftpd/internal_test.go b/sftpd/internal_test.go index 4b7b9766..9572a54e 100644 --- a/sftpd/internal_test.go +++ b/sftpd/internal_test.go @@ -423,7 +423,7 @@ func TestMockFsErrors(t *testing.T) { flags.Write = true flags.Trunc = false flags.Append = true - _, err = c.handleSFTPUploadToExistingFile(flags, testfile, testfile, 0) + _, err = c.handleSFTPUploadToExistingFile(flags, testfile, testfile, 0, false) if err != sftp.ErrSSHFxOpUnsupported { t.Errorf("unexpected error: %v", err) } @@ -439,12 +439,12 @@ func TestUploadFiles(t *testing.T) { var flags sftp.FileOpenFlags flags.Write = true flags.Trunc = true - _, err := c.handleSFTPUploadToExistingFile(flags, "missing_path", "other_missing_path", 0) + _, err := c.handleSFTPUploadToExistingFile(flags, "missing_path", "other_missing_path", 0, false) if err == nil { t.Errorf("upload to existing file must fail if one or both paths are invalid") } uploadMode = uploadModeStandard - _, err = c.handleSFTPUploadToExistingFile(flags, "missing_path", "other_missing_path", 0) + _, err = c.handleSFTPUploadToExistingFile(flags, "missing_path", "other_missing_path", 0, false) if err == nil { t.Errorf("upload to existing file must fail if one or both paths are invalid") } @@ -452,14 +452,14 @@ func TestUploadFiles(t *testing.T) { if runtime.GOOS == "windows" { missingFile = "missing\\relative\\file.txt" } - _, err = c.handleSFTPUploadToNewFile(".", missingFile) + _, err = c.handleSFTPUploadToNewFile(".", missingFile, false) if err == nil { t.Errorf("upload new file in missing path must fail") } c.fs = newMockOsFs(nil, nil, false, "123", os.TempDir()) f, _ := ioutil.TempFile("", "temp") f.Close() - _, err = c.handleSFTPUploadToExistingFile(flags, f.Name(), f.Name(), 123) + _, err = c.handleSFTPUploadToExistingFile(flags, f.Name(), f.Name(), 123, false) if err != nil { t.Errorf("unexpected error: %v", err) } @@ -1437,7 +1437,7 @@ func TestSCPErrorsMockFs(t *testing.T) { if err != errFake { t.Errorf("unexpected error: %v", err) } - err = scpCommand.handleUploadFile(testfile, testfile, 0, false, 4) + err = scpCommand.handleUploadFile(testfile, testfile, 0, false, 4, false) if err != nil { t.Errorf("unexpected error: %v", err) } diff --git a/sftpd/scp.go b/sftpd/scp.go index c9543001..45a594b0 100644 --- a/sftpd/scp.go +++ b/sftpd/scp.go @@ -187,7 +187,8 @@ func (c *scpCommand) getUploadFileData(sizeToRead int64, transfer *Transfer) err return c.sendConfirmationMessage() } -func (c *scpCommand) handleUploadFile(requestPath, filePath string, sizeToRead int64, isNewFile bool, fileSize int64) error { +func (c *scpCommand) handleUploadFile(requestPath, filePath string, sizeToRead int64, isNewFile bool, fileSize int64, + isExcludedFromQuota bool) error { if !c.connection.hasSpace(true) { err := fmt.Errorf("denying file write due to space limit") c.connection.Log(logger.LevelWarn, logSenderSCP, "error uploading file: %#v, err: %v", filePath, err) @@ -198,7 +199,9 @@ func (c *scpCommand) handleUploadFile(requestPath, filePath string, sizeToRead i initialSize := int64(0) if !isNewFile { if vfs.IsLocalOsFs(c.connection.fs) { - dataprovider.UpdateUserQuota(dataProvider, c.connection.User, 0, -fileSize, false) //nolint:errcheck + if !isExcludedFromQuota { + dataprovider.UpdateUserQuota(dataProvider, c.connection.User, 0, -fileSize, false) //nolint:errcheck + } } else { initialSize = fileSize } @@ -213,25 +216,26 @@ func (c *scpCommand) handleUploadFile(requestPath, filePath string, sizeToRead i vfs.SetPathPermissions(c.connection.fs, filePath, c.connection.User.GetUID(), c.connection.User.GetGID()) transfer := Transfer{ - file: file, - readerAt: nil, - writerAt: w, - cancelFn: cancelFn, - path: requestPath, - start: time.Now(), - bytesSent: 0, - bytesReceived: 0, - user: c.connection.User, - connectionID: c.connection.ID, - transferType: transferUpload, - lastActivity: time.Now(), - isNewFile: isNewFile, - protocol: c.connection.protocol, - transferError: nil, - isFinished: false, - minWriteOffset: 0, - initialSize: initialSize, - lock: new(sync.Mutex), + file: file, + readerAt: nil, + writerAt: w, + cancelFn: cancelFn, + path: requestPath, + start: time.Now(), + bytesSent: 0, + bytesReceived: 0, + user: c.connection.User, + connectionID: c.connection.ID, + transferType: transferUpload, + lastActivity: time.Now(), + isNewFile: isNewFile, + protocol: c.connection.protocol, + transferError: nil, + isFinished: false, + minWriteOffset: 0, + initialSize: initialSize, + isExcludedFromQuota: isExcludedFromQuota, + lock: new(sync.Mutex), } addTransfer(&transfer) @@ -265,7 +269,7 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error c.sendErrorMessage(errPermission) return errPermission } - return c.handleUploadFile(p, filePath, sizeToRead, true, 0) + return c.handleUploadFile(p, filePath, sizeToRead, true, 0, c.connection.User.IsFileExcludedFromQuota(uploadFilePath)) } if statErr != nil { @@ -297,7 +301,7 @@ func (c *scpCommand) handleUpload(uploadFilePath string, sizeToRead int64) error } } - return c.handleUploadFile(p, filePath, sizeToRead, false, stat.Size()) + return c.handleUploadFile(p, filePath, sizeToRead, false, stat.Size(), c.connection.User.IsFileExcludedFromQuota(uploadFilePath)) } func (c *scpCommand) sendDownloadProtocolMessages(dirPath string, stat os.FileInfo) error { diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index 0c57e63f..7ce537bf 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -2449,15 +2449,13 @@ func TestVirtualFoldersQuota(t *testing.T) { MappedPath: mappedPath1, }) u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ - VirtualPath: vdirPath2, - MappedPath: mappedPath2, + VirtualPath: vdirPath2, + MappedPath: mappedPath2, + ExcludeFromQuota: true, }) os.MkdirAll(mappedPath1, 0777) os.MkdirAll(mappedPath2, 0777) - user, _, err := httpd.AddUser(u, http.StatusOK) - if err != nil { - t.Errorf("unable to add user: %v", err) - } + user, _, _ := httpd.AddUser(u, http.StatusOK) client, err := getSftpClient(user, usePubKey) if err != nil { t.Errorf("unable to create sftp client: %v", err) @@ -2482,8 +2480,26 @@ func TestVirtualFoldersQuota(t *testing.T) { if err != nil { t.Errorf("file upload error: %v", err) } - expectedQuotaFiles := 3 - expectedQuotaSize := testFileSize * 3 + err = sftpUploadFile(testFilePath, path.Join(vdirPath2, testFileName), testFileSize, client) + if err != nil { + t.Errorf("file upload error: %v", err) + } + expectedQuotaFiles := 2 + expectedQuotaSize := testFileSize * 2 + user, _, err = httpd.GetUserByID(user.ID, http.StatusOK) + if err != nil { + t.Errorf("error getting user: %v", err) + } + if expectedQuotaFiles != user.UsedQuotaFiles { + t.Errorf("quota files does not match, expected: %v, actual: %v", expectedQuotaFiles, user.UsedQuotaFiles) + } + if expectedQuotaSize != user.UsedQuotaSize { + t.Errorf("quota size does not match, expected: %v, actual: %v", expectedQuotaSize, user.UsedQuotaSize) + } + err = client.Remove(path.Join(vdirPath2, testFileName)) + if err != nil { + t.Errorf("unexpected error removing uploaded file: %v", err) + } user, _, err = httpd.GetUserByID(user.ID, http.StatusOK) if err != nil { t.Errorf("error getting user: %v", err) @@ -3878,6 +3894,52 @@ func TestResolveVirtualPaths(t *testing.T) { } } +func TestVirtualFoldersExcludeQuota(t *testing.T) { + user := getTestUser(true) + mappedPath := filepath.Join(os.TempDir(), "vdir") + vdirPath := "/vdir/sub" + vSubDirPath := path.Join(vdirPath, "subdir", "subdir") + vSubDir1Path := path.Join(vSubDirPath, "subdir", "subdir") + user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{ + VirtualPath: vdirPath, + MappedPath: mappedPath, + ExcludeFromQuota: false, + }) + user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{ + VirtualPath: vSubDir1Path, + MappedPath: mappedPath, + ExcludeFromQuota: false, + }) + user.VirtualFolders = append(user.VirtualFolders, vfs.VirtualFolder{ + VirtualPath: vSubDirPath, + MappedPath: mappedPath, + ExcludeFromQuota: true, + }) + + if user.IsFileExcludedFromQuota("/file") { + t.Errorf("unexpected file excluded from quota") + } + if user.IsFileExcludedFromQuota(path.Join(vdirPath, "file")) { + t.Errorf("unexpected file excluded from quota") + } + if !user.IsFileExcludedFromQuota(path.Join(vSubDirPath, "file")) { + t.Errorf("unexpected file included in quota") + } + if !user.IsFileExcludedFromQuota(path.Join(vSubDir1Path, "..", "file")) { + t.Errorf("unexpected file included in quota") + } + if user.IsFileExcludedFromQuota(path.Join(vSubDir1Path, "file")) { + t.Errorf("unexpected file excluded from quota") + } + if user.IsFileExcludedFromQuota(path.Join(vSubDirPath, "..", "file")) { + t.Errorf("unexpected file excluded from quota") + } + // we check the parent dir for a file + if user.IsFileExcludedFromQuota(vSubDirPath) { + t.Errorf("unexpected file excluded from quota") + } +} + func TestUserPerms(t *testing.T) { user := getTestUser(true) user.Permissions = make(map[string][]string) @@ -4719,6 +4781,86 @@ func TestSCPVirtualFolders(t *testing.T) { os.RemoveAll(mappedPath) } +func TestSCPVirtualFoldersQuota(t *testing.T) { + if len(scpPath) == 0 { + t.Skip("scp command not found, unable to execute this test") + } + usePubKey := true + u := getTestUser(usePubKey) + u.QuotaFiles = 100 + mappedPath1 := filepath.Join(os.TempDir(), "vdir1") + vdirPath1 := "/vdir1" + mappedPath2 := filepath.Join(os.TempDir(), "vdir2") + vdirPath2 := "/vdir2" + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + VirtualPath: vdirPath1, + MappedPath: mappedPath1, + }) + u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{ + VirtualPath: vdirPath2, + MappedPath: mappedPath2, + ExcludeFromQuota: true, + }) + os.MkdirAll(mappedPath1, 0777) + os.MkdirAll(mappedPath2, 0777) + user, _, err := httpd.AddUser(u, http.StatusOK) + if err != nil { + t.Errorf("unable to add user: %v", err) + } + testFileName := "test_file.dat" + testBaseDirName := "test_dir" + testBaseDirPath := filepath.Join(homeBasePath, testBaseDirName) + testBaseDirDownName := "test_dir_down" + testBaseDirDownPath := filepath.Join(homeBasePath, testBaseDirDownName) + testFilePath := filepath.Join(homeBasePath, testBaseDirName, testFileName) + testFilePath1 := filepath.Join(homeBasePath, testBaseDirName, testBaseDirName, testFileName) + testFileSize := int64(131074) + createTestFile(testFilePath, testFileSize) + createTestFile(testFilePath1, testFileSize) + remoteDownPath1 := fmt.Sprintf("%v@127.0.0.1:%v", user.Username, path.Join("/", vdirPath1)) + remoteUpPath1 := fmt.Sprintf("%v@127.0.0.1:%v", user.Username, vdirPath1) + remoteDownPath2 := fmt.Sprintf("%v@127.0.0.1:%v", user.Username, path.Join("/", vdirPath2)) + remoteUpPath2 := fmt.Sprintf("%v@127.0.0.1:%v", user.Username, vdirPath2) + err = scpUpload(testBaseDirPath, remoteUpPath1, true, false) + if err != nil { + t.Errorf("error uploading dir via scp: %v", err) + } + err = scpDownload(testBaseDirDownPath, remoteDownPath1, true, true) + if err != nil { + t.Errorf("error downloading dir via scp: %v", err) + } + err = scpUpload(testBaseDirPath, remoteUpPath2, true, false) + if err != nil { + t.Errorf("error uploading dir via scp: %v", err) + } + err = scpDownload(testBaseDirDownPath, remoteDownPath2, true, true) + if err != nil { + t.Errorf("error downloading dir via scp: %v", err) + } + expectedQuotaFiles := 2 + expectedQuotaSize := testFileSize * 2 + user, _, err = httpd.GetUserByID(user.ID, http.StatusOK) + if err != nil { + t.Errorf("error getting user: %v", err) + } + if expectedQuotaFiles != user.UsedQuotaFiles { + t.Errorf("quota files does not match, expected: %v, actual: %v", expectedQuotaFiles, user.UsedQuotaFiles) + } + if expectedQuotaSize != user.UsedQuotaSize { + t.Errorf("quota size does not match, expected: %v, actual: %v", expectedQuotaSize, user.UsedQuotaSize) + } + + _, err = httpd.RemoveUser(user, http.StatusOK) + if err != nil { + t.Errorf("unable to remove user: %v", err) + } + os.RemoveAll(testBaseDirPath) + os.RemoveAll(testBaseDirDownPath) + os.RemoveAll(user.GetHomeDir()) + os.RemoveAll(mappedPath1) + os.RemoveAll(mappedPath1) +} + func TestSCPPermsSubDirs(t *testing.T) { if len(scpPath) == 0 { t.Skip("scp command not found, unable to execute this test") diff --git a/sftpd/transfer.go b/sftpd/transfer.go index a177cde5..dc282901 100644 --- a/sftpd/transfer.go +++ b/sftpd/transfer.go @@ -26,26 +26,27 @@ var ( // Transfer contains the transfer details for an upload or a download. // It implements the io Reader and Writer interface to handle files downloads and uploads type Transfer struct { - file *os.File - writerAt *pipeat.PipeWriterAt - readerAt *pipeat.PipeReaderAt - cancelFn func() - path string - start time.Time - bytesSent int64 - bytesReceived int64 - user dataprovider.User - connectionID string - transferType int - lastActivity time.Time - protocol string - transferError error - minWriteOffset int64 - expectedSize int64 - initialSize int64 - lock *sync.Mutex - isNewFile bool - isFinished bool + file *os.File + writerAt *pipeat.PipeWriterAt + readerAt *pipeat.PipeReaderAt + cancelFn func() + path string + start time.Time + bytesSent int64 + bytesReceived int64 + user dataprovider.User + connectionID string + transferType int + lastActivity time.Time + protocol string + transferError error + minWriteOffset int64 + expectedSize int64 + initialSize int64 + lock *sync.Mutex + isNewFile bool + isFinished bool + isExcludedFromQuota bool } // TransferError is called if there is an unexpected error. @@ -184,6 +185,9 @@ func (t *Transfer) updateQuota(numFiles int) bool { if t.file == nil && t.transferError != nil { return false } + if t.isExcludedFromQuota { + return false + } if t.transferType == transferUpload && (numFiles != 0 || t.bytesReceived > 0) { dataprovider.UpdateUserQuota(dataProvider, t.user, numFiles, t.bytesReceived-t.initialSize, false) //nolint:errcheck return true diff --git a/templates/user.html b/templates/user.html index 519e0e89..bcab79d6 100644 --- a/templates/user.html +++ b/templates/user.html @@ -124,10 +124,10 @@
- One mapping per line as vpath::path, for example /vdir::/home/adir or /vdir::C:\adir, ignored for non local filesystems + One mapping per line as vpath::path::[exclude_from_quota], for example /vdir::/home/adir or /vdir::C:\adir::1, ignored for non local filesystems
diff --git a/vfs/osfs.go b/vfs/osfs.go index 4deaf306..aa032698 100644 --- a/vfs/osfs.go +++ b/vfs/osfs.go @@ -174,6 +174,9 @@ func (fs OsFs) CheckRootPath(username string, uid int, gid int) bool { func (fs OsFs) ScanRootDirContents() (int, int64, error) { numFiles, size, err := fs.getDirSize(fs.rootDir) for _, v := range fs.virtualFolders { + if v.ExcludeFromQuota { + continue + } num, s, err := fs.getDirSize(v.MappedPath) if err != nil { if fs.IsNotExist(err) { diff --git a/vfs/vfs.go b/vfs/vfs.go index a0f70c58..5e8b948b 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -52,6 +52,8 @@ type Fs interface { type VirtualFolder struct { VirtualPath string `json:"virtual_path"` MappedPath string `json:"mapped_path"` + // This folder will be excluded from user quota + ExcludeFromQuota bool `json:"exclude_from_quota"` } // IsDirectory checks if a path exists and is a directory