add sftpfs storage backend

Fixes #224

vfs/azblobfs.go

@@ -37,7 +37,7 @@ var maxTryTimeout = time.Hour * 24 * 365
 type AzureBlobFs struct {
 	connectionID   string
 	localTempDir   string
-	config         AzBlobFsConfig
+	config         *AzBlobFsConfig
 	svc            *azblob.ServiceURL
 	containerURL   azblob.ContainerURL
 	ctxTimeout     time.Duration
@@ -53,11 +53,11 @@ func NewAzBlobFs(connectionID, localTempDir string, config AzBlobFsConfig) (Fs,
 	fs := &AzureBlobFs{
 		connectionID:   connectionID,
 		localTempDir:   localTempDir,
-		config:         config,
+		config:         &config,
 		ctxTimeout:     30 * time.Second,
 		ctxLongTimeout: 300 * time.Second,
 	}
-	if err := ValidateAzBlobFsConfig(&fs.config); err != nil {
+	if err := fs.config.Validate(); err != nil {
 		return fs, err
 	}
 	if fs.config.AccountKey.IsEncrypted() {
@@ -695,6 +695,11 @@ func (fs *AzureBlobFs) GetMimeType(name string) (string, error) {
 	return response.ContentType(), nil
 }
 
+// Close closes the fs
+func (*AzureBlobFs) Close() error {
+	return nil
+}
+
 func (fs *AzureBlobFs) isEqual(key string, virtualName string) bool {
 	if key == virtualName {
 		return true

vfs/cryptfs.go

@@ -32,7 +32,7 @@ type CryptFs struct {
 
 // NewCryptFs returns a CryptFs object
 func NewCryptFs(connectionID, rootDir string, config CryptFsConfig) (Fs, error) {
-	if err := ValidateCryptFsConfig(&config); err != nil {
+	if err := config.Validate(); err != nil {
 		return nil, err
 	}
 	if config.Passphrase.IsEncrypted() {

vfs/gcsfs.go

@@ -37,7 +37,7 @@ var (
 type GCSFs struct {
 	connectionID   string
 	localTempDir   string
-	config         GCSFsConfig
+	config         *GCSFsConfig
 	svc            *storage.Client
 	ctxTimeout     time.Duration
 	ctxLongTimeout time.Duration
@@ -53,11 +53,11 @@ func NewGCSFs(connectionID, localTempDir string, config GCSFsConfig) (Fs, error)
 	fs := &GCSFs{
 		connectionID:   connectionID,
 		localTempDir:   localTempDir,
-		config:         config,
+		config:         &config,
 		ctxTimeout:     30 * time.Second,
 		ctxLongTimeout: 300 * time.Second,
 	}
-	if err = ValidateGCSFsConfig(&fs.config, fs.config.CredentialFile); err != nil {
+	if err = fs.config.Validate(fs.config.CredentialFile); err != nil {
 		return fs, err
 	}
 	ctx := context.Background()
@@ -749,3 +749,8 @@ func (fs *GCSFs) GetMimeType(name string) (string, error) {
 	}
 	return attrs.ContentType, nil
 }
+
+// Close closes the fs
+func (fs *GCSFs) Close() error {
+	return nil
+}

vfs/osfs.go

@@ -291,14 +291,15 @@ func (fs *OsFs) ResolvePath(sftpPath string) (string, error) {
 		// path chain until we hit a directory that _does_ exist and can be validated.
 		_, err = fs.findFirstExistingDir(r, basePath)
 		if err != nil {
-			fsLog(fs, logger.LevelWarn, "error resolving non-existent path: %#v", err)
+			fsLog(fs, logger.LevelWarn, "error resolving non-existent path %#v", err)
 		}
 		return r, err
 	}
 
 	err = fs.isSubDir(p, basePath)
 	if err != nil {
-		fsLog(fs, logger.LevelWarn, "Invalid path resolution, dir: %#v outside user home: %#v err: %v", p, fs.rootDir, err)
+		fsLog(fs, logger.LevelWarn, "Invalid path resolution, dir %#v original path %#v resolved %#v err: %v",
+			p, sftpPath, r, err)
 	}
 	return r, err
 }
@@ -427,13 +428,11 @@ func (fs *OsFs) isSubDir(sub, rootPath string) error {
 		return nil
 	}
 	if len(sub) < len(parent) {
-		err = fmt.Errorf("path %#v is not inside: %#v", sub, parent)
-		fsLog(fs, logger.LevelWarn, "error: %v ", err)
+		err = fmt.Errorf("path %#v is not inside %#v", sub, parent)
 		return err
 	}
 	if !strings.HasPrefix(sub, parent+string(os.PathSeparator)) {
-		err = fmt.Errorf("path %#v is not inside: %#v", sub, parent)
-		fsLog(fs, logger.LevelWarn, "error: %v ", err)
+		err = fmt.Errorf("path %#v is not inside %#v", sub, parent)
 		return err
 	}
 	return nil
@@ -473,3 +472,8 @@ func (fs *OsFs) GetMimeType(name string) (string, error) {
 	_, err = f.Seek(0, io.SeekStart)
 	return ctype, err
 }
+
+// Close closes the fs
+func (*OsFs) Close() error {
+	return nil
+}

vfs/s3fs.go

@@ -31,7 +31,7 @@ import (
 type S3Fs struct {
 	connectionID   string
 	localTempDir   string
-	config         S3FsConfig
+	config         *S3FsConfig
 	svc            *s3.S3
 	ctxTimeout     time.Duration
 	ctxLongTimeout time.Duration
@@ -47,11 +47,11 @@ func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) {
 	fs := &S3Fs{
 		connectionID:   connectionID,
 		localTempDir:   localTempDir,
-		config:         config,
+		config:         &config,
 		ctxTimeout:     30 * time.Second,
 		ctxLongTimeout: 300 * time.Second,
 	}
-	if err := ValidateS3FsConfig(&fs.config); err != nil {
+	if err := fs.config.Validate(); err != nil {
 		return fs, err
 	}
 	awsConfig := aws.NewConfig()
@@ -542,7 +542,7 @@ func (fs *S3Fs) GetRelativePath(name string) string {
 	if rel == "." {
 		rel = ""
 	}
-	if !strings.HasPrefix(rel, "/") {
+	if !path.IsAbs(rel) {
 		return "/" + rel
 	}
 	if fs.config.KeyPrefix != "" {
@@ -697,3 +697,8 @@ func (fs *S3Fs) GetMimeType(name string) (string, error) {
 	}
 	return *obj.ContentType, err
 }
+
+// Close closes the fs
+func (*S3Fs) Close() error {
+	return nil
+}

vfs/sftpfs.go

@@ -0,0 +1,562 @@
+package vfs
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/eikenb/pipeat"
+	"github.com/pkg/sftp"
+	"github.com/rs/xid"
+	"golang.org/x/crypto/ssh"
+
+	"github.com/drakkan/sftpgo/kms"
+	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/utils"
+	"github.com/drakkan/sftpgo/version"
+)
+
+const (
+	// osFsName is the name for the local Fs implementation
+	sftpFsName = "sftpfs"
+)
+
+// SFTPFsConfig defines the configuration for SFTP based filesystem
+type SFTPFsConfig struct {
+	Endpoint     string      `json:"endpoint,omitempty"`
+	Username     string      `json:"username,omitempty"`
+	Password     *kms.Secret `json:"password,omitempty"`
+	PrivateKey   *kms.Secret `json:"private_key,omitempty"`
+	Fingerprints []string    `json:"fingerprints,omitempty"`
+	// Prefix is the path prefix to strip from SFTP resource paths.
+	Prefix string `json:"prefix,omitempty"`
+}
+
+func (c *SFTPFsConfig) setEmptyCredentialsIfNil() {
+	if c.Password == nil {
+		c.Password = kms.NewEmptySecret()
+	}
+	if c.PrivateKey == nil {
+		c.PrivateKey = kms.NewEmptySecret()
+	}
+}
+
+// Validate returns an error if the configuration is not valid
+func (c *SFTPFsConfig) Validate() error {
+	c.setEmptyCredentialsIfNil()
+	if c.Endpoint == "" {
+		return errors.New("endpoint cannot be empty")
+	}
+	if c.Username == "" {
+		return errors.New("username cannot be empty")
+	}
+	if c.Password.IsEmpty() && c.PrivateKey.IsEmpty() {
+		return errors.New("credentials cannot be empty")
+	}
+	if c.Password.IsEncrypted() && !c.Password.IsValid() {
+		return errors.New("invalid encrypted password")
+	}
+	if !c.Password.IsEmpty() && !c.Password.IsValidInput() {
+		return errors.New("invalid password")
+	}
+	if c.PrivateKey.IsEncrypted() && !c.PrivateKey.IsValid() {
+		return errors.New("invalid encrypted private key")
+	}
+	if !c.PrivateKey.IsEmpty() && !c.PrivateKey.IsValidInput() {
+		return errors.New("invalid private key")
+	}
+	if c.Prefix != "" {
+		c.Prefix = utils.CleanPath(c.Prefix)
+	} else {
+		c.Prefix = "/"
+	}
+	return nil
+}
+
+// EncryptCredentials encrypts password and/or private key if they are in plain text
+func (c *SFTPFsConfig) EncryptCredentials(additionalData string) error {
+	if c.Password.IsPlain() {
+		c.Password.SetAdditionalData(additionalData)
+		if err := c.Password.Encrypt(); err != nil {
+			return err
+		}
+	}
+	if c.PrivateKey.IsPlain() {
+		c.PrivateKey.SetAdditionalData(additionalData)
+		if err := c.PrivateKey.Encrypt(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// SFTPFs is a Fs implementation for SFTP backends
+type SFTPFs struct {
+	connectionID string
+	config       *SFTPFsConfig
+	sshClient    *ssh.Client
+	sftpClient   *sftp.Client
+	err          chan error
+}
+
+// NewSFTPFs returns an SFTPFa object that allows to interact with an SFTP server
+func NewSFTPFs(connectionID string, config SFTPFsConfig) (Fs, error) {
+	if err := config.Validate(); err != nil {
+		return nil, err
+	}
+	if !config.Password.IsEmpty() && config.Password.IsEncrypted() {
+		if err := config.Password.Decrypt(); err != nil {
+			return nil, err
+		}
+	}
+	if !config.PrivateKey.IsEmpty() && config.PrivateKey.IsEncrypted() {
+		if err := config.PrivateKey.Decrypt(); err != nil {
+			return nil, err
+		}
+	}
+	sftpFs := &SFTPFs{
+		connectionID: connectionID,
+		config:       &config,
+		err:          make(chan error, 1),
+	}
+	err := sftpFs.createConnection()
+	return sftpFs, err
+}
+
+// Name returns the name for the Fs implementation
+func (fs *SFTPFs) Name() string {
+	return fmt.Sprintf("%v %#v", sftpFsName, fs.config.Endpoint)
+}
+
+// ConnectionID returns the connection ID associated to this Fs implementation
+func (fs *SFTPFs) ConnectionID() string {
+	return fs.connectionID
+}
+
+// Stat returns a FileInfo describing the named file
+func (fs *SFTPFs) Stat(name string) (os.FileInfo, error) {
+	if err := fs.checkConnection(); err != nil {
+		return nil, err
+	}
+	return fs.sftpClient.Stat(name)
+}
+
+// Lstat returns a FileInfo describing the named file
+func (fs *SFTPFs) Lstat(name string) (os.FileInfo, error) {
+	if err := fs.checkConnection(); err != nil {
+		return nil, err
+	}
+	return fs.sftpClient.Lstat(name)
+}
+
+// Open opens the named file for reading
+func (fs *SFTPFs) Open(name string, offset int64) (File, *pipeat.PipeReaderAt, func(), error) {
+	if err := fs.checkConnection(); err != nil {
+		return nil, nil, nil, err
+	}
+	f, err := fs.sftpClient.Open(name)
+	return f, nil, nil, err
+}
+
+// Create creates or opens the named file for writing
+func (fs *SFTPFs) Create(name string, flag int) (File, *PipeWriter, func(), error) {
+	err := fs.checkConnection()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+	var f File
+	if flag == 0 {
+		f, err = fs.sftpClient.Create(name)
+	} else {
+		f, err = fs.sftpClient.OpenFile(name, flag)
+	}
+	return f, nil, nil, err
+}
+
+// Rename renames (moves) source to target.
+func (fs *SFTPFs) Rename(source, target string) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Rename(source, target)
+}
+
+// Remove removes the named file or (empty) directory.
+func (fs *SFTPFs) Remove(name string, isDir bool) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Remove(name)
+}
+
+// Mkdir creates a new directory with the specified name and default permissions
+func (fs *SFTPFs) Mkdir(name string) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Mkdir(name)
+}
+
+// Symlink creates source as a symbolic link to target.
+func (fs *SFTPFs) Symlink(source, target string) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Symlink(source, target)
+}
+
+// Readlink returns the destination of the named symbolic link
+func (fs *SFTPFs) Readlink(name string) (string, error) {
+	if err := fs.checkConnection(); err != nil {
+		return "", err
+	}
+	return fs.sftpClient.ReadLink(name)
+}
+
+// Chown changes the numeric uid and gid of the named file.
+func (fs *SFTPFs) Chown(name string, uid int, gid int) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Chown(name, uid, gid)
+}
+
+// Chmod changes the mode of the named file to mode.
+func (fs *SFTPFs) Chmod(name string, mode os.FileMode) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Chmod(name, mode)
+}
+
+// Chtimes changes the access and modification times of the named file.
+func (fs *SFTPFs) Chtimes(name string, atime, mtime time.Time) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Chtimes(name, atime, mtime)
+}
+
+// Truncate changes the size of the named file.
+func (fs *SFTPFs) Truncate(name string, size int64) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	return fs.sftpClient.Truncate(name, size)
+}
+
+// ReadDir reads the directory named by dirname and returns
+// a list of directory entries.
+func (fs *SFTPFs) ReadDir(dirname string) ([]os.FileInfo, error) {
+	if err := fs.checkConnection(); err != nil {
+		return nil, err
+	}
+	return fs.sftpClient.ReadDir(dirname)
+}
+
+// IsUploadResumeSupported returns true if upload resume is supported.
+func (*SFTPFs) IsUploadResumeSupported() bool {
+	return true
+}
+
+// IsAtomicUploadSupported returns true if atomic upload is supported.
+func (*SFTPFs) IsAtomicUploadSupported() bool {
+	return true
+}
+
+// IsNotExist returns a boolean indicating whether the error is known to
+// report that a file or directory does not exist
+func (*SFTPFs) IsNotExist(err error) bool {
+	return os.IsNotExist(err)
+}
+
+// IsPermission returns a boolean indicating whether the error is known to
+// report that permission is denied.
+func (*SFTPFs) IsPermission(err error) bool {
+	return os.IsPermission(err)
+}
+
+// IsNotSupported returns true if the error indicate an unsupported operation
+func (*SFTPFs) IsNotSupported(err error) bool {
+	if err == nil {
+		return false
+	}
+	return err == ErrVfsUnsupported
+}
+
+// CheckRootPath creates the specified local root directory if it does not exists
+func (fs *SFTPFs) CheckRootPath(username string, uid int, gid int) bool {
+	return true
+}
+
+// ScanRootDirContents returns the number of files contained in a directory and
+// their size
+func (fs *SFTPFs) ScanRootDirContents() (int, int64, error) {
+	return fs.GetDirSize(fs.config.Prefix)
+}
+
+// GetAtomicUploadPath returns the path to use for an atomic upload
+func (*SFTPFs) GetAtomicUploadPath(name string) string {
+	dir := path.Dir(name)
+	guid := xid.New().String()
+	return path.Join(dir, ".sftpgo-upload."+guid+"."+path.Base(name))
+}
+
+// GetRelativePath returns the path for a file relative to the sftp prefix if any.
+// This is the path as seen by SFTPGo users
+func (fs *SFTPFs) GetRelativePath(name string) string {
+	rel := path.Clean(name)
+	if rel == "." {
+		rel = ""
+	}
+	if !path.IsAbs(rel) {
+		return "/" + rel
+	}
+	if fs.config.Prefix != "/" {
+		if !strings.HasPrefix(rel, fs.config.Prefix) {
+			rel = "/"
+		}
+		rel = path.Clean("/" + strings.TrimPrefix(rel, fs.config.Prefix))
+	}
+	return rel
+}
+
+// Walk walks the file tree rooted at root, calling walkFn for each file or
+// directory in the tree, including root
+func (fs *SFTPFs) Walk(root string, walkFn filepath.WalkFunc) error {
+	if err := fs.checkConnection(); err != nil {
+		return err
+	}
+	walker := fs.sftpClient.Walk(root)
+	for walker.Step() {
+		err := walker.Err()
+		if err != nil {
+			return err
+		}
+		err = walkFn(walker.Path(), walker.Stat(), err)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Join joins any number of path elements into a single path
+func (*SFTPFs) Join(elem ...string) string {
+	return path.Join(elem...)
+}
+
+// HasVirtualFolders returns true if folders are emulated
+func (*SFTPFs) HasVirtualFolders() bool {
+	return false
+}
+
+// ResolvePath returns the matching filesystem path for the specified virtual path
+func (fs *SFTPFs) ResolvePath(virtualPath string) (string, error) {
+	if !path.IsAbs(virtualPath) {
+		virtualPath = path.Clean("/" + virtualPath)
+	}
+	fsPath := fs.Join(fs.config.Prefix, virtualPath)
+	if fs.config.Prefix != "/" && fsPath != "/" {
+		// we need to check if this path is a symlink outside the given prefix
+		// or a file/dir inside a dir symlinked outside the prefix
+		if err := fs.checkConnection(); err != nil {
+			return "", err
+		}
+		var validatedPath string
+		var err error
+		validatedPath, err = fs.getRealPath(fsPath)
+		if err != nil && !os.IsNotExist(err) {
+			fsLog(fs, logger.LevelWarn, "Invalid path resolution, original path %v resolved %#v err: %v",
+				virtualPath, fsPath, err)
+			return "", err
+		} else if os.IsNotExist(err) {
+			for os.IsNotExist(err) {
+				validatedPath = path.Dir(validatedPath)
+				if validatedPath == "/" {
+					err = nil
+					break
+				}
+				validatedPath, err = fs.getRealPath(validatedPath)
+			}
+			if err != nil {
+				fsLog(fs, logger.LevelWarn, "Invalid path resolution, dir %#v original path %#v resolved %#v err: %v",
+					validatedPath, virtualPath, fsPath, err)
+				return "", err
+			}
+		}
+		if err := fs.isSubDir(validatedPath); err != nil {
+			fsLog(fs, logger.LevelWarn, "Invalid path resolution, dir %#v original path %#v resolved %#v err: %v",
+				validatedPath, virtualPath, fsPath, err)
+			return "", err
+		}
+	}
+	return fsPath, nil
+}
+
+// getRealPath returns the real remote path trying to resolve symbolic links if any
+func (fs *SFTPFs) getRealPath(name string) (string, error) {
+	info, err := fs.sftpClient.Lstat(name)
+	if err != nil {
+		return name, err
+	}
+	if info.Mode()&os.ModeSymlink != 0 {
+		return fs.sftpClient.ReadLink(name)
+	}
+	return name, err
+}
+
+func (fs *SFTPFs) isSubDir(name string) error {
+	if name == fs.config.Prefix {
+		return nil
+	}
+	if len(name) < len(fs.config.Prefix) {
+		err := fmt.Errorf("path %#v is not inside: %#v", name, fs.config.Prefix)
+		return err
+	}
+	if !strings.HasPrefix(name, fs.config.Prefix+"/") {
+		err := fmt.Errorf("path %#v is not inside: %#v", name, fs.config.Prefix)
+		return err
+	}
+	return nil
+}
+
+// GetDirSize returns the number of files and the size for a folder
+// including any subfolders
+func (fs *SFTPFs) GetDirSize(dirname string) (int, int64, error) {
+	numFiles := 0
+	size := int64(0)
+	if err := fs.checkConnection(); err != nil {
+		return numFiles, size, err
+	}
+	isDir, err := IsDirectory(fs, dirname)
+	if err == nil && isDir {
+		walker := fs.sftpClient.Walk(dirname)
+		for walker.Step() {
+			err := walker.Err()
+			if err != nil {
+				return numFiles, size, err
+			}
+			if walker.Stat().Mode().IsRegular() {
+				size += walker.Stat().Size()
+				numFiles++
+			}
+		}
+	}
+	return numFiles, size, err
+}
+
+// GetMimeType returns the content type
+func (fs *SFTPFs) GetMimeType(name string) (string, error) {
+	if err := fs.checkConnection(); err != nil {
+		return "", err
+	}
+	f, err := fs.sftpClient.OpenFile(name, os.O_RDONLY)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	var buf [512]byte
+	n, err := io.ReadFull(f, buf[:])
+	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+		return "", err
+	}
+	ctype := http.DetectContentType(buf[:n])
+	// Rewind file.
+	_, err = f.Seek(0, io.SeekStart)
+	return ctype, err
+}
+
+// Close the connection
+func (fs *SFTPFs) Close() error {
+	var sftpErr, sshErr error
+	if fs.sftpClient != nil {
+		sftpErr = fs.sftpClient.Close()
+	}
+	if fs.sshClient != nil {
+		sshErr = fs.sshClient.Close()
+	}
+	if sftpErr != nil {
+		return sftpErr
+	}
+	return sshErr
+}
+
+func (fs *SFTPFs) checkConnection() error {
+	err := fs.closed()
+	if err == nil {
+		return nil
+	}
+	return fs.createConnection()
+}
+
+func (fs *SFTPFs) createConnection() error {
+	var err error
+	clientConfig := &ssh.ClientConfig{
+		User: fs.config.Username,
+		HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
+			if len(fs.config.Fingerprints) > 0 {
+				fp := ssh.FingerprintSHA256(key)
+				for _, provided := range fs.config.Fingerprints {
+					if provided == fp {
+						return nil
+					}
+				}
+				return fmt.Errorf("Invalid fingerprint %#v", fp)
+			}
+			fsLog(fs, logger.LevelWarn, "login without host key validation, please provide at least a fingerprint!")
+			return nil
+		},
+		ClientVersion: fmt.Sprintf("SSH-2.0-SFTPGo_%v", version.Get().Version),
+	}
+	if fs.config.PrivateKey.GetPayload() != "" {
+		signer, err := ssh.ParsePrivateKey([]byte(fs.config.PrivateKey.GetPayload()))
+		if err != nil {
+			fs.err <- err
+			return err
+		}
+		clientConfig.Auth = append(clientConfig.Auth, ssh.PublicKeys(signer))
+	}
+	if fs.config.Password.GetPayload() != "" {
+		clientConfig.Auth = append(clientConfig.Auth, ssh.Password(fs.config.Password.GetPayload()))
+	}
+	fs.sshClient, err = ssh.Dial("tcp", fs.config.Endpoint, clientConfig)
+	if err != nil {
+		fs.err <- err
+		return err
+	}
+	fs.sftpClient, err = sftp.NewClient(fs.sshClient)
+	if err != nil {
+		fs.sshClient.Close()
+		fs.err <- err
+		return err
+	}
+	go fs.wait()
+	return nil
+}
+
+func (fs *SFTPFs) wait() {
+	// we wait on the sftp client otherwise if the channel is closed but not the connection
+	// we don't detect the event.
+	fs.err <- fs.sftpClient.Wait()
+	fsLog(fs, logger.LevelDebug, "sftp channel closed")
+	if fs.sshClient != nil {
+		fs.sshClient.Close()
+	}
+}
+
+func (fs *SFTPFs) closed() error {
+	select {
+	case err := <-fs.err:
+		return err
+	default:
+		return nil
+	}
+}

vfs/vfs.go

@@ -57,6 +57,7 @@ type Fs interface {
 	Join(elem ...string) string
 	HasVirtualFolders() bool
 	GetMimeType(name string) (string, error)
+	Close() error
 }
 
 // File defines an interface representing a SFTPGo file
@@ -129,6 +130,66 @@ type S3FsConfig struct {
 	UploadConcurrency int `json:"upload_concurrency,omitempty"`
 }
 
+func (c *S3FsConfig) checkCredentials() error {
+	if c.AccessKey == "" && !c.AccessSecret.IsEmpty() {
+		return errors.New("access_key cannot be empty with access_secret not empty")
+	}
+	if c.AccessSecret.IsEmpty() && c.AccessKey != "" {
+		return errors.New("access_secret cannot be empty with access_key not empty")
+	}
+	if c.AccessSecret.IsEncrypted() && !c.AccessSecret.IsValid() {
+		return errors.New("invalid encrypted access_secret")
+	}
+	if !c.AccessSecret.IsEmpty() && !c.AccessSecret.IsValidInput() {
+		return errors.New("invalid access_secret")
+	}
+	return nil
+}
+
+// EncryptCredentials encrypts access secret if it is in plain text
+func (c *S3FsConfig) EncryptCredentials(additionalData string) error {
+	if c.AccessSecret.IsPlain() {
+		c.AccessSecret.SetAdditionalData(additionalData)
+		err := c.AccessSecret.Encrypt()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Validate returns an error if the configuration is not valid
+func (c *S3FsConfig) Validate() error {
+	if c.AccessSecret == nil {
+		c.AccessSecret = kms.NewEmptySecret()
+	}
+	if c.Bucket == "" {
+		return errors.New("bucket cannot be empty")
+	}
+	if c.Region == "" {
+		return errors.New("region cannot be empty")
+	}
+	if err := c.checkCredentials(); err != nil {
+		return err
+	}
+	if c.KeyPrefix != "" {
+		if strings.HasPrefix(c.KeyPrefix, "/") {
+			return errors.New("key_prefix cannot start with /")
+		}
+		c.KeyPrefix = path.Clean(c.KeyPrefix)
+		if !strings.HasSuffix(c.KeyPrefix, "/") {
+			c.KeyPrefix += "/"
+		}
+	}
+	if c.UploadPartSize != 0 && (c.UploadPartSize < 5 || c.UploadPartSize > 5000) {
+		return errors.New("upload_part_size cannot be != 0, lower than 5 (MB) or greater than 5000 (MB)")
+	}
+	if c.UploadConcurrency < 0 || c.UploadConcurrency > 64 {
+		return fmt.Errorf("invalid upload concurrency: %v", c.UploadConcurrency)
+	}
+	return nil
+}
+
 // GCSFsConfig defines the configuration for Google Cloud Storage based filesystem
 type GCSFsConfig struct {
 	Bucket string `json:"bucket,omitempty"`
@@ -146,6 +207,38 @@ type GCSFsConfig struct {
 	StorageClass         string `json:"storage_class,omitempty"`
 }
 
+// Validate returns an error if the configuration is not valid
+func (c *GCSFsConfig) Validate(credentialsFilePath string) error {
+	if c.Credentials == nil {
+		c.Credentials = kms.NewEmptySecret()
+	}
+	if c.Bucket == "" {
+		return errors.New("bucket cannot be empty")
+	}
+	if c.KeyPrefix != "" {
+		if strings.HasPrefix(c.KeyPrefix, "/") {
+			return errors.New("key_prefix cannot start with /")
+		}
+		c.KeyPrefix = path.Clean(c.KeyPrefix)
+		if !strings.HasSuffix(c.KeyPrefix, "/") {
+			c.KeyPrefix += "/"
+		}
+	}
+	if c.Credentials.IsEncrypted() && !c.Credentials.IsValid() {
+		return errors.New("invalid encrypted credentials")
+	}
+	if !c.Credentials.IsValidInput() && c.AutomaticCredentials == 0 {
+		fi, err := os.Stat(credentialsFilePath)
+		if err != nil {
+			return fmt.Errorf("invalid credentials %v", err)
+		}
+		if fi.Size() == 0 {
+			return errors.New("credentials cannot be empty")
+		}
+	}
+	return nil
+}
+
 // AzBlobFsConfig defines the configuration for Azure Blob Storage based filesystem
 type AzBlobFsConfig struct {
 	Container string `json:"container,omitempty"`
@@ -183,11 +276,93 @@ type AzBlobFsConfig struct {
 	AccessTier string `json:"access_tier,omitempty"`
 }
 
+// EncryptCredentials encrypts access secret if it is in plain text
+func (c *AzBlobFsConfig) EncryptCredentials(additionalData string) error {
+	if c.AccountKey.IsPlain() {
+		c.AccountKey.SetAdditionalData(additionalData)
+		if err := c.AccountKey.Encrypt(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *AzBlobFsConfig) checkCredentials() error {
+	if c.AccountName == "" || !c.AccountKey.IsValidInput() {
+		return errors.New("credentials cannot be empty or invalid")
+	}
+	if c.AccountKey.IsEncrypted() && !c.AccountKey.IsValid() {
+		return errors.New("invalid encrypted account_key")
+	}
+	return nil
+}
+
+// Validate returns an error if the configuration is not valid
+func (c *AzBlobFsConfig) Validate() error {
+	if c.AccountKey == nil {
+		c.AccountKey = kms.NewEmptySecret()
+	}
+	if c.SASURL != "" {
+		_, err := url.Parse(c.SASURL)
+		return err
+	}
+	if c.Container == "" {
+		return errors.New("container cannot be empty")
+	}
+	if err := c.checkCredentials(); err != nil {
+		return err
+	}
+	if c.KeyPrefix != "" {
+		if strings.HasPrefix(c.KeyPrefix, "/") {
+			return errors.New("key_prefix cannot start with /")
+		}
+		c.KeyPrefix = path.Clean(c.KeyPrefix)
+		if !strings.HasSuffix(c.KeyPrefix, "/") {
+			c.KeyPrefix += "/"
+		}
+	}
+	if c.UploadPartSize < 0 || c.UploadPartSize > 100 {
+		return fmt.Errorf("invalid upload part size: %v", c.UploadPartSize)
+	}
+	if c.UploadConcurrency < 0 || c.UploadConcurrency > 64 {
+		return fmt.Errorf("invalid upload concurrency: %v", c.UploadConcurrency)
+	}
+	if !utils.IsStringInSlice(c.AccessTier, validAzAccessTier) {
+		return fmt.Errorf("invalid access tier %#v, valid values: \"''%v\"", c.AccessTier, strings.Join(validAzAccessTier, ", "))
+	}
+	return nil
+}
+
 // CryptFsConfig defines the configuration to store local files as encrypted
 type CryptFsConfig struct {
 	Passphrase *kms.Secret `json:"passphrase,omitempty"`
 }
 
+// EncryptCredentials encrypts access secret if it is in plain text
+func (c *CryptFsConfig) EncryptCredentials(additionalData string) error {
+	if c.Passphrase.IsPlain() {
+		c.Passphrase.SetAdditionalData(additionalData)
+		if err := c.Passphrase.Encrypt(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Validate returns an error if the configuration is not valid
+func (c *CryptFsConfig) Validate() error {
+	if c.Passphrase == nil || c.Passphrase.IsEmpty() {
+		return errors.New("invalid passphrase")
+	}
+	if !c.Passphrase.IsValidInput() {
+		return errors.New("passphrase cannot be empty or invalid")
+	}
+	if c.Passphrase.IsEncrypted() && !c.Passphrase.IsValid() {
+		return errors.New("invalid encrypted passphrase")
+	}
+	return nil
+}
+
 // PipeWriter defines a wrapper for pipeat.PipeWriterAt.
 type PipeWriter struct {
 	writer *pipeat.PipeWriterAt
@@ -247,149 +422,22 @@ func IsCryptOsFs(fs Fs) bool {
 	return fs.Name() == cryptFsName
 }
 
-func checkS3Credentials(config *S3FsConfig) error {
-	if config.AccessKey == "" && !config.AccessSecret.IsEmpty() {
-		return errors.New("access_key cannot be empty with access_secret not empty")
-	}
-	if config.AccessSecret.IsEmpty() && config.AccessKey != "" {
-		return errors.New("access_secret cannot be empty with access_key not empty")
-	}
-	if config.AccessSecret.IsEncrypted() && !config.AccessSecret.IsValid() {
-		return errors.New("invalid encrypted access_secret")
-	}
-	if !config.AccessSecret.IsEmpty() && !config.AccessSecret.IsValidInput() {
-		return errors.New("invalid access_secret")
-	}
-	return nil
+// IsSFTPFs returns true if fs is a SFTP filesystem
+func IsSFTPFs(fs Fs) bool {
+	return strings.HasPrefix(fs.Name(), sftpFsName)
 }
 
-// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error
-func ValidateS3FsConfig(config *S3FsConfig) error {
-	if config.AccessSecret == nil {
-		config.AccessSecret = kms.NewEmptySecret()
-	}
-	if config.Bucket == "" {
-		return errors.New("bucket cannot be empty")
-	}
-	if config.Region == "" {
-		return errors.New("region cannot be empty")
-	}
-	if err := checkS3Credentials(config); err != nil {
-		return err
-	}
-	if config.KeyPrefix != "" {
-		if strings.HasPrefix(config.KeyPrefix, "/") {
-			return errors.New("key_prefix cannot start with /")
-		}
-		config.KeyPrefix = path.Clean(config.KeyPrefix)
-		if !strings.HasSuffix(config.KeyPrefix, "/") {
-			config.KeyPrefix += "/"
-		}
-	}
-	if config.UploadPartSize != 0 && (config.UploadPartSize < 5 || config.UploadPartSize > 5000) {
-		return errors.New("upload_part_size cannot be != 0, lower than 5 (MB) or greater than 5000 (MB)")
-	}
-	if config.UploadConcurrency < 0 || config.UploadConcurrency > 64 {
-		return fmt.Errorf("invalid upload concurrency: %v", config.UploadConcurrency)
-	}
-	return nil
-}
-
-// ValidateGCSFsConfig returns nil if the specified GCS config is valid, otherwise an error
-func ValidateGCSFsConfig(config *GCSFsConfig, credentialsFilePath string) error {
-	if config.Credentials == nil {
-		config.Credentials = kms.NewEmptySecret()
-	}
-	if config.Bucket == "" {
-		return errors.New("bucket cannot be empty")
-	}
-	if config.KeyPrefix != "" {
-		if strings.HasPrefix(config.KeyPrefix, "/") {
-			return errors.New("key_prefix cannot start with /")
-		}
-		config.KeyPrefix = path.Clean(config.KeyPrefix)
-		if !strings.HasSuffix(config.KeyPrefix, "/") {
-			config.KeyPrefix += "/"
-		}
-	}
-	if config.Credentials.IsEncrypted() && !config.Credentials.IsValid() {
-		return errors.New("invalid encrypted credentials")
-	}
-	if !config.Credentials.IsValidInput() && config.AutomaticCredentials == 0 {
-		fi, err := os.Stat(credentialsFilePath)
-		if err != nil {
-			return fmt.Errorf("invalid credentials %v", err)
-		}
-		if fi.Size() == 0 {
-			return errors.New("credentials cannot be empty")
-		}
-	}
-	return nil
-}
-
-func checkAzCredentials(config *AzBlobFsConfig) error {
-	if config.AccountName == "" || !config.AccountKey.IsValidInput() {
-		return errors.New("credentials cannot be empty or invalid")
-	}
-	if config.AccountKey.IsEncrypted() && !config.AccountKey.IsValid() {
-		return errors.New("invalid encrypted account_key")
-	}
-	return nil
-}
-
-// ValidateAzBlobFsConfig returns nil if the specified Azure Blob config is valid, otherwise an error
-func ValidateAzBlobFsConfig(config *AzBlobFsConfig) error {
-	if config.AccountKey == nil {
-		config.AccountKey = kms.NewEmptySecret()
-	}
-	if config.SASURL != "" {
-		_, err := url.Parse(config.SASURL)
-		return err
-	}
-	if config.Container == "" {
-		return errors.New("container cannot be empty")
-	}
-	if err := checkAzCredentials(config); err != nil {
-		return err
-	}
-	if config.KeyPrefix != "" {
-		if strings.HasPrefix(config.KeyPrefix, "/") {
-			return errors.New("key_prefix cannot start with /")
-		}
-		config.KeyPrefix = path.Clean(config.KeyPrefix)
-		if !strings.HasSuffix(config.KeyPrefix, "/") {
-			config.KeyPrefix += "/"
-		}
-	}
-	if config.UploadPartSize < 0 || config.UploadPartSize > 100 {
-		return fmt.Errorf("invalid upload part size: %v", config.UploadPartSize)
-	}
-	if config.UploadConcurrency < 0 || config.UploadConcurrency > 64 {
-		return fmt.Errorf("invalid upload concurrency: %v", config.UploadConcurrency)
-	}
-	if !utils.IsStringInSlice(config.AccessTier, validAzAccessTier) {
-		return fmt.Errorf("invalid access tier %#v, valid values: \"''%v\"", config.AccessTier, strings.Join(validAzAccessTier, ", "))
-	}
-	return nil
-}
-
-// ValidateCryptFsConfig returns nil if the specified CryptFs config is valid, otherwise an error
-func ValidateCryptFsConfig(config *CryptFsConfig) error {
-	if config.Passphrase == nil || config.Passphrase.IsEmpty() {
-		return errors.New("invalid passphrase")
-	}
-	if !config.Passphrase.IsValidInput() {
-		return errors.New("passphrase cannot be empty or invalid")
-	}
-	if config.Passphrase.IsEncrypted() && !config.Passphrase.IsValid() {
-		return errors.New("invalid encrypted passphrase")
-	}
-	return nil
+// IsLocalOrSFTPFs returns true if fs is local or SFTP
+func IsLocalOrSFTPFs(fs Fs) bool {
+	return IsLocalOsFs(fs) || IsSFTPFs(fs)
 }
 
 // SetPathPermissions calls fs.Chown.
 // It does nothing for local filesystem on windows
 func SetPathPermissions(fs Fs, path string, uid int, gid int) {
+	if uid == -1 && gid == -1 {
+		return
+	}
 	if IsLocalOsFs(fs) {
 		if runtime.GOOS == "windows" {
 			return