diff --git a/README.md b/README.md index 5a03751d..1ec3c6ea 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ It can serve local filesystem, S3 (compatible) Object Storage, Google Cloud Stor - [Web based administration interface](./docs/web-admin.md) to easily manage users, folders and connections. - Easy [migration](./examples/rest-api-cli#convert-users-from-other-stores) from Linux system user accounts. - [Portable mode](./docs/portable-mode.md): a convenient way to share a single directory on demand. +- [SFTP subsystem mode](./docs/sftp-subsystem.md): you can use SFTPGo as OpenSSH's SFTP subsystem. - Performance analysis using built-in [profiler](./docs/profiling.md). - Configuration format is at your choice: JSON, TOML, YAML, HCL, envfile are supported. - Log files are accurate and they are saved in the easily parsable JSON format ([more information](./docs/logs.md)). diff --git a/cmd/startsubsys.go b/cmd/startsubsys.go new file mode 100644 index 00000000..749edd97 --- /dev/null +++ b/cmd/startsubsys.go @@ -0,0 +1,167 @@ +package cmd + +import ( + "io" + "os" + "os/user" + "path/filepath" + + "github.com/rs/xid" + "github.com/rs/zerolog" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "github.com/drakkan/sftpgo/common" + "github.com/drakkan/sftpgo/config" + "github.com/drakkan/sftpgo/dataprovider" + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/sftpd" + "github.com/drakkan/sftpgo/version" +) + +var ( + logJournalD = false + preserveHomeDir = false + subsystemCmd = &cobra.Command{ + Use: "startsubsys", + Short: "Use SFTPGo as SFTP file transfer subsystem", + Long: `In this mode SFTPGo speaks the server side of SFTP protocol to stdout and +expects client requests from stdin. +This mode is not intended to be called directly, but from sshd using the +Subsystem option. +For example adding a line like this one in "/etc/ssh/sshd_config": + +Subsystem sftp sftpgo startsubsys + +Command-line flags should be specified in the Subsystem declaration. +`, + Run: func(cmd *cobra.Command, args []string) { + logSender := "startsubsys" + connectionID := xid.New().String() + logLevel := zerolog.DebugLevel + if !logVerbose { + logLevel = zerolog.InfoLevel + } + if logJournalD { + logger.InitJournalDLogger(logLevel) + } else { + logger.InitStdErrLogger(logLevel) + } + osUser, err := user.Current() + if err != nil { + logger.Error(logSender, connectionID, "unable to get the current user: %v", err) + os.Exit(1) + } + username := osUser.Username + homedir := osUser.HomeDir + logger.Info(logSender, connectionID, "starting SFTPGo %v as subsystem, user %#v home dir %#v config dir %#v", + version.Get(), username, homedir, configDir) + err = config.LoadConfig(configDir, configFile) + if err != nil { + logger.Error(logSender, connectionID, "unable to load configuration: %v", err) + os.Exit(1) + } + commonConfig := config.GetCommonConfig() + // idle connection are managed externally + commonConfig.IdleTimeout = 0 + config.SetCommonConfig(commonConfig) + common.Initialize(config.GetCommonConfig()) + dataProviderConf := config.GetProviderConf() + if dataProviderConf.Driver == dataprovider.SQLiteDataProviderName || dataProviderConf.Driver == dataprovider.BoltDataProviderName { + logger.Debug(logSender, connectionID, "data provider %#v not supported in subsystem mode, using %#v provider", + dataProviderConf.Driver, dataprovider.MemoryDataProviderName) + dataProviderConf.Driver = dataprovider.MemoryDataProviderName + dataProviderConf.Name = "" + dataProviderConf.PreferDatabaseCredentials = true + } + config.SetProviderConf(dataProviderConf) + err = dataprovider.Initialize(dataProviderConf, configDir) + if err != nil { + logger.Error(logSender, connectionID, "unable to initialize the data provider: %v", err) + os.Exit(1) + } + httpConfig := config.GetHTTPConfig() + httpConfig.Initialize(configDir) + user, err := dataprovider.UserExists(username) + if err == nil { + if user.HomeDir != filepath.Clean(homedir) && !preserveHomeDir { + // update the user + user.HomeDir = filepath.Clean(homedir) + err = dataprovider.UpdateUser(user) + if err != nil { + logger.Error(logSender, connectionID, "unable to update user %#v: %v", username, err) + os.Exit(1) + } + } + } else { + user.Username = username + user.HomeDir = homedir + user.Password = connectionID + user.Permissions = make(map[string][]string) + user.Permissions["/"] = []string{dataprovider.PermAny} + err = dataprovider.AddUser(user) + if err != nil { + logger.Error(logSender, connectionID, "unable to add user %#v: %v", username, err) + os.Exit(1) + } + } + err = sftpd.ServeSubSystemConnection(user, connectionID, os.Stdin, os.Stdout) + if err != nil && err != io.EOF { + logger.Warn(logSender, connectionID, "serving subsystem finished with error: %v", err) + os.Exit(1) + } + logger.Info(logSender, connectionID, "serving subsystem finished") + os.Exit(0) + }, + } +) + +func init() { + subsystemCmd.Flags().BoolVarP(&preserveHomeDir, "preserve-home", "p", false, `If the user already exists, the existing home +directory will not be changed`) + subsystemCmd.Flags().BoolVarP(&logJournalD, "log-to-journald", "j", false, `Send logs to journald. Only available on Linux. +Use: + +$ journalctl -o verbose -f + +To see full logs. +If not set, the logs will be sent to the standard +error`) + viper.SetDefault(configDirKey, defaultConfigDir) + viper.BindEnv(configDirKey, "SFTPGO_CONFIG_DIR") //nolint:errcheck // err is not nil only if the key to bind is missing + subsystemCmd.Flags().StringVarP(&configDir, configDirFlag, "c", viper.GetString(configDirKey), + `Location for SFTPGo config dir. This directory +should contain the "sftpgo" configuration file +or the configured config-file and it is used as +the base for files with a relative path (eg. the +private keys for the SFTP server, the SQLite +database if you use SQLite as data provider). +This flag can be set using SFTPGO_CONFIG_DIR +env var too.`) + viper.BindPFlag(configDirKey, subsystemCmd.Flags().Lookup(configDirFlag)) //nolint:errcheck + + viper.SetDefault(configFileKey, defaultConfigName) + viper.BindEnv(configFileKey, "SFTPGO_CONFIG_FILE") //nolint:errcheck + subsystemCmd.Flags().StringVarP(&configFile, configFileFlag, "f", viper.GetString(configFileKey), + `Name for SFTPGo configuration file. It must be +the name of a file stored in config-dir not the +absolute path to the configuration file. The +specified file name must have no extension we +automatically load JSON, YAML, TOML, HCL and +Java properties. Therefore if you set "sftpgo" +then "sftpgo.json", "sftpgo.yaml" and so on +are searched. +This flag can be set using SFTPGO_CONFIG_FILE +env var too.`) + viper.BindPFlag(configFileKey, subsystemCmd.Flags().Lookup(configFileFlag)) //nolint:errcheck + + viper.SetDefault(logVerboseKey, defaultLogVerbose) + viper.BindEnv(logVerboseKey, "SFTPGO_LOG_VERBOSE") //nolint:errcheck + subsystemCmd.Flags().BoolVarP(&logVerbose, logVerboseFlag, "v", viper.GetBool(logVerboseKey), + `Enable verbose logs. This flag can be set +using SFTPGO_LOG_VERBOSE env var too. +`) + viper.BindPFlag(logVerboseKey, subsystemCmd.Flags().Lookup(logVerboseFlag)) //nolint:errcheck + + rootCmd.AddCommand(subsystemCmd) +} diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 0e0318f5..174610af 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -384,8 +384,10 @@ func Initialize(cnf Config, basePath string) error { if err = validateHooks(); err != nil { return err } - if err = validateCredentialsDir(basePath); err != nil { - return err + if !cnf.PreferDatabaseCredentials { + if err = validateCredentialsDir(basePath); err != nil { + return err + } } err = createProvider(basePath) if err != nil { diff --git a/docs/sftp-subsystem.md b/docs/sftp-subsystem.md new file mode 100644 index 00000000..b743a48b --- /dev/null +++ b/docs/sftp-subsystem.md @@ -0,0 +1,57 @@ +# SFTP subsystem mode + +In this mode SFTPGo speaks the server side of SFTP protocol to stdout and expects client requests from stdin. +You can use SFTPGo as subsystem via the `startsubsys` command. +This mode is not intended to be called directly, but from sshd using the `Subsystem` option. +For example adding a line like this one in `/etc/ssh/sshd_config`: + +```shell +Subsystem sftp sftpgo startsubsys +``` + +Command-line flags should be specified in the Subsystem declaration. + +```shell +Usage: + sftpgo startsubsys [flags] + +Flags: + -c, --config-dir string Location for SFTPGo config dir. This directory + should contain the "sftpgo" configuration file + or the configured config-file and it is used as + the base for files with a relative path (eg. the + private keys for the SFTP server, the SQLite + database if you use SQLite as data provider). + This flag can be set using SFTPGO_CONFIG_DIR + env var too. (default ".") + -f, --config-file string Name for SFTPGo configuration file. It must be + the name of a file stored in config-dir not the + absolute path to the configuration file. The + specified file name must have no extension we + automatically load JSON, YAML, TOML, HCL and + Java properties. Therefore if you set "sftpgo" + then "sftpgo.json", "sftpgo.yaml" and so on + are searched. + This flag can be set using SFTPGO_CONFIG_FILE + env var too. (default "sftpgo") + -h, --help help for startsubsys + -j, --log-to-journald Send logs to journald. Only available on Linux. + Use: + + $ journalctl -o verbose -f + + To see full logs. + If not set, the logs will be sent to the standard + error + -v, --log-verbose Enable verbose logs. This flag can be set + using SFTPGO_LOG_VERBOSE env var too. + (default true) + -p, --preserve-home If the user already exists, the existing home + directory will not be changed +``` + +In this mode `bolt` and `sqlite` providers are not usable as the same database file cannot be shared among multiple processes, if one of these provider is configured it will be automatically changed to `memory` provider. + +The username and home directory for the logged in user are determined using [user.Current()](https://golang.org/pkg/os/user/#Current). +If the user who is logging is not found within the SFTPGo data provider, it is added automatically. +You can pre-configure the users inside the SFTPGo data provider, this way you can use a different home directory, restrict permissions and such. diff --git a/examples/ldapauthserver/logger/request_logger.go b/examples/ldapauthserver/logger/request_logger.go index 9a7d806f..079884f0 100644 --- a/examples/ldapauthserver/logger/request_logger.go +++ b/examples/ldapauthserver/logger/request_logger.go @@ -58,7 +58,7 @@ func (l *StructuredLoggerEntry) Write(status, bytes int, header http.Header, ela Int("resp_status", status). Int("resp_size", bytes). Int64("elapsed_ms", elapsed.Nanoseconds()/1000000). - Msg("") + Send() } // Panic logs panics @@ -69,5 +69,5 @@ func (l *StructuredLoggerEntry) Panic(v interface{}, stack []byte) { Fields(l.fields). Str("stack", string(stack)). Str("panic", fmt.Sprintf("%+v", v)). - Msg("") + Send() } diff --git a/go.sum b/go.sum index d295a1a5..bcd6801a 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,7 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= diff --git a/logger/journald.go b/logger/journald.go new file mode 100644 index 00000000..ee036fd2 --- /dev/null +++ b/logger/journald.go @@ -0,0 +1,14 @@ +// +build linux + +package logger + +import ( + "github.com/rs/zerolog" + "github.com/rs/zerolog/journald" +) + +// InitJournalDLogger configures the logger to write to journald +func InitJournalDLogger(level zerolog.Level) { + logger = zerolog.New(journald.NewJournalDWriter()).Level(level) + consoleLogger = zerolog.Nop() +} diff --git a/logger/journald_nolinux.go b/logger/journald_nolinux.go new file mode 100644 index 00000000..77cee1a4 --- /dev/null +++ b/logger/journald_nolinux.go @@ -0,0 +1,10 @@ +// +build !linux + +package logger + +import "github.com/rs/zerolog" + +// InitJournalDLogger configures the logger to write to journald +func InitJournalDLogger(level zerolog.Level) { + InitStdErrLogger(level) +} diff --git a/logger/logger.go b/logger/logger.go index da33272b..98014a1a 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -67,6 +67,14 @@ func InitLogger(logFilePath string, logMaxSize int, logMaxBackups int, logMaxAge logger = logger.Level(level) } +// InitStdErrLogger configures the logger to write to stderr +func InitStdErrLogger(level zerolog.Level) { + logger = zerolog.New(&logSyncWrapper{ + output: os.Stderr, + }).Level(level) + consoleLogger = zerolog.Nop() +} + // DisableLogger disable the main logger. // ConsoleLogger will not be affected func DisableLogger() { @@ -163,7 +171,7 @@ func TransferLog(operation string, path string, elapsed int64, size int64, user Str("file_path", path). Str("connection_id", connectionID). Str("protocol", protocol). - Msg("") + Send() } // CommandLog logs an SFTP/SCP/SSH command @@ -184,7 +192,7 @@ func CommandLog(command, path, target, user, fileMode, connectionID, protocol st Str("ssh_command", sshCommand). Str("connection_id", connectionID). Str("protocol", protocol). - Msg("") + Send() } // ConnectionFailedLog logs failed attempts to initialize a connection. @@ -200,7 +208,7 @@ func ConnectionFailedLog(user, ip, loginType, protocol, errorString string) { Str("login_type", loginType). Str("protocol", protocol). Str("error", errorString). - Msg("") + Send() } func isLogFilePathValid(logFilePath string) bool { diff --git a/logger/request_logger.go b/logger/request_logger.go index eeb84afa..69e1cef8 100644 --- a/logger/request_logger.go +++ b/logger/request_logger.go @@ -64,7 +64,7 @@ func (l *StructuredLoggerEntry) Write(status, bytes int, header http.Header, ela Int("resp_status", status). Int("resp_size", bytes). Int64("elapsed_ms", elapsed.Nanoseconds()/1000000). - Msg("") + Send() } // Panic logs panics @@ -75,5 +75,5 @@ func (l *StructuredLoggerEntry) Panic(v interface{}, stack []byte) { Fields(l.fields). Str("stack", string(stack)). Str("panic", fmt.Sprintf("%+v", v)). - Msg("") + Send() } diff --git a/sftpd/handler.go b/sftpd/handler.go index b262c9c7..b0a999df 100644 --- a/sftpd/handler.go +++ b/sftpd/handler.go @@ -8,7 +8,6 @@ import ( "time" "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/dataprovider" @@ -23,7 +22,7 @@ type Connection struct { ClientVersion string // Remote address for this connection RemoteAddr net.Addr - channel ssh.Channel + channel io.ReadWriteCloser command string } diff --git a/sftpd/internal_test.go b/sftpd/internal_test.go index 57e02fb9..af921d74 100644 --- a/sftpd/internal_test.go +++ b/sftpd/internal_test.go @@ -1814,3 +1814,35 @@ func TestRecursiveCopyErrors(t *testing.T) { err = sshCmd.checkRecursiveCopyPermissions("adir", "another", "/another") assert.Error(t, err) } + +func TestSFTPSubSystem(t *testing.T) { + permissions := make(map[string][]string) + permissions["/"] = []string{dataprovider.PermAny} + user := dataprovider.User{ + Permissions: permissions, + HomeDir: os.TempDir(), + } + user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider + err := ServeSubSystemConnection(user, "connID", nil, nil) + assert.Error(t, err) + user.FsConfig.Provider = dataprovider.LocalFilesystemProvider + + buf := make([]byte, 0, 4096) + stdErrBuf := make([]byte, 0, 4096) + mockSSHChannel := &MockChannel{ + Buffer: bytes.NewBuffer(buf), + StdErrBuffer: bytes.NewBuffer(stdErrBuf), + } + // this is 327680 and it will result in packet too long error + _, err = mockSSHChannel.Write([]byte{0x00, 0x05, 0x00, 0x00, 0x00, 0x00}) + assert.NoError(t, err) + err = ServeSubSystemConnection(user, "id", mockSSHChannel, mockSSHChannel) + assert.EqualError(t, err, "packet too long") + + subsystemChannel := newSubsystemChannel(mockSSHChannel, mockSSHChannel) + n, err := subsystemChannel.Write([]byte{0x00}) + assert.NoError(t, err) + assert.Equal(t, n, 1) + err = subsystemChannel.Close() + assert.NoError(t, err) +} diff --git a/sftpd/ssh_cmd.go b/sftpd/ssh_cmd.go index 51f115ca..1b879387 100644 --- a/sftpd/ssh_cmd.go +++ b/sftpd/ssh_cmd.go @@ -381,7 +381,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error { common.TransferDownload, 0, 0, 0, false, c.connection.Fs) transfer := newTransfer(baseTransfer, nil, nil, nil) - w, e := transfer.copyFromReaderToWriter(c.connection.channel.Stderr(), stderr) + w, e := transfer.copyFromReaderToWriter(c.connection.channel.(ssh.Channel).Stderr(), stderr) c.connection.Log(logger.LevelDebug, "command: %#v, copy from sdterr to remote command ended, written: %v err: %v", c.connection.command, w, e) // os.ErrClosed means that the command is finished so we don't need to do anything @@ -707,7 +707,7 @@ func (c *sshCommand) sendExitStatus(err error) { exitStatus := sshSubsystemExitStatus{ Status: status, } - c.connection.channel.SendRequest("exit-status", false, ssh.Marshal(&exitStatus)) //nolint:errcheck + c.connection.channel.(ssh.Channel).SendRequest("exit-status", false, ssh.Marshal(&exitStatus)) //nolint:errcheck c.connection.channel.Close() // for scp we notify single uploads/downloads if c.command != scpCmdName { diff --git a/sftpd/subsystem.go b/sftpd/subsystem.go new file mode 100644 index 00000000..a201bd85 --- /dev/null +++ b/sftpd/subsystem.go @@ -0,0 +1,63 @@ +package sftpd + +import ( + "io" + "net" + + "github.com/pkg/sftp" + + "github.com/drakkan/sftpgo/common" + "github.com/drakkan/sftpgo/dataprovider" +) + +type subsystemChannel struct { + reader io.Reader + writer io.Writer +} + +func (s *subsystemChannel) Read(p []byte) (int, error) { + return s.reader.Read(p) +} + +func (s *subsystemChannel) Write(p []byte) (int, error) { + return s.writer.Write(p) +} + +func (s *subsystemChannel) Close() error { + return nil +} + +func newSubsystemChannel(reader io.Reader, writer io.Writer) *subsystemChannel { + return &subsystemChannel{ + reader: reader, + writer: writer, + } +} + +// ServeSubSystemConnection handles a connection as SSH subsystem +func ServeSubSystemConnection(user dataprovider.User, connectionID string, reader io.Reader, writer io.Writer) error { + fs, err := user.GetFilesystem(connectionID) + if err != nil { + return err + } + fs.CheckRootPath(user.Username, user.GetUID(), user.GetGID()) + dataprovider.UpdateLastLogin(user) //nolint:errcheck + + connection := &Connection{ + BaseConnection: common.NewBaseConnection(fs.ConnectionID(), common.ProtocolSFTP, user, fs), + ClientVersion: "", + RemoteAddr: &net.IPAddr{}, + channel: newSubsystemChannel(reader, writer), + } + common.Connections.Add(connection) + defer common.Connections.Remove(connection.GetID()) + + server := sftp.NewRequestServer(connection.channel, sftp.Handlers{ + FileGet: connection, + FilePut: connection, + FileCmd: connection, + FileList: connection, + }, sftp.WithRSAllocator()) + + return server.Serve() +} diff --git a/webdavd/server.go b/webdavd/server.go index bd02499a..fd32882d 100644 --- a/webdavd/server.go +++ b/webdavd/server.go @@ -255,7 +255,7 @@ func writeLog(r *http.Request, err error) { Str("sender", logSender). Fields(fields). Err(err). - Msg("") + Send() } func checkRemoteAddress(r *http.Request) {