sftpfs: add buffering support

this way we improve performance over high latency networks

httpd/httpd_test.go

@@ -690,6 +690,19 @@ func TestAddUserInvalidFsConfig(t *testing.T) {
 	u.FsConfig.SFTPConfig.PrivateKey = kms.NewSecret(kms.SecretStatusRedacted, "keyforpkey", "", "")
 	_, _, err = httpdtest.AddUser(u, http.StatusBadRequest)
 	assert.NoError(t, err)
+	u.FsConfig.SFTPConfig.PrivateKey = kms.NewPlainSecret("pk")
+	u.FsConfig.SFTPConfig.Endpoint = "127.1.1.1:22"
+	u.FsConfig.SFTPConfig.Username = defaultUsername
+	u.FsConfig.SFTPConfig.BufferSize = -1
+	_, resp, err := httpdtest.AddUser(u, http.StatusBadRequest)
+	if assert.NoError(t, err) {
+		assert.Contains(t, string(resp), "invalid buffer_size")
+	}
+	u.FsConfig.SFTPConfig.BufferSize = 1000
+	_, resp, err = httpdtest.AddUser(u, http.StatusBadRequest)
+	if assert.NoError(t, err) {
+		assert.Contains(t, string(resp), "invalid buffer_size")
+	}
 }
 
 func TestUserRedactedPassword(t *testing.T) {
@@ -1545,6 +1558,7 @@ func TestUserSFTPFs(t *testing.T) {
 	user.FsConfig.SFTPConfig.Password = kms.NewPlainSecret("sftp_pwd")
 	user.FsConfig.SFTPConfig.PrivateKey = kms.NewPlainSecret(sftpPrivateKey)
 	user.FsConfig.SFTPConfig.Fingerprints = []string{sftpPkeyFingerprint}
+	user.FsConfig.SFTPConfig.BufferSize = 2
 	_, resp, err := httpdtest.UpdateUser(user, http.StatusBadRequest, "")
 	assert.NoError(t, err)
 	assert.Contains(t, string(resp), "invalid endpoint")
@@ -1555,6 +1569,7 @@ func TestUserSFTPFs(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, "/", user.FsConfig.SFTPConfig.Prefix)
 	assert.True(t, user.FsConfig.SFTPConfig.DisableCouncurrentReads)
+	assert.Equal(t, int64(2), user.FsConfig.SFTPConfig.BufferSize)
 	initialPwdPayload := user.FsConfig.SFTPConfig.Password.GetPayload()
 	initialPkeyPayload := user.FsConfig.SFTPConfig.PrivateKey.GetPayload()
 	assert.Equal(t, kms.SecretStatusSecretBox, user.FsConfig.SFTPConfig.Password.GetStatus())
@@ -6079,6 +6094,7 @@ func TestWebUserSFTPFsMock(t *testing.T) {
 	user.FsConfig.SFTPConfig.Fingerprints = []string{sftpPkeyFingerprint}
 	user.FsConfig.SFTPConfig.Prefix = "/home/sftpuser"
 	user.FsConfig.SFTPConfig.DisableCouncurrentReads = true
+	user.FsConfig.SFTPConfig.BufferSize = 5
 	form := make(url.Values)
 	form.Set(csrfFormToken, csrfToken)
 	form.Set("username", user.Username)
@@ -6116,6 +6132,7 @@ func TestWebUserSFTPFsMock(t *testing.T) {
 	form.Set("sftp_fingerprints", user.FsConfig.SFTPConfig.Fingerprints[0])
 	form.Set("sftp_prefix", user.FsConfig.SFTPConfig.Prefix)
 	form.Set("sftp_disable_concurrent_reads", "true")
+	form.Set("sftp_buffer_size", strconv.FormatInt(user.FsConfig.SFTPConfig.BufferSize, 10))
 	b, contentType, _ = getMultipartFormData(form, "", "")
 	req, _ = http.NewRequest(http.MethodPost, path.Join(webUserPath, user.Username), &b)
 	setJWTCookieForReq(req, webToken)
@@ -6144,6 +6161,7 @@ func TestWebUserSFTPFsMock(t *testing.T) {
 	assert.Equal(t, updateUser.FsConfig.SFTPConfig.Endpoint, user.FsConfig.SFTPConfig.Endpoint)
 	assert.True(t, updateUser.FsConfig.SFTPConfig.DisableCouncurrentReads)
 	assert.Len(t, updateUser.FsConfig.SFTPConfig.Fingerprints, 1)
+	assert.Equal(t, user.FsConfig.SFTPConfig.BufferSize, updateUser.FsConfig.SFTPConfig.BufferSize)
 	assert.Contains(t, updateUser.FsConfig.SFTPConfig.Fingerprints, sftpPkeyFingerprint)
 	// now check that a redacted credentials are not saved
 	form.Set("sftp_password", redactedSecret+" ")

httpd/schema/openapi.yaml

@@ -1613,6 +1613,9 @@ components:
         disable_concurrent_reads:
           type: boolean
           description: Concurrent reads are safe to use and disabling them will degrade performance. Some servers automatically delete files once they are downloaded. Using concurrent reads is problematic with such servers.
+        buffer_size:
+          type: intger
+          description: The size of the buffer (in MB) to use for transfers. By enabling buffering, the reads and writes, from/to the remote SFTP server, are split in multiple concurrent requests and this allows data to be transferred at a faster rate, over high latency networks, by overlapping round-trip times. With buffering enabled, resuming uploads is not supported and a file cannot be opened for both reading and writing at the same time. 0 means disabled.
     FilesystemConfig:
       type: object
       properties:

httpd/web.go

@@ -727,7 +727,8 @@ func getGCSConfig(r *http.Request) (vfs.GCSFsConfig, error) {
 	return config, err
 }
 
-func getSFTPConfig(r *http.Request) vfs.SFTPFsConfig {
+func getSFTPConfig(r *http.Request) (vfs.SFTPFsConfig, error) {
+	var err error
 	config := vfs.SFTPFsConfig{}
 	config.Endpoint = r.Form.Get("sftp_endpoint")
 	config.Username = r.Form.Get("sftp_username")
@@ -737,7 +738,8 @@ func getSFTPConfig(r *http.Request) vfs.SFTPFsConfig {
 	config.Fingerprints = getSliceFromDelimitedValues(fingerprintsFormValue, "\n")
 	config.Prefix = r.Form.Get("sftp_prefix")
 	config.DisableCouncurrentReads = len(r.Form.Get("sftp_disable_concurrent_reads")) > 0
-	return config
+	config.BufferSize, err = strconv.ParseInt(r.Form.Get("sftp_buffer_size"), 10, 64)
+	return config, err
 }
 
 func getAzureConfig(r *http.Request) (vfs.AzBlobFsConfig, error) {
@@ -788,7 +790,11 @@ func getFsConfigFromPostFields(r *http.Request) (vfs.Filesystem, error) {
 	case vfs.CryptedFilesystemProvider:
 		fs.CryptConfig.Passphrase = getSecretFromFormField(r, "crypt_passphrase")
 	case vfs.SFTPFilesystemProvider:
-		fs.SFTPConfig = getSFTPConfig(r)
+		config, err := getSFTPConfig(r)
+		if err != nil {
+			return fs, err
+		}
+		fs.SFTPConfig = config
 	}
 	return fs, nil
 }