WebDAV: allow to set last modification time

This commit add a minimal dead properties implementation

Fixes #1018

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>

docs/webdav.md

@@ -29,6 +29,8 @@ Know issues:
 - if a file or a directory cannot be accessed, for example due to OS permissions issues or because a mapped path for a virtual folder is a missing, it will be omitted from the directory listing. If there is a different error then the whole directory listing will fail. This behavior is different from SFTP/FTP where you will be able to see the problematic file/directory in the directory listing, you will only get an error if you try to access it
 - if you use the native Windows client please check its usage and pay particular attention to the [registry settings](https://docs.microsoft.com/en-us/iis/publish/using-webdav/using-the-webdav-redirector#webdav-redirector-registry-settings). The default file size limit is 50MB and if you don't configure SFTPGo to use HTTPS you have to set `BasicAuthLevel` to `2`
 
-We plan to add [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3) support in future releases. We need a design decision here, probably the best solution is to store dead properties inside the data provider but this could increase a lot its size. Alternately we could store them on disk for local filesystem and add as metadata for Cloud Storage, this means that we need to do a separate `HEAD` request to retrieve dead properties for an S3 file. For big folders will do a lot of requests to the Cloud Provider, I don't like this solution. Another option is to expose a hook and allow you to implement `dead properties` outside SFTPGo.
+SFTPGo has a minimal implementation for [Dead Properties](https://tools.ietf.org/html/rfc4918#section-3). We support setting the last modification time and we return the value in the "live" properties, so basically we don't store anything.
+
+To properly support dead properties we need a design decision, probably the best solution is to write a plugin and store them inside a supported data provider.
 
 If you find any other quirks or problems please let us know opening a GitHub issue, thank you!

go.mod

@@ -5,7 +5,7 @@ go 1.19
 require (
 	cloud.google.com/go/storage v1.27.0
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.4
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1
 	github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962
 	github.com/alexedwards/argon2id v0.0.0-20211130144151-3585854a6387
 	github.com/aws/aws-sdk-go-v2 v1.16.16
@@ -54,7 +54,7 @@ require (
 	github.com/sftpgo/sdk v0.1.2-0.20220913155952-81743fa5ded5
 	github.com/shirou/gopsutil/v3 v3.22.9
 	github.com/spf13/afero v1.9.2
-	github.com/spf13/cobra v1.5.0
+	github.com/spf13/cobra v1.6.0
 	github.com/spf13/viper v1.13.0
 	github.com/stretchr/testify v1.8.0
 	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
@@ -65,7 +65,7 @@ require (
 	go.etcd.io/bbolt v1.3.6
 	go.uber.org/automaxprocs v1.5.1
 	gocloud.dev v0.27.0
-	golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b
+	golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2
 	golang.org/x/net v0.0.0-20221004154528-8021a29435af
 	golang.org/x/oauth2 v0.0.0-20221006150949-b44042a4b9c1
 	golang.org/x/sys v0.0.0-20221010170243-090e33056c14
@@ -155,7 +155,7 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.3.8 // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
@@ -171,6 +171,6 @@ require (
 replace (
 	github.com/jlaffaye/ftp => github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9
 	github.com/pkg/sftp => github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7
-	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2
+	golang.org/x/crypto => github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0
-	golang.org/x/net => github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9
+	golang.org/x/net => github.com/drakkan/net v0.0.0-20221011170324-793589996ca2
 )

go.sum

@@ -109,8 +109,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 h1:XUNQ4mw+zJmaA2KXzP9JlQi
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
 github.com/Azure/azure-sdk-for-go/sdk/messaging/azservicebus v1.0.2/go.mod h1:LH9XQnMr2ZYxQdVdCrzLO9mxeDyrDFa6wbSI3x5zCZk=
 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0 h1:fe+kSd9btgTTeHeUlMTyEsjoe6L/zd+Q61iWEMPwHmc=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1 h1:BMTdr+ib5ljLa9MxTJK8x/Ds0MbBb4MfuW5BL0zMJnI=
-github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.0/go.mod h1:T7nxmZ9i42Dqy7kwnn8AZYNjqxd4TloKXdIbhosHSqo=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.5.1/go.mod h1:c6WvOhtmjNUWbLfOG1qxM/q0SPvQNSVJvolm+C52dIU=
 github.com/Azure/go-amqp v0.17.0/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg=
 github.com/Azure/go-amqp v0.17.5/go.mod h1:9YJ3RhxRT1gquYnzpZO1vcYMMpAdJT+QEg6fwmw9Zlg=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
@@ -535,12 +535,12 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD
 github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
-github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2 h1:VW04POuNbYg0uLKufFxgr9RnvvNKVc/Mhqq3EUx5tnE=
+github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0 h1:H8T0AOkrwrWacTEY8nP1PDQ+kUMeTQbCu8OY+x0/mXY=
-github.com/drakkan/crypto v0.0.0-20221007045301-0163d4cbe5a2/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro=
+github.com/drakkan/crypto v0.0.0-20221011170652-7c454d6a47a0/go.mod h1:SiM6ypd8Xu1xldObYtbDztuUU7xUzMnUULfphXFZmro=
 github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9 h1:LPH1dEblAOO/LoG7yHPMtBLXhQmjaga91/DDjWk9jWA=
 github.com/drakkan/ftp v0.0.0-20201114075148-9b9adce499a9/go.mod h1:2lmrmq866uF2tnje75wQHzmPXhmSWUt7Gyx2vgK1RCU=
-github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9 h1:LuHrwSqNqg+MNxDUgg+5uSoxADH1z/PMU3zKM4V/bqM=
+github.com/drakkan/net v0.0.0-20221011170324-793589996ca2 h1:nFZmCADOhW2YZ51/IKcODGsJsg3cX1VGVgjFDQYuVKs=
-github.com/drakkan/net v0.0.0-20221007045029-3c39cb455af9/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+github.com/drakkan/net v0.0.0-20221011170324-793589996ca2/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7 h1:Hj7AAfZ5yt9QuCxSQDllRygmL33xJ2sZLOmcyyOAdYU=
 github.com/drakkan/sftp v0.0.0-20220930161944-e8c89afc13a7/go.mod h1:wHDZ0IZX6JcBYRK1TH9bcVq8G7TLpVHYIGJRFnmPfxg=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -1490,8 +1490,8 @@ github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKv
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
 github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
+github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
@@ -1931,8 +1931,9 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

internal/dataprovider/mysql.go

@@ -220,6 +220,7 @@ func initializeMySQLProvider() error {
 			dbHandle.SetMaxIdleConns(2)
 		}
 		dbHandle.SetConnMaxLifetime(240 * time.Second)
+		dbHandle.SetConnMaxIdleTime(120 * time.Second)
 		provider = &MySQLProvider{dbHandle: dbHandle}
 	} else {
 		providerLog(logger.LevelError, "error creating mysql database handler, connection string: %#v, error: %v",

internal/dataprovider/pgsql.go

@@ -226,6 +226,7 @@ func initializePGSQLProvider() error {
 			dbHandle.SetMaxIdleConns(2)
 		}
 		dbHandle.SetConnMaxLifetime(240 * time.Second)
+		dbHandle.SetConnMaxIdleTime(120 * time.Second)
 		provider = &PGSQLProvider{dbHandle: dbHandle}
 	} else {
 		providerLog(logger.LevelError, "error creating postgres database handler, connection string: %#v, error: %v",

internal/webdavd/file.go

@@ -16,9 +16,11 @@ package webdavd
 
 import (
 	"context"
+	"encoding/xml"
 	"errors"
 	"io"
 	"mime"
+	"net/http"
 	"os"
 	"path"
 	"sync/atomic"
@@ -30,10 +32,14 @@ import (
 	"github.com/drakkan/sftpgo/v2/internal/common"
 	"github.com/drakkan/sftpgo/v2/internal/dataprovider"
 	"github.com/drakkan/sftpgo/v2/internal/logger"
+	"github.com/drakkan/sftpgo/v2/internal/util"
 	"github.com/drakkan/sftpgo/v2/internal/vfs"
 )
 
-var errTransferAborted = errors.New("transfer aborted")
+var (
+	errTransferAborted = errors.New("transfer aborted")
+	lastModifiedProps  = []string{"Win32LastModifiedTime", "getlastmodified"}
+)
 
 type webDavFile struct {
 	*common.BaseTransfer
@@ -374,3 +380,53 @@ func (f *webDavFile) isTransfer() bool {
 	}
 	return true
 }
+
+// DeadProps returns a copy of the dead properties held.
+// We always return nil for now, we only support the last modification time
+// and it is already included in "live" properties
+func (f *webDavFile) DeadProps() (map[xml.Name]webdav.Property, error) {
+	return nil, nil
+}
+
+// Patch patches the dead properties held.
+// In our minimal implementation we just support Win32LastModifiedTime and
+// getlastmodified to set the the modification time.
+// We ignore any other property and just return an OK response if the patch sets
+// the modification time, otherwise a Forbidden response
+func (f *webDavFile) Patch(patches []webdav.Proppatch) ([]webdav.Propstat, error) {
+	resp := make([]webdav.Propstat, 0, len(patches))
+	hasError := false
+	for _, patch := range patches {
+		status := http.StatusForbidden
+		pstat := webdav.Propstat{}
+		for _, p := range patch.Props {
+			if status == http.StatusForbidden && !hasError {
+				if !patch.Remove && util.Contains(lastModifiedProps, p.XMLName.Local) {
+					parsed, err := http.ParseTime(string(p.InnerXML))
+					if err != nil {
+						f.Connection.Log(logger.LevelWarn, "unsupported last modification time: %q, err: %v",
+							string(p.InnerXML), err)
+						hasError = true
+						continue
+					}
+					attrs := &common.StatAttributes{
+						Flags: common.StatAttrTimes,
+						Atime: parsed,
+						Mtime: parsed,
+					}
+					if err := f.Connection.SetStat(f.GetVirtualPath(), attrs); err != nil {
+						f.Connection.Log(logger.LevelWarn, "unable to set modification time for %q, err :%v",
+							f.GetVirtualPath(), err)
+						hasError = true
+						continue
+					}
+					status = http.StatusOK
+				}
+			}
+			pstat.Props = append(pstat.Props, webdav.Property{XMLName: p.XMLName})
+		}
+		pstat.Status = status
+		resp = append(resp, pstat)
+	}
+	return resp, nil
+}

internal/webdavd/internal_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
@@ -663,10 +664,46 @@ func TestFileAccessErrors(t *testing.T) {
 			assert.NoError(t, err)
 			assert.Equal(t, 0, len(connection.GetTransfers()))
 		}
-	}
+		// test PROPPATCH date parsing error
+		pstats, err := transfer.Patch([]webdav.Proppatch{
+			{
+				Props: []webdav.Property{
+					{
+						XMLName: xml.Name{
+							Space: "DAV",
+							Local: "getlastmodified",
+						},
+						InnerXML: []byte(`Wid, 04 Nov 2020 13:25:51 GMT`),
+					},
+				},
+			},
+		})
+		assert.NoError(t, err)
+		for _, pstat := range pstats {
+			assert.Equal(t, http.StatusForbidden, pstat.Status)
+		}
 
-	err = os.Remove(f.Name())
+		err = os.Remove(f.Name())
-	assert.NoError(t, err)
+		assert.NoError(t, err)
+		// the file is deleted PROPPATCH should fail
+		pstats, err = transfer.Patch([]webdav.Proppatch{
+			{
+				Props: []webdav.Property{
+					{
+						XMLName: xml.Name{
+							Space: "DAV",
+							Local: "getlastmodified",
+						},
+						InnerXML: []byte(`Wed, 04 Nov 2020 13:25:51 GMT`),
+					},
+				},
+			},
+		})
+		assert.NoError(t, err)
+		for _, pstat := range pstats {
+			assert.Equal(t, http.StatusForbidden, pstat.Status)
+		}
+	}
 }
 
 func TestRemoveDirTree(t *testing.T) {

internal/webdavd/webdavd_test.go

@@ -895,13 +895,26 @@ func TestPropPatch(t *testing.T) {
 		req.SetBasicAuth(u.Username, u.Password)
 		resp, err := httpClient.Do(req)
 		assert.NoError(t, err)
-		assert.Equal(t, 207, resp.StatusCode)
+		assert.Equal(t, http.StatusMultiStatus, resp.StatusCode)
 		err = resp.Body.Close()
 		assert.NoError(t, err)
 		info, err := client.Stat(testFileName)
 		if assert.NoError(t, err) {
+			expected, err := http.ParseTime("Wed, 04 Nov 2020 13:25:51 GMT")
+			assert.NoError(t, err)
 			assert.Equal(t, testFileSize, info.Size())
+			assert.Equal(t, expected.Format(http.TimeFormat), info.ModTime().Format(http.TimeFormat))
 		}
+		// wrong date
+		propatchBody = `<?xml version="1.0" encoding="utf-8" ?><D:propertyupdate xmlns:D="DAV:" xmlns:Z="urn:schemas-microsoft-com:"><D:set><D:prop><Z:Win32CreationTime>Wed, 04 Nov 2020 13:25:51 GMT</Z:Win32CreationTime><Z:Win32LastAccessTime>Sat, 05 Dec 2020 21:16:12 GMT</Z:Win32LastAccessTime><Z:Win32LastModifiedTime>Wid, 04 Nov 2020 13:25:51 GMT</Z:Win32LastModifiedTime><Z:Win32FileAttributes>00000000</Z:Win32FileAttributes></D:prop></D:set></D:propertyupdate>`
+		req, err = http.NewRequest("PROPPATCH", fmt.Sprintf("http://%v/%v", webDavServerAddr, testFileName), bytes.NewReader([]byte(propatchBody)))
+		assert.NoError(t, err)
+		req.SetBasicAuth(u.Username, u.Password)
+		resp, err = httpClient.Do(req)
+		assert.NoError(t, err)
+		assert.Equal(t, http.StatusMultiStatus, resp.StatusCode)
+		err = resp.Body.Close()
+		assert.NoError(t, err)
 		err = os.Remove(testFilePath)
 		assert.NoError(t, err)
 		_, err = httpdtest.RemoveUser(user, http.StatusOK)