From 6cebc037a09cc760428dd324b950cb07a631028a Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sat, 31 Dec 2022 16:41:32 +0100 Subject: [PATCH] eventmanager: check disk quota before executing the compress action Signed-off-by: Nicola Murino --- go.mod | 2 +- go.sum | 5 +- internal/common/eventmanager.go | 53 ++++++++- internal/common/eventmanager_test.go | 37 ++++++ internal/common/protocol_test.go | 168 +++++++++++++++++++++++++++ 5 files changed, 259 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2151c7d0..e24be4c0 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.29.6 github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.17.0 github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 - github.com/cockroachdb/cockroach-go/v2 v2.2.19 + github.com/cockroachdb/cockroach-go/v2 v2.2.20 github.com/coreos/go-oidc/v3 v3.4.0 github.com/drakkan/webdav v0.0.0-20221101181759-17ed21f9337b github.com/eikenb/pipeat v0.0.0-20210730190139-06b3e6902001 diff --git a/go.sum b/go.sum index 81c38166..82e431a4 100644 --- a/go.sum +++ b/go.sum @@ -360,8 +360,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20220314180256-7f1daf1720fc/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= -github.com/cockroachdb/cockroach-go/v2 v2.2.19 h1:YIHyz17jZumBeXPuoZKq/0nrITsqDoDD8/KQt3/xiyc= -github.com/cockroachdb/cockroach-go/v2 v2.2.19/go.mod h1:mzlIDDBALQfEjv/7DU12fb2AfQ/MUYTlychcMpWp9QI= +github.com/cockroachdb/cockroach-go/v2 v2.2.20 h1:TLSzwdTdIwgsbdApHzaxunhSMrmbGf5YY6oxtaP2kvw= +github.com/cockroachdb/cockroach-go/v2 v2.2.20/go.mod h1:73vQi5H/H7kE8SgOt+XA6729Tubvj5hxKIEgbQQhp4c= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= @@ -1337,7 +1337,6 @@ github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqgg github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/common/eventmanager.go b/internal/common/eventmanager.go index 800106e0..9d217609 100644 --- a/internal/common/eventmanager.go +++ b/internal/common/eventmanager.go @@ -1615,6 +1615,50 @@ func getArchiveBaseDir(paths []string) string { return baseDir } +func getSizeForPath(conn *BaseConnection, p string, info os.FileInfo) (int64, error) { + if info.IsDir() { + var dirSize int64 + entries, err := conn.ListDir(p) + if err != nil { + return 0, err + } + for _, entry := range entries { + size, err := getSizeForPath(conn, path.Join(p, entry.Name()), entry) + if err != nil { + return 0, err + } + dirSize += size + } + return dirSize, nil + } + if info.Mode().IsRegular() { + return info.Size(), nil + } + return 0, nil +} + +func estimateZipSize(conn *BaseConnection, zipPath string, paths []string) (int64, error) { + q, _ := conn.HasSpace(false, false, zipPath) + if q.HasSpace && q.GetRemainingSize() > 0 { + var size int64 + for _, item := range paths { + info, err := conn.DoStat(item, 1, false) + if err != nil { + return size, err + } + itemSize, err := getSizeForPath(conn, item, info) + if err != nil { + return size, err + } + size += itemSize + } + eventManagerLog(logger.LevelDebug, "archive paths %v, archive name %q, size: %d", paths, zipPath, size) + // we assume the zip size will be half of the real size + return size / 2, nil + } + return -1, nil +} + func executeCompressFsActionForUser(c dataprovider.EventActionFsCompress, replacer *strings.Replacer, user dataprovider.User, ) error { @@ -1638,14 +1682,19 @@ func executeCompressFsActionForUser(c dataprovider.EventActionFsCompress, replac } paths = append(paths, p) } - writer, numFiles, truncatedSize, cancelFn, err := getFileWriter(conn, name, -1) + paths = util.RemoveDuplicates(paths, false) + estimatedSize, err := estimateZipSize(conn, name, paths) + if err != nil { + eventManagerLog(logger.LevelError, "unable to estimate size for archive %q: %v", name, err) + return fmt.Errorf("unable to estimate archive size: %w", err) + } + writer, numFiles, truncatedSize, cancelFn, err := getFileWriter(conn, name, estimatedSize) if err != nil { eventManagerLog(logger.LevelError, "unable to create archive %q: %v", name, err) return fmt.Errorf("unable to create archive: %w", err) } defer cancelFn() - paths = util.RemoveDuplicates(paths, false) baseDir := getArchiveBaseDir(paths) eventManagerLog(logger.LevelDebug, "creating archive %q for paths %+v", name, paths) diff --git a/internal/common/eventmanager_test.go b/internal/common/eventmanager_test.go index 8e7ff4cc..b096e055 100644 --- a/internal/common/eventmanager_test.go +++ b/internal/common/eventmanager_test.go @@ -1720,6 +1720,43 @@ func TestReplacePathsPlaceholders(t *testing.T) { assert.Equal(t, []string{"/path1", "/path2"}, paths) } +func TestEstimateZipSizeErrors(t *testing.T) { + u := dataprovider.User{ + BaseUser: sdk.BaseUser{ + Username: "u", + HomeDir: filepath.Join(os.TempDir(), "u"), + Status: 1, + Permissions: map[string][]string{ + "/": {dataprovider.PermAny}, + }, + QuotaSize: 1000, + }, + } + err := dataprovider.AddUser(&u, "", "", "") + assert.NoError(t, err) + err = os.MkdirAll(u.GetHomeDir(), os.ModePerm) + assert.NoError(t, err) + conn := NewBaseConnection("", ProtocolFTP, "", "", u) + _, err = getSizeForPath(conn, "/missing", vfs.NewFileInfo("missing", true, 0, time.Now(), false)) + assert.True(t, conn.IsNotExistError(err)) + if runtime.GOOS != osWindows { + err = os.MkdirAll(filepath.Join(u.HomeDir, "d1", "d2", "sub"), os.ModePerm) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(u.HomeDir, "d1", "d2", "sub", "file.txt"), []byte("data"), 0666) + assert.NoError(t, err) + err = os.Chmod(filepath.Join(u.HomeDir, "d1", "d2"), 0001) + assert.NoError(t, err) + size, err := estimateZipSize(conn, "/archive.zip", []string{"/d1"}) + assert.Error(t, err, "size %d", size) + err = os.Chmod(filepath.Join(u.HomeDir, "d1", "d2"), os.ModePerm) + assert.NoError(t, err) + } + err = dataprovider.DeleteUser(u.Username, "", "", "") + assert.NoError(t, err) + err = os.RemoveAll(u.GetHomeDir()) + assert.NoError(t, err) +} + func getErrorString(err error) string { if err == nil { return "" diff --git a/internal/common/protocol_test.go b/internal/common/protocol_test.go index 2ed28c4e..0f8d26cd 100644 --- a/internal/common/protocol_test.go +++ b/internal/common/protocol_test.go @@ -4714,6 +4714,174 @@ func TestEventActionCompress(t *testing.T) { assert.NoError(t, err) } +func TestEventActionCompressQuotaErrors(t *testing.T) { + smtpCfg := smtp.Config{ + Host: "127.0.0.1", + Port: 2525, + From: "notify@example.com", + TemplatesPath: "templates", + } + err := smtpCfg.Initialize(configDir) + require.NoError(t, err) + + testDir := "archiveDir" + zipPath := "/archive.zip" + a1 := dataprovider.BaseEventAction{ + Name: "action1", + Type: dataprovider.ActionTypeFilesystem, + Options: dataprovider.BaseEventActionOptions{ + FsConfig: dataprovider.EventActionFilesystemConfig{ + Type: dataprovider.FilesystemActionCompress, + Compress: dataprovider.EventActionFsCompress{ + Name: zipPath, + Paths: []string{"/" + testDir}, + }, + }, + }, + } + action1, _, err := httpdtest.AddEventAction(a1, http.StatusCreated) + assert.NoError(t, err) + a2 := dataprovider.BaseEventAction{ + Name: "action2", + Type: dataprovider.ActionTypeEmail, + Options: dataprovider.BaseEventActionOptions{ + EmailConfig: dataprovider.EventActionEmailConfig{ + Recipients: []string{"test@example.com"}, + Subject: `"Compress failed"`, + Body: "Error: {{ErrorString}}", + }, + }, + } + action2, _, err := httpdtest.AddEventAction(a2, http.StatusCreated) + assert.NoError(t, err) + r1 := dataprovider.EventRule{ + Name: "test compress", + Trigger: dataprovider.EventTriggerFsEvent, + Conditions: dataprovider.EventConditions{ + FsEvents: []string{"rename"}, + }, + Actions: []dataprovider.EventAction{ + { + BaseEventAction: dataprovider.BaseEventAction{ + Name: action1.Name, + }, + Order: 1, + }, + { + BaseEventAction: dataprovider.BaseEventAction{ + Name: action2.Name, + }, + Options: dataprovider.EventActionOptions{ + IsFailureAction: true, + }, + Order: 2, + }, + }, + } + rule1, _, err := httpdtest.AddEventRule(r1, http.StatusCreated) + assert.NoError(t, err) + fileSize := int64(100) + u := getTestUser() + u.QuotaSize = 10 * fileSize + user, _, err := httpdtest.AddUser(u, http.StatusCreated) + assert.NoError(t, err) + conn, client, err := getSftpClient(user) + if assert.NoError(t, err) { + defer conn.Close() + defer client.Close() + + err = client.MkdirAll(path.Join(testDir, "1", "1")) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, "1", testFileName), fileSize, client) + assert.NoError(t, err) + err = client.MkdirAll(path.Join(testDir, "2", "2")) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, "2", testFileName), fileSize, client) + assert.NoError(t, err) + err = client.Symlink(path.Join(testDir, "2", testFileName), path.Join(testDir, "2", testFileName+"_link")) + assert.NoError(t, err) + // trigger the compress action + err = client.Mkdir("a") + assert.NoError(t, err) + err = client.Rename("a", "b") + assert.NoError(t, err) + assert.Eventually(t, func() bool { + _, err := client.Stat(zipPath) + return err == nil + }, 3*time.Second, 100*time.Millisecond) + err = client.Remove(zipPath) + assert.NoError(t, err) + // add other 6 file, the compress action should fail with a quota error + err = writeSFTPFile(path.Join(testDir, "1", "1", testFileName), fileSize, client) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, "2", "2", testFileName), fileSize, client) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, "1", "1", testFileName+"1"), fileSize, client) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, "2", "2", testFileName+"2"), fileSize, client) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, "1", testFileName+"1"), fileSize, client) + assert.NoError(t, err) + err = writeSFTPFile(path.Join(testDir, "2", testFileName+"2"), fileSize, client) + assert.NoError(t, err) + lastReceivedEmail.reset() + err = client.Rename("b", "a") + assert.NoError(t, err) + assert.Eventually(t, func() bool { + return lastReceivedEmail.get().From != "" + }, 3*time.Second, 100*time.Millisecond) + email := lastReceivedEmail.get() + assert.Len(t, email.To, 1) + assert.True(t, util.Contains(email.To, "test@example.com")) + assert.Contains(t, email.Data, `Subject: "Compress failed"`) + assert.Contains(t, email.Data, common.ErrQuotaExceeded.Error()) + // update quota size so the user is already overquota + user.QuotaSize = 7 * fileSize + _, _, err = httpdtest.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + lastReceivedEmail.reset() + err = client.Rename("a", "b") + assert.NoError(t, err) + assert.Eventually(t, func() bool { + return lastReceivedEmail.get().From != "" + }, 3*time.Second, 100*time.Millisecond) + email = lastReceivedEmail.get() + assert.Len(t, email.To, 1) + assert.True(t, util.Contains(email.To, "test@example.com")) + assert.Contains(t, email.Data, `Subject: "Compress failed"`) + assert.Contains(t, email.Data, common.ErrQuotaExceeded.Error()) + // remove the path to compress to trigger an error for size estimation + out, err := runSSHCommand(fmt.Sprintf("sftpgo-remove %s", testDir), user) + assert.NoError(t, err, string(out)) + lastReceivedEmail.reset() + err = client.Rename("b", "a") + assert.NoError(t, err) + assert.Eventually(t, func() bool { + return lastReceivedEmail.get().From != "" + }, 3*time.Second, 100*time.Millisecond) + email = lastReceivedEmail.get() + assert.Len(t, email.To, 1) + assert.True(t, util.Contains(email.To, "test@example.com")) + assert.Contains(t, email.Data, `Subject: "Compress failed"`) + assert.Contains(t, email.Data, "unable to estimate archive size") + } + + _, err = httpdtest.RemoveEventRule(rule1, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveEventAction(action1, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveEventAction(action2, http.StatusOK) + assert.NoError(t, err) + _, err = httpdtest.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) + err = os.RemoveAll(user.GetHomeDir()) + assert.NoError(t, err) + + smtpCfg = smtp.Config{} + err = smtpCfg.Initialize(configDir) + require.NoError(t, err) +} + func TestEventActionCompressQuotaFolder(t *testing.T) { testDir := "/folder" a1 := dataprovider.BaseEventAction{