From dccc583b5d20f48a59d28aa8a03aaa0fedc1b574 Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Sun, 22 Nov 2020 21:53:04 +0100 Subject: [PATCH] add a dedicated struct to store encrypted credentials also gcs credentials are now encrypted, both on disk and inside the provider. Data provider is automatically migrated and load data will accept old format too but you should upgrade to the new format to avoid future issues --- cmd/portable.go | 27 +- dataprovider/bolt.go | 142 ++++++--- dataprovider/compat.go | 222 ++++++++++++++ dataprovider/dataprovider.go | 122 +++++--- dataprovider/memory.go | 13 +- dataprovider/mysql.go | 52 +++- dataprovider/pgsql.go | 52 +++- dataprovider/sqlcommon.go | 90 +++++- dataprovider/sqlite.go | 52 +++- dataprovider/sqlqueries.go | 8 + dataprovider/user.go | 14 + docker/README.md | 13 +- docs/full-configuration.md | 2 +- examples/rest-api-cli/README.md | 5 +- examples/rest-api-cli/sftpgo_api_cli | 19 +- ftpd/ftpd_test.go | 17 +- httpd/api_maintenance.go | 3 +- httpd/api_user.go | 53 +++- httpd/api_utils.go | 31 +- httpd/httpd_test.go | 429 +++++++++++++++++++++++---- httpd/internal_test.go | 34 ++- httpd/schema/openapi.yaml | 31 +- httpd/web.go | 41 ++- service/service.go | 5 +- sftpd/sftpd_test.go | 17 +- templates/user.html | 6 +- utils/utils.go | 3 + vfs/azblobfs.go | 8 +- vfs/gcsfs.go | 26 +- vfs/s3fs.go | 7 +- vfs/secret.go | 209 +++++++++++++ vfs/vfs.go | 56 ++-- webdavd/webdavd_test.go | 17 +- 33 files changed, 1512 insertions(+), 314 deletions(-) create mode 100644 dataprovider/compat.go create mode 100644 vfs/secret.go diff --git a/cmd/portable.go b/cmd/portable.go index 2c585f8b..27b5559c 100644 --- a/cmd/portable.go +++ b/cmd/portable.go @@ -143,10 +143,13 @@ Please take a look at the usage below to customize the serving parameters`, FsConfig: dataprovider.Filesystem{ Provider: dataprovider.FilesystemProvider(portableFsProvider), S3Config: vfs.S3FsConfig{ - Bucket: portableS3Bucket, - Region: portableS3Region, - AccessKey: portableS3AccessKey, - AccessSecret: portableS3AccessSecret, + Bucket: portableS3Bucket, + Region: portableS3Region, + AccessKey: portableS3AccessKey, + AccessSecret: vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: portableS3AccessSecret, + }, Endpoint: portableS3Endpoint, StorageClass: portableS3StorageClass, KeyPrefix: portableS3KeyPrefix, @@ -154,16 +157,22 @@ Please take a look at the usage below to customize the serving parameters`, UploadConcurrency: portableS3ULConcurrency, }, GCSConfig: vfs.GCSFsConfig{ - Bucket: portableGCSBucket, - Credentials: portableGCSCredentials, + Bucket: portableGCSBucket, + Credentials: vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: string(portableGCSCredentials), + }, AutomaticCredentials: portableGCSAutoCredentials, StorageClass: portableGCSStorageClass, KeyPrefix: portableGCSKeyPrefix, }, AzBlobConfig: vfs.AzBlobFsConfig{ - Container: portableAzContainer, - AccountName: portableAzAccountName, - AccountKey: portableAzAccountKey, + Container: portableAzContainer, + AccountName: portableAzAccountName, + AccountKey: vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: portableAzAccountKey, + }, Endpoint: portableAzEndpoint, AccessTier: portableAzAccessTier, SASURL: portableAzSASURL, diff --git a/dataprovider/bolt.go b/dataprovider/bolt.go index 33b0e2dc..debda928 100644 --- a/dataprovider/bolt.go +++ b/dataprovider/bolt.go @@ -19,7 +19,7 @@ import ( ) const ( - boltDatabaseVersion = 4 + boltDatabaseVersion = 5 ) var ( @@ -35,28 +35,6 @@ type BoltProvider struct { dbHandle *bolt.DB } -type compatUserV2 struct { - ID int64 `json:"id"` - Username string `json:"username"` - Password string `json:"password,omitempty"` - PublicKeys []string `json:"public_keys,omitempty"` - HomeDir string `json:"home_dir"` - UID int `json:"uid"` - GID int `json:"gid"` - MaxSessions int `json:"max_sessions"` - QuotaSize int64 `json:"quota_size"` - QuotaFiles int `json:"quota_files"` - Permissions []string `json:"permissions"` - UsedQuotaSize int64 `json:"used_quota_size"` - UsedQuotaFiles int `json:"used_quota_files"` - LastQuotaUpdate int64 `json:"last_quota_update"` - UploadBandwidth int64 `json:"upload_bandwidth"` - DownloadBandwidth int64 `json:"download_bandwidth"` - ExpirationDate int64 `json:"expiration_date"` - LastLogin int64 `json:"last_login"` - Status int `json:"status"` -} - func init() { version.AddFeature("+bolt") } @@ -425,7 +403,8 @@ func (p BoltProvider) getUserWithUsername(username string) ([]User, error) { var user User user, err := p.userExists(username) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) return users, nil } if _, ok := err.(*RecordNotFoundError); ok { @@ -465,7 +444,8 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str } user, err := joinUserAndFolders(v, folderBucket) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) } if len(users) >= limit { break @@ -479,7 +459,8 @@ func (p BoltProvider) getUsers(limit int, offset int, order string, username str } user, err := joinUserAndFolders(v, folderBucket) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) } if len(users) >= limit { break @@ -718,28 +699,46 @@ func (p BoltProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updateDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updateDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateDatabaseFrom3To4(p.dbHandle) + return updateBoltDatabaseFromV1(p.dbHandle) case 2: - err = updateDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateDatabaseFrom3To4(p.dbHandle) + return updateBoltDatabaseFromV2(p.dbHandle) case 3: - return updateDatabaseFrom3To4(p.dbHandle) + return updateBoltDatabaseFromV3(p.dbHandle) + case 4: + return updateBoltDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updateBoltDatabaseFromV1(dbHandle *bolt.DB) error { + err := updateDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updateBoltDatabaseFromV2(dbHandle) +} + +func updateBoltDatabaseFromV2(dbHandle *bolt.DB) error { + err := updateDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updateBoltDatabaseFromV3(dbHandle) +} + +func updateBoltDatabaseFromV3(dbHandle *bolt.DB) error { + err := updateDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updateBoltDatabaseFromV4(dbHandle) +} + +func updateBoltDatabaseFromV4(dbHandle *bolt.DB) error { + return updateDatabaseFrom4To5(dbHandle) +} + // itob returns an 8-byte big endian representation of v. func itob(v int64) []byte { b := make([]byte, 8) @@ -847,6 +846,27 @@ func removeUserFromFolderMapping(folder vfs.VirtualFolder, user User, bucket *bo return err } +func updateV4BoltUser(dbHandle *bolt.DB, user User) error { + err := validateUser(&user) + if err != nil { + return err + } + return dbHandle.Update(func(tx *bolt.Tx) error { + bucket, _, err := getBuckets(tx) + if err != nil { + return err + } + if u := bucket.Get([]byte(user.Username)); u == nil { + return &RecordNotFoundError{err: fmt.Sprintf("username %v does not exist", user.Username)} + } + buf, err := json.Marshal(user) + if err != nil { + return err + } + return bucket.Put([]byte(user.Username), buf) + }) +} + func getBuckets(tx *bolt.Tx) (*bolt.Bucket, *bolt.Bucket, error) { var err error bucket := tx.Bucket(usersBucket) @@ -1007,6 +1027,46 @@ func updateDatabaseFrom3To4(dbHandle *bolt.DB) error { return err } +func updateDatabaseFrom4To5(dbHandle *bolt.DB) error { + logger.InfoToConsole("updating bolt database version: 4 -> 5") + providerLog(logger.LevelInfo, "updating bolt database version: 4 -> 5") + users := []User{} + err := dbHandle.View(func(tx *bolt.Tx) error { + bucket, _, err := getBuckets(tx) + if err != nil { + return err + } + cursor := bucket.Cursor() + for k, v := cursor.First(); k != nil; k, v = cursor.Next() { + var compatUser compatUserV4 + err = json.Unmarshal(v, &compatUser) + if err != nil { + logger.WarnToConsole("failed to unmarshal v4 user %#v, is it already migrated?", string(k)) + continue + } + fsConfig, err := convertFsConfigFromV4(compatUser.FsConfig, compatUser.Username) + if err != nil { + return err + } + users = append(users, createUserFromV4(compatUser, fsConfig)) + } + return nil + }) + if err != nil { + return err + } + + for _, user := range users { + err = updateV4BoltUser(dbHandle, user) + if err != nil { + return err + } + providerLog(logger.LevelInfo, "filesystem config updated for user %#v", user.Username) + } + + return updateBoltDatabaseVersion(dbHandle, 5) +} + func getBoltAvailableUsernames(dbHandle *bolt.DB) ([]string, error) { usernames := []string{} err := dbHandle.View(func(tx *bolt.Tx) error { diff --git a/dataprovider/compat.go b/dataprovider/compat.go new file mode 100644 index 00000000..1a31af50 --- /dev/null +++ b/dataprovider/compat.go @@ -0,0 +1,222 @@ +package dataprovider + +import ( + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/drakkan/sftpgo/logger" + "github.com/drakkan/sftpgo/vfs" +) + +type compatUserV2 struct { + ID int64 `json:"id"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + HomeDir string `json:"home_dir"` + UID int `json:"uid"` + GID int `json:"gid"` + MaxSessions int `json:"max_sessions"` + QuotaSize int64 `json:"quota_size"` + QuotaFiles int `json:"quota_files"` + Permissions []string `json:"permissions"` + UsedQuotaSize int64 `json:"used_quota_size"` + UsedQuotaFiles int `json:"used_quota_files"` + LastQuotaUpdate int64 `json:"last_quota_update"` + UploadBandwidth int64 `json:"upload_bandwidth"` + DownloadBandwidth int64 `json:"download_bandwidth"` + ExpirationDate int64 `json:"expiration_date"` + LastLogin int64 `json:"last_login"` + Status int `json:"status"` +} + +type compatS3FsConfigV4 struct { + Bucket string `json:"bucket,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + Region string `json:"region,omitempty"` + AccessKey string `json:"access_key,omitempty"` + AccessSecret string `json:"access_secret,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + StorageClass string `json:"storage_class,omitempty"` + UploadPartSize int64 `json:"upload_part_size,omitempty"` + UploadConcurrency int `json:"upload_concurrency,omitempty"` +} + +type compatGCSFsConfigV4 struct { + Bucket string `json:"bucket,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + CredentialFile string `json:"-"` + Credentials []byte `json:"credentials,omitempty"` + AutomaticCredentials int `json:"automatic_credentials,omitempty"` + StorageClass string `json:"storage_class,omitempty"` +} + +type compatAzBlobFsConfigV4 struct { + Container string `json:"container,omitempty"` + AccountName string `json:"account_name,omitempty"` + AccountKey string `json:"account_key,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + SASURL string `json:"sas_url,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + UploadPartSize int64 `json:"upload_part_size,omitempty"` + UploadConcurrency int `json:"upload_concurrency,omitempty"` + UseEmulator bool `json:"use_emulator,omitempty"` + AccessTier string `json:"access_tier,omitempty"` +} + +type compatFilesystemV4 struct { + Provider FilesystemProvider `json:"provider"` + S3Config compatS3FsConfigV4 `json:"s3config,omitempty"` + GCSConfig compatGCSFsConfigV4 `json:"gcsconfig,omitempty"` + AzBlobConfig compatAzBlobFsConfigV4 `json:"azblobconfig,omitempty"` +} + +type compatUserV4 struct { + ID int64 `json:"id"` + Status int `json:"status"` + Username string `json:"username"` + ExpirationDate int64 `json:"expiration_date"` + Password string `json:"password,omitempty"` + PublicKeys []string `json:"public_keys,omitempty"` + HomeDir string `json:"home_dir"` + VirtualFolders []vfs.VirtualFolder `json:"virtual_folders,omitempty"` + UID int `json:"uid"` + GID int `json:"gid"` + MaxSessions int `json:"max_sessions"` + QuotaSize int64 `json:"quota_size"` + QuotaFiles int `json:"quota_files"` + Permissions map[string][]string `json:"permissions"` + UsedQuotaSize int64 `json:"used_quota_size"` + UsedQuotaFiles int `json:"used_quota_files"` + LastQuotaUpdate int64 `json:"last_quota_update"` + UploadBandwidth int64 `json:"upload_bandwidth"` + DownloadBandwidth int64 `json:"download_bandwidth"` + LastLogin int64 `json:"last_login"` + Filters UserFilters `json:"filters"` + FsConfig compatFilesystemV4 `json:"filesystem"` +} + +type backupDataV4Compat struct { + Users []compatUserV4 `json:"users"` + Folders []vfs.BaseVirtualFolder `json:"folders"` +} + +func createUserFromV4(u compatUserV4, fsConfig Filesystem) User { + user := User{ + ID: u.ID, + Status: u.Status, + Username: u.Username, + ExpirationDate: u.ExpirationDate, + Password: u.Password, + PublicKeys: u.PublicKeys, + HomeDir: u.HomeDir, + VirtualFolders: u.VirtualFolders, + UID: u.UID, + GID: u.GID, + MaxSessions: u.MaxSessions, + QuotaSize: u.QuotaSize, + QuotaFiles: u.QuotaFiles, + Permissions: u.Permissions, + UsedQuotaSize: u.UsedQuotaSize, + UsedQuotaFiles: u.UsedQuotaFiles, + LastQuotaUpdate: u.LastQuotaUpdate, + UploadBandwidth: u.UploadBandwidth, + DownloadBandwidth: u.DownloadBandwidth, + LastLogin: u.LastLogin, + Filters: u.Filters, + } + user.FsConfig = fsConfig + return user +} + +func getCGSCredentialsFromV4(config compatGCSFsConfigV4) (vfs.Secret, error) { + var secret vfs.Secret + var err error + if len(config.Credentials) > 0 { + secret.Status = vfs.SecretStatusPlain + secret.Payload = string(config.Credentials) + return secret, nil + } + if config.CredentialFile != "" { + creds, err := ioutil.ReadFile(config.CredentialFile) + if err != nil { + return secret, err + } + secret.Status = vfs.SecretStatusPlain + secret.Payload = string(creds) + return secret, nil + } + return secret, err +} + +func convertFsConfigFromV4(compatFs compatFilesystemV4, username string) (Filesystem, error) { + fsConfig := Filesystem{ + Provider: compatFs.Provider, + S3Config: vfs.S3FsConfig{}, + AzBlobConfig: vfs.AzBlobFsConfig{}, + GCSConfig: vfs.GCSFsConfig{}, + } + switch compatFs.Provider { + case S3FilesystemProvider: + fsConfig.S3Config = vfs.S3FsConfig{ + Bucket: compatFs.S3Config.Bucket, + KeyPrefix: compatFs.S3Config.KeyPrefix, + Region: compatFs.S3Config.Region, + AccessKey: compatFs.S3Config.AccessKey, + AccessSecret: vfs.Secret{}, + Endpoint: compatFs.S3Config.Endpoint, + StorageClass: compatFs.S3Config.StorageClass, + UploadPartSize: compatFs.S3Config.UploadPartSize, + UploadConcurrency: compatFs.S3Config.UploadConcurrency, + } + if compatFs.S3Config.AccessSecret != "" { + secret, err := vfs.GetSecretFromCompatString(compatFs.S3Config.AccessSecret) + if err != nil { + providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err) + return fsConfig, err + } + fsConfig.S3Config.AccessSecret = secret + } + case AzureBlobFilesystemProvider: + fsConfig.AzBlobConfig = vfs.AzBlobFsConfig{ + Container: compatFs.AzBlobConfig.Container, + AccountName: compatFs.AzBlobConfig.AccountName, + AccountKey: vfs.Secret{}, + Endpoint: compatFs.AzBlobConfig.Endpoint, + SASURL: compatFs.AzBlobConfig.SASURL, + KeyPrefix: compatFs.AzBlobConfig.KeyPrefix, + UploadPartSize: compatFs.AzBlobConfig.UploadPartSize, + UploadConcurrency: compatFs.AzBlobConfig.UploadConcurrency, + UseEmulator: compatFs.AzBlobConfig.UseEmulator, + AccessTier: compatFs.AzBlobConfig.AccessTier, + } + if compatFs.AzBlobConfig.AccountKey != "" { + secret, err := vfs.GetSecretFromCompatString(compatFs.AzBlobConfig.AccountKey) + if err != nil { + providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err) + return fsConfig, err + } + fsConfig.AzBlobConfig.AccountKey = secret + } + case GCSFilesystemProvider: + fsConfig.GCSConfig = vfs.GCSFsConfig{ + Bucket: compatFs.GCSConfig.Bucket, + KeyPrefix: compatFs.GCSConfig.KeyPrefix, + CredentialFile: compatFs.GCSConfig.CredentialFile, + AutomaticCredentials: compatFs.GCSConfig.AutomaticCredentials, + StorageClass: compatFs.GCSConfig.StorageClass, + } + if compatFs.GCSConfig.AutomaticCredentials == 0 { + compatFs.GCSConfig.CredentialFile = filepath.Join(credentialsDirPath, fmt.Sprintf("%v_gcs_credentials.json", + username)) + } + secret, err := getCGSCredentialsFromV4(compatFs.GCSConfig) + if err != nil { + providerLog(logger.LevelError, "unable to convert v4 filesystem for user %#v: %v", username, err) + return fsConfig, err + } + fsConfig.GCSConfig.Credentials = secret + } + return fsConfig, nil +} diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index 3eda62b3..9dfd0e22 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -59,6 +59,9 @@ const ( BoltDataProviderName = "bolt" // MemoryDataProviderName name for memory provider MemoryDataProviderName = "memory" + // DumpVersion defines the version for the dump. + // For restore/load we support the current version and the previous one + DumpVersion = 5 argonPwdPrefix = "$argon2id$" bcryptPwdPrefix = "$2a$" @@ -265,6 +268,7 @@ type Config struct { type BackupData struct { Users []User `json:"users"` Folders []vfs.BaseVirtualFolder `json:"folders"` + Version int `json:"version"` } type keyboardAuthHookRequest struct { @@ -384,10 +388,8 @@ func Initialize(cnf Config, basePath string) error { if err = validateHooks(); err != nil { return err } - if !cnf.PreferDatabaseCredentials { - if err = validateCredentialsDir(basePath); err != nil { - return err - } + if err = validateCredentialsDir(basePath, cnf.PreferDatabaseCredentials); err != nil { + return err } err = createProvider(basePath) if err != nil { @@ -689,6 +691,7 @@ func GetFolders(limit, offset int, order, folderPath string) ([]vfs.BaseVirtualF // DumpData returns all users and folders func DumpData() (BackupData, error) { var data BackupData + data.Version = DumpVersion users, err := provider.dumpUsers() if err != nil { return data, err @@ -702,6 +705,33 @@ func DumpData() (BackupData, error) { return data, err } +// ParseDumpData tries to parse data as BackupData +func ParseDumpData(data []byte) (BackupData, error) { + var dump BackupData + err := json.Unmarshal(data, &dump) + if err == nil { + return dump, err + } + dump = BackupData{} + // try to parse as version 4 + var dumpCompat backupDataV4Compat + err = json.Unmarshal(data, &dumpCompat) + if err != nil { + return dump, err + } + logger.WarnToConsole("You are loading data from an old format, please update to the latest supported one. We only support the current and the previous format.") + providerLog(logger.LevelWarn, "You are loading data from an old format, please update to the latest supported one. We only support the current and the previous format.") + dump.Folders = dumpCompat.Folders + for _, compatUser := range dumpCompat.Users { + fsConfig, err := convertFsConfigFromV4(compatUser.FsConfig, compatUser.Username) + if err != nil { + return dump, err + } + dump.Users = append(dump.Users, createUserFromV4(compatUser, fsConfig)) + } + return dump, err +} + // GetProviderStatus returns an error if the provider is not available func GetProviderStatus() error { return provider.checkAvailability() @@ -1038,17 +1068,35 @@ func saveGCSCredentials(user *User) error { if user.FsConfig.Provider != GCSFilesystemProvider { return nil } - if len(user.FsConfig.GCSConfig.Credentials) == 0 { + if user.FsConfig.GCSConfig.Credentials.Payload == "" { return nil } if config.PreferDatabaseCredentials { + if user.FsConfig.GCSConfig.Credentials.IsPlain() { + user.FsConfig.GCSConfig.Credentials.AdditionalData = user.Username + err := user.FsConfig.GCSConfig.Credentials.Encrypt() + if err != nil { + return err + } + } return nil } - err := ioutil.WriteFile(user.getGCSCredentialsFilePath(), user.FsConfig.GCSConfig.Credentials, 0600) + if user.FsConfig.GCSConfig.Credentials.IsPlain() { + user.FsConfig.GCSConfig.Credentials.AdditionalData = user.Username + err := user.FsConfig.GCSConfig.Credentials.Encrypt() + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not encrypt GCS credentials: %v", err)} + } + } + creds, err := json.Marshal(user.FsConfig.GCSConfig.Credentials) + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not marshal GCS credentials: %v", err)} + } + err = ioutil.WriteFile(user.getGCSCredentialsFilePath(), creds, 0600) if err != nil { return &ValidationError{err: fmt.Sprintf("could not save GCS credentials: %v", err)} } - user.FsConfig.GCSConfig.Credentials = nil + user.FsConfig.GCSConfig.Credentials = vfs.Secret{} return nil } @@ -1058,38 +1106,38 @@ func validateFilesystemConfig(user *User) error { if err != nil { return &ValidationError{err: fmt.Sprintf("could not validate s3config: %v", err)} } - if user.FsConfig.S3Config.AccessSecret != "" { - vals := strings.Split(user.FsConfig.S3Config.AccessSecret, "$") - if !strings.HasPrefix(user.FsConfig.S3Config.AccessSecret, "$aes$") || len(vals) != 4 { - accessSecret, err := utils.EncryptData(user.FsConfig.S3Config.AccessSecret) - if err != nil { - return &ValidationError{err: fmt.Sprintf("could not encrypt s3 access secret: %v", err)} - } - user.FsConfig.S3Config.AccessSecret = accessSecret + if user.FsConfig.S3Config.AccessSecret.IsPlain() { + user.FsConfig.S3Config.AccessSecret.AdditionalData = user.Username + err = user.FsConfig.S3Config.AccessSecret.Encrypt() + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not encrypt s3 access secret: %v", err)} } } + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} + user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} return nil } else if user.FsConfig.Provider == GCSFilesystemProvider { err := vfs.ValidateGCSFsConfig(&user.FsConfig.GCSConfig, user.getGCSCredentialsFilePath()) if err != nil { return &ValidationError{err: fmt.Sprintf("could not validate GCS config: %v", err)} } + user.FsConfig.S3Config = vfs.S3FsConfig{} + user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} return nil } else if user.FsConfig.Provider == AzureBlobFilesystemProvider { err := vfs.ValidateAzBlobFsConfig(&user.FsConfig.AzBlobConfig) if err != nil { return &ValidationError{err: fmt.Sprintf("could not validate Azure Blob config: %v", err)} } - if user.FsConfig.AzBlobConfig.AccountKey != "" { - vals := strings.Split(user.FsConfig.AzBlobConfig.AccountKey, "$") - if !strings.HasPrefix(user.FsConfig.AzBlobConfig.AccountKey, "$aes$") || len(vals) != 4 { - accountKey, err := utils.EncryptData(user.FsConfig.AzBlobConfig.AccountKey) - if err != nil { - return &ValidationError{err: fmt.Sprintf("could not encrypt Azure blob account key: %v", err)} - } - user.FsConfig.AzBlobConfig.AccountKey = accountKey + if user.FsConfig.AzBlobConfig.AccountKey.IsPlain() { + user.FsConfig.AzBlobConfig.AccountKey.AdditionalData = user.Username + err = user.FsConfig.AzBlobConfig.AccountKey.Encrypt() + if err != nil { + return &ValidationError{err: fmt.Sprintf("could not encrypt Azure blob account key: %v", err)} } } + user.FsConfig.S3Config = vfs.S3FsConfig{} + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} return nil } user.FsConfig.Provider = LocalFilesystemProvider @@ -1321,19 +1369,6 @@ func comparePbkdf2PasswordAndHash(password, hashedPassword string) (bool, error) return subtle.ConstantTimeCompare(df, expected) == 1, nil } -// HideUserSensitiveData hides user sensitive data -func HideUserSensitiveData(user *User) User { - user.Password = "" - if user.FsConfig.Provider == S3FilesystemProvider { - user.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(user.FsConfig.S3Config.AccessSecret) - } else if user.FsConfig.Provider == GCSFilesystemProvider { - user.FsConfig.GCSConfig.Credentials = nil - } else if user.FsConfig.Provider == AzureBlobFilesystemProvider { - user.FsConfig.AzBlobConfig.AccountKey = utils.RemoveDecryptionKey(user.FsConfig.AzBlobConfig.AccountKey) - } - return *user -} - func addCredentialsToUser(user *User) error { if user.FsConfig.Provider != GCSFilesystemProvider { return nil @@ -1343,7 +1378,7 @@ func addCredentialsToUser(user *User) error { } // Don't read from file if credentials have already been set - if len(user.FsConfig.GCSConfig.Credentials) > 0 { + if user.FsConfig.GCSConfig.Credentials.IsValid() { return nil } @@ -1351,8 +1386,7 @@ func addCredentialsToUser(user *User) error { if err != nil { return err } - user.FsConfig.GCSConfig.Credentials = cred - return nil + return json.Unmarshal(cred, &user.FsConfig.GCSConfig.Credentials) } func getSSLMode() string { @@ -1396,12 +1430,18 @@ func startAvailabilityTimer() { }() } -func validateCredentialsDir(basePath string) error { +func validateCredentialsDir(basePath string, preferDbCredentials bool) error { if filepath.IsAbs(config.CredentialsPath) { credentialsDirPath = config.CredentialsPath } else { credentialsDirPath = filepath.Join(basePath, config.CredentialsPath) } + // if we want to store credentials inside the database just stop here + // we just populate credentialsDirPath to be able to use existing users + // with credential files + if preferDbCredentials { + return nil + } fi, err := os.Stat(credentialsDirPath) if err == nil { if !fi.IsDir() { @@ -2013,7 +2053,7 @@ func executeAction(operation string, user User) { q := url.Query() q.Add("action", operation) url.RawQuery = q.Encode() - HideUserSensitiveData(&user) + user.HideConfidentialData() userAsJSON, err := json.Marshal(user) if err != nil { return diff --git a/dataprovider/memory.go b/dataprovider/memory.go index 1ad36a91..923f36bb 100644 --- a/dataprovider/memory.go +++ b/dataprovider/memory.go @@ -1,7 +1,6 @@ package dataprovider import ( - "encoding/json" "errors" "fmt" "io/ioutil" @@ -300,7 +299,8 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s if offset == 0 { user, err := p.userExistsInternal(username) if err == nil { - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) } } return users, err @@ -313,7 +313,8 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s continue } user := p.dbHandle.users[username] - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) if len(users) >= limit { break } @@ -326,7 +327,8 @@ func (p MemoryProvider) getUsers(limit int, offset int, order string, username s } username := p.dbHandle.usernames[i] user := p.dbHandle.users[username] - users = append(users, HideUserSensitiveData(&user)) + user.HideConfidentialData() + users = append(users, user) if len(users) >= limit { break } @@ -624,8 +626,7 @@ func (p MemoryProvider) reloadConfig() error { providerLog(logger.LevelWarn, "error loading users: %v", err) return err } - var dump BackupData - err = json.Unmarshal(content, &dump) + dump, err := ParseDumpData(content) if err != nil { providerLog(logger.LevelWarn, "error loading users: %v", err) return err diff --git a/dataprovider/mysql.go b/dataprovider/mysql.go index c5f11d3c..93d2d741 100644 --- a/dataprovider/mysql.go +++ b/dataprovider/mysql.go @@ -210,28 +210,46 @@ func (p MySQLProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updateMySQLDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updateMySQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateMySQLDatabaseFrom3To4(p.dbHandle) + return updateMySQLDatabaseFromV1(p.dbHandle) case 2: - err = updateMySQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateMySQLDatabaseFrom3To4(p.dbHandle) + return updateMySQLDatabaseFromV2(p.dbHandle) case 3: - return updateMySQLDatabaseFrom3To4(p.dbHandle) + return updateMySQLDatabaseFromV3(p.dbHandle) + case 4: + return updateMySQLDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updateMySQLDatabaseFromV1(dbHandle *sql.DB) error { + err := updateMySQLDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updateMySQLDatabaseFromV2(dbHandle) +} + +func updateMySQLDatabaseFromV2(dbHandle *sql.DB) error { + err := updateMySQLDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updateMySQLDatabaseFromV3(dbHandle) +} + +func updateMySQLDatabaseFromV3(dbHandle *sql.DB) error { + err := updateMySQLDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updateMySQLDatabaseFromV4(dbHandle) +} + +func updateMySQLDatabaseFromV4(dbHandle *sql.DB) error { + return updateMySQLDatabaseFrom4To5(dbHandle) +} + func updateMySQLDatabaseFrom1To2(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") @@ -249,3 +267,7 @@ func updateMySQLDatabaseFrom2To3(dbHandle *sql.DB) error { func updateMySQLDatabaseFrom3To4(dbHandle *sql.DB) error { return sqlCommonUpdateDatabaseFrom3To4(mysqlV4SQL, dbHandle) } + +func updateMySQLDatabaseFrom4To5(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom4To5(dbHandle) +} diff --git a/dataprovider/pgsql.go b/dataprovider/pgsql.go index dd9b055d..1bdad1d4 100644 --- a/dataprovider/pgsql.go +++ b/dataprovider/pgsql.go @@ -209,28 +209,46 @@ func (p PGSQLProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updatePGSQLDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updatePGSQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updatePGSQLDatabaseFrom3To4(p.dbHandle) + return updatePGSQLDatabaseFromV1(p.dbHandle) case 2: - err = updatePGSQLDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updatePGSQLDatabaseFrom3To4(p.dbHandle) + return updatePGSQLDatabaseFromV2(p.dbHandle) case 3: - return updatePGSQLDatabaseFrom3To4(p.dbHandle) + return updatePGSQLDatabaseFromV3(p.dbHandle) + case 4: + return updatePGSQLDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updatePGSQLDatabaseFromV1(dbHandle *sql.DB) error { + err := updatePGSQLDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updatePGSQLDatabaseFromV2(dbHandle) +} + +func updatePGSQLDatabaseFromV2(dbHandle *sql.DB) error { + err := updatePGSQLDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updatePGSQLDatabaseFromV3(dbHandle) +} + +func updatePGSQLDatabaseFromV3(dbHandle *sql.DB) error { + err := updatePGSQLDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updatePGSQLDatabaseFromV4(dbHandle) +} + +func updatePGSQLDatabaseFromV4(dbHandle *sql.DB) error { + return updatePGSQLDatabaseFrom4To5(dbHandle) +} + func updatePGSQLDatabaseFrom1To2(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") @@ -248,3 +266,7 @@ func updatePGSQLDatabaseFrom2To3(dbHandle *sql.DB) error { func updatePGSQLDatabaseFrom3To4(dbHandle *sql.DB) error { return sqlCommonUpdateDatabaseFrom3To4(pgsqlV4SQL, dbHandle) } + +func updatePGSQLDatabaseFrom4To5(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom4To5(dbHandle) +} diff --git a/dataprovider/sqlcommon.go b/dataprovider/sqlcommon.go index cca11bed..a5111798 100644 --- a/dataprovider/sqlcommon.go +++ b/dataprovider/sqlcommon.go @@ -14,7 +14,7 @@ import ( ) const ( - sqlDatabaseVersion = 4 + sqlDatabaseVersion = 5 initialDBVersionSQL = "INSERT INTO {{schema_version}} (version) VALUES (1);" defaultSQLQueryTimeout = 10 * time.Second longSQLQueryTimeout = 60 * time.Second @@ -354,7 +354,8 @@ func sqlCommonGetUsers(limit int, offset int, order string, username string, dbH if err != nil { return users, err } - users = append(users, HideUserSensitiveData(&u)) + u.HideConfidentialData() + users = append(users, u) } } err = rows.Err() @@ -940,3 +941,88 @@ func sqlCommonUpdateDatabaseFrom3To4(sqlV4 string, dbHandle *sql.DB) error { } return err } + +func sqlCommonUpdateDatabaseFrom4To5(dbHandle *sql.DB) error { + logger.InfoToConsole("updating database version: 4 -> 5") + providerLog(logger.LevelInfo, "updating database version: 4 -> 5") + ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) + defer cancel() + q := getCompatV4FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + rows, err := stmt.QueryContext(ctx) + if err != nil { + return err + } + defer rows.Close() + + users := []User{} + for rows.Next() { + var compatUser compatUserV4 + var fsConfigString sql.NullString + err = rows.Scan(&compatUser.ID, &compatUser.Username, &fsConfigString) + if err != nil { + return err + } + if fsConfigString.Valid { + err = json.Unmarshal([]byte(fsConfigString.String), &compatUser.FsConfig) + if err != nil { + logger.WarnToConsole("failed to unmarshal v4 user %#v, is it already migrated?", compatUser.Username) + continue + } + fsConfig, err := convertFsConfigFromV4(compatUser.FsConfig, compatUser.Username) + if err != nil { + return err + } + users = append(users, createUserFromV4(compatUser, fsConfig)) + } + } + if err := rows.Err(); err != nil { + return err + } + + for _, user := range users { + err = sqlCommonUpdateV4User(dbHandle, user) + if err != nil { + return err + } + providerLog(logger.LevelInfo, "filesystem config updated for user %#v", user.Username) + } + + ctxVersion, cancelVersion := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancelVersion() + + return sqlCommonUpdateDatabaseVersion(ctxVersion, dbHandle, 5) +} + +func sqlCommonUpdateV4User(dbHandle *sql.DB, user User) error { + err := validateFilesystemConfig(&user) + if err != nil { + return err + } + err = saveGCSCredentials(&user) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout) + defer cancel() + + q := updateCompatV4FsConfigQuery() + stmt, err := dbHandle.PrepareContext(ctx, q) + if err != nil { + providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err) + return err + } + defer stmt.Close() + + fsConfig, err := user.GetFsConfigAsJSON() + if err != nil { + return err + } + _, err = stmt.ExecContext(ctx, string(fsConfig), user.ID) + return err +} diff --git a/dataprovider/sqlite.go b/dataprovider/sqlite.go index d96f5aca..00374448 100644 --- a/dataprovider/sqlite.go +++ b/dataprovider/sqlite.go @@ -232,28 +232,46 @@ func (p SQLiteProvider) migrateDatabase() error { } switch dbVersion.Version { case 1: - err = updateSQLiteDatabaseFrom1To2(p.dbHandle) - if err != nil { - return err - } - err = updateSQLiteDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateSQLiteDatabaseFrom3To4(p.dbHandle) + return updateSQLiteDatabaseFromV1(p.dbHandle) case 2: - err = updateSQLiteDatabaseFrom2To3(p.dbHandle) - if err != nil { - return err - } - return updateSQLiteDatabaseFrom3To4(p.dbHandle) + return updateSQLiteDatabaseFromV2(p.dbHandle) case 3: - return updateSQLiteDatabaseFrom3To4(p.dbHandle) + return updateSQLiteDatabaseFromV3(p.dbHandle) + case 4: + return updateSQLiteDatabaseFromV4(p.dbHandle) default: return fmt.Errorf("Database version not handled: %v", dbVersion.Version) } } +func updateSQLiteDatabaseFromV1(dbHandle *sql.DB) error { + err := updateSQLiteDatabaseFrom1To2(dbHandle) + if err != nil { + return err + } + return updateSQLiteDatabaseFromV2(dbHandle) +} + +func updateSQLiteDatabaseFromV2(dbHandle *sql.DB) error { + err := updateSQLiteDatabaseFrom2To3(dbHandle) + if err != nil { + return err + } + return updateSQLiteDatabaseFromV3(dbHandle) +} + +func updateSQLiteDatabaseFromV3(dbHandle *sql.DB) error { + err := updateSQLiteDatabaseFrom3To4(dbHandle) + if err != nil { + return err + } + return updateSQLiteDatabaseFromV4(dbHandle) +} + +func updateSQLiteDatabaseFromV4(dbHandle *sql.DB) error { + return updateSQLiteDatabaseFrom4To5(dbHandle) +} + func updateSQLiteDatabaseFrom1To2(dbHandle *sql.DB) error { logger.InfoToConsole("updating database version: 1 -> 2") providerLog(logger.LevelInfo, "updating database version: 1 -> 2") @@ -271,3 +289,7 @@ func updateSQLiteDatabaseFrom2To3(dbHandle *sql.DB) error { func updateSQLiteDatabaseFrom3To4(dbHandle *sql.DB) error { return sqlCommonUpdateDatabaseFrom3To4(sqliteV4SQL, dbHandle) } + +func updateSQLiteDatabaseFrom4To5(dbHandle *sql.DB) error { + return sqlCommonUpdateDatabaseFrom4To5(dbHandle) +} diff --git a/dataprovider/sqlqueries.go b/dataprovider/sqlqueries.go index 193b3044..564531fb 100644 --- a/dataprovider/sqlqueries.go +++ b/dataprovider/sqlqueries.go @@ -184,3 +184,11 @@ func getUpdateDBVersionQuery() string { func getCompatVirtualFoldersQuery() string { return fmt.Sprintf(`SELECT id,username,virtual_folders FROM %v`, sqlTableUsers) } + +func getCompatV4FsConfigQuery() string { + return fmt.Sprintf(`SELECT id,username,filesystem FROM %v`, sqlTableUsers) +} + +func updateCompatV4FsConfigQuery() string { + return fmt.Sprintf(`UPDATE %v SET filesystem=%v WHERE id=%v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1]) +} diff --git a/dataprovider/user.go b/dataprovider/user.go index 04b85987..344f61be 100644 --- a/dataprovider/user.go +++ b/dataprovider/user.go @@ -230,6 +230,19 @@ func (u *User) GetFilesystem(connectionID string) (vfs.Fs, error) { return vfs.NewOsFs(connectionID, u.GetHomeDir(), u.VirtualFolders), nil } +// HideConfidentialData hides user confidential data +func (u *User) HideConfidentialData() { + u.Password = "" + switch u.FsConfig.Provider { + case S3FilesystemProvider: + u.FsConfig.S3Config.AccessSecret.Hide() + case GCSFilesystemProvider: + u.FsConfig.GCSConfig.Credentials.Hide() + case AzureBlobFilesystemProvider: + u.FsConfig.AzBlobConfig.AccountKey.Hide() + } +} + // GetPermissionsForPath returns the permissions for the given path. // The path must be an SFTP path func (u *User) GetPermissionsForPath(p string) []string { @@ -809,6 +822,7 @@ func (u *User) getACopy() User { UploadPartSize: u.FsConfig.AzBlobConfig.UploadPartSize, UploadConcurrency: u.FsConfig.AzBlobConfig.UploadConcurrency, UseEmulator: u.FsConfig.AzBlobConfig.UseEmulator, + AccessTier: u.FsConfig.AzBlobConfig.AccessTier, }, } diff --git a/docker/README.md b/docker/README.md index 01016f5e..09525e3f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -79,15 +79,24 @@ Please take a look [here](../docs/full-configuration.md#environment-variables) t Alternately you can mount your custom configuration file to `/var/lib/sftpgo` or `/var/lib/sftpgo/.config/sftpgo`. +### Loading initial data + +Initial data can be loaded in the following ways: + +- via the `--loaddata-from` flag or the `SFTPGO_LOADDATA_FROM` environment variable +- by providing a dump file to the memory provider + +Please take a look [here](../docs/full-configuration.md) for more details. + ### Running as an arbitrary user The SFTPGo image runs using `1000` as UID/GID by default. If you know the permissions of your data and/or configuration directory are already set appropriately or you have need of running SFTPGo with a specific UID/GID, it is possible to invoke this image with `--user` set to any value (other than `root/0`) in order to achieve the desired access/configuration: ```shell $ ls -lnd data -drwxr-xr-x 2 1100 11000 6 6 nov 09.09 data +drwxr-xr-x 2 1100 1100 6 7 nov 09.09 data $ ls -lnd config -drwxr-xr-x 2 1100 11000 6 6 nov 09.19 config +drwxr-xr-x 2 1100 1100 6 7 nov 09.19 config ``` With the above directory permissions, you can start a SFTPGo instance like this: diff --git a/docs/full-configuration.md b/docs/full-configuration.md index 1404a934..91de40ee 100644 --- a/docs/full-configuration.md +++ b/docs/full-configuration.md @@ -115,7 +115,7 @@ The configuration file contains the following sections: - `max_size`, integer. Maximum number of users to cache. 0 means unlimited. Default: 50. - **"data_provider"**, the configuration for the data provider - `driver`, string. Supported drivers are `sqlite`, `mysql`, `postgresql`, `bolt`, `memory` - - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the users dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted + - `name`, string. Database name. For driver `sqlite` this can be the database name relative to the config dir or the absolute path to the SQLite database. For driver `memory` this is the (optional) path relative to the config dir or the absolute path to the provider dump, obtained using the `dumpdata` REST API, to load. This dump will be loaded at startup and can be reloaded on demand sending a `SIGHUP` signal on Unix based systems and a `paramchange` request to the running service on Windows. The `memory` provider will not modify the provided file so quota usage and last login will not be persisted - `host`, string. Database host. Leave empty for drivers `sqlite`, `bolt` and `memory` - `port`, integer. Database port. Leave empty for drivers `sqlite`, `bolt` and `memory` - `username`, string. Database user. Leave empty for drivers `sqlite`, `bolt` and `memory` diff --git a/examples/rest-api-cli/README.md b/examples/rest-api-cli/README.md index 56c479c4..4f100afd 100644 --- a/examples/rest-api-cli/README.md +++ b/examples/rest-api-cli/README.md @@ -58,7 +58,10 @@ Output: "provider": 1, "s3config": { "access_key": "accesskey", - "access_secret": "$aes$6c088ba12b0b261247c8cf331c46d9260b8e58002957d89ad1c0495e3af665cd0227", + "access_secret": { + "payload": "ac46cec75466ba77e47f536436783b729ca5bbbb53252fda0de51f785a6da11ffb03", + "status": "AES-256-GCM" + }, "bucket": "test", "endpoint": "http://127.0.0.1:9000", "key_prefix": "vfolder/", diff --git a/examples/rest-api-cli/sftpgo_api_cli b/examples/rest-api-cli/sftpgo_api_cli index 9c351dc2..55661dcc 100755 --- a/examples/rest-api-cli/sftpgo_api_cli +++ b/examples/rest-api-cli/sftpgo_api_cli @@ -238,23 +238,30 @@ class SFTPGoApiRequests: az_upload_concurrency, az_key_prefix, az_use_emulator, az_access_tier): fs_config = {'provider':0} if fs_provider == 'S3': + secret = {} + if s3_access_secret: + secret.update({"status":"Plain", "payload":s3_access_secret}) s3config = {'bucket':s3_bucket, 'region':s3_region, 'access_key':s3_access_key, 'access_secret': - s3_access_secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class, 'key_prefix': + secret, 'endpoint':s3_endpoint, 'storage_class':s3_storage_class, 'key_prefix': s3_key_prefix, 'upload_part_size':s3_upload_part_size, 'upload_concurrency':s3_upload_concurrency} fs_config.update({'provider':1, 's3config':s3config}) elif fs_provider == 'GCS': - gcsconfig = {'bucket':gcs_bucket, 'key_prefix':gcs_key_prefix, 'storage_class':gcs_storage_class} + gcsconfig = {'bucket':gcs_bucket, 'key_prefix':gcs_key_prefix, 'storage_class':gcs_storage_class, + 'credentials':{}} if gcs_automatic_credentials == "automatic": gcsconfig.update({'automatic_credentials':1}) else: gcsconfig.update({'automatic_credentials':0}) if gcs_credentials_file: with open(gcs_credentials_file) as creds: - gcsconfig.update({'credentials':base64.b64encode(creds.read().encode('UTF-8')).decode('UTF-8'), - 'automatic_credentials':0}) + secret = {"status":"Plain", "payload":creds.read()} + gcsconfig.update({'credentials':secret, 'automatic_credentials':0}) fs_config.update({'provider':2, 'gcsconfig':gcsconfig}) elif fs_provider == "AzureBlob": - azureconfig = {'container':az_container, 'account_name':az_account_name, 'account_key':az_account_key, + secret = {} + if az_account_key: + secret.update({"status":"Plain", "payload":az_account_key}) + azureconfig = {'container':az_container, 'account_name':az_account_name, 'account_key':secret, 'sas_url':az_sas_url, 'endpoint':az_endpoint, 'upload_part_size':az_upload_part_size, 'upload_concurrency':az_upload_concurrency, 'key_prefix':az_key_prefix, 'use_emulator': az_use_emulator, 'access_tier':az_access_tier} @@ -609,7 +616,7 @@ def addCommonUserArguments(parser): help='Denied IP/Mask in CIDR notation. For example "192.168.2.0/24" or "2001:db8::/32". Default: %(default)s') parser.add_argument('--denied-patterns', type=str, nargs='*', default=[], help='Denied file patterns case insensitive. ' +'The format is /dir::pattern1,pattern2. For example: "/somedir::*.jpg,*.png" "/otherdir/subdir::a*b?.zip,*.rar". ' + - 'You have to set both denied and allowed patterns to update existing values or none to preserve them.' + + ' You have to set both denied and allowed patterns to update existing values or none to preserve them.' + ' If you only set allowed or denied patterns the missing one is assumed to be an empty list. Default: %(default)s') parser.add_argument('--allowed-patterns', type=str, nargs='*', default=[], help='Allowed file patterns case insensitive. ' +'The format is /dir::pattern1,pattern2. For example: "/somedir::*.jpg,a*b?.png" "/otherdir/subdir::*.zip,*.rar". ' + diff --git a/ftpd/ftpd_test.go b/ftpd/ftpd_test.go index a9769496..9f88f7e0 100644 --- a/ftpd/ftpd_test.go +++ b/ftpd/ftpd_test.go @@ -876,7 +876,10 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: `{ "type": "service_account" }`, + } providerConf := config.GetProviderConf() providerConf.PreferDatabaseCredentials = true @@ -897,9 +900,12 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.Key) - _, err = os.Stat(credentialsFile) - assert.Error(t, err) + assert.NoFileExists(t, credentialsFile) client, err := getFTPClient(user, false) if assert.NoError(t, err) { @@ -922,7 +928,10 @@ func TestLoginInvalidFs(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "invalid JSON for credentials", + } user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) diff --git a/httpd/api_maintenance.go b/httpd/api_maintenance.go index b218d37b..5ea0e1b8 100644 --- a/httpd/api_maintenance.go +++ b/httpd/api_maintenance.go @@ -96,8 +96,7 @@ func loadData(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", getRespStatus(err)) return } - var dump dataprovider.BackupData - err = json.Unmarshal(content, &dump) + dump, err := dataprovider.ParseDumpData(content) if err != nil { sendAPIResponse(w, r, err, fmt.Sprintf("Unable to parse input file: %#v", inputFile), http.StatusBadRequest) return diff --git a/httpd/api_user.go b/httpd/api_user.go index 5c257335..5d786b02 100644 --- a/httpd/api_user.go +++ b/httpd/api_user.go @@ -11,7 +11,7 @@ import ( "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/dataprovider" - "github.com/drakkan/sftpgo/utils" + "github.com/drakkan/sftpgo/vfs" ) func getUsers(w http.ResponseWriter, r *http.Request) { @@ -67,7 +67,8 @@ func getUserByID(w http.ResponseWriter, r *http.Request) { } user, err := dataprovider.GetUserByID(userID) if err == nil { - render.JSON(w, r, dataprovider.HideUserSensitiveData(&user)) + user.HideConfidentialData() + render.JSON(w, r, user) } else { sendAPIResponse(w, r, err, "", getRespStatus(err)) } @@ -81,11 +82,29 @@ func addUser(w http.ResponseWriter, r *http.Request) { sendAPIResponse(w, r, err, "", http.StatusBadRequest) return } + switch user.FsConfig.Provider { + case dataprovider.S3FilesystemProvider: + if user.FsConfig.S3Config.AccessSecret.IsRedacted() { + sendAPIResponse(w, r, errors.New("invalid access_secret"), "", http.StatusBadRequest) + return + } + case dataprovider.GCSFilesystemProvider: + if user.FsConfig.GCSConfig.Credentials.IsRedacted() { + sendAPIResponse(w, r, errors.New("invalid credentials"), "", http.StatusBadRequest) + return + } + case dataprovider.AzureBlobFilesystemProvider: + if user.FsConfig.AzBlobConfig.AccountKey.IsRedacted() { + sendAPIResponse(w, r, errors.New("invalid account_key"), "", http.StatusBadRequest) + return + } + } err = dataprovider.AddUser(user) if err == nil { user, err = dataprovider.UserExists(user.Username) if err == nil { - render.JSON(w, r, dataprovider.HideUserSensitiveData(&user)) + user.HideConfidentialData() + render.JSON(w, r, user) } else { sendAPIResponse(w, r, err, "", getRespStatus(err)) } @@ -117,15 +136,22 @@ func updateUser(w http.ResponseWriter, r *http.Request) { return } currentPermissions := user.Permissions - currentS3AccessSecret := "" - currentAzAccountKey := "" + var currentS3AccessSecret vfs.Secret + var currentAzAccountKey vfs.Secret + var currentGCSCredentials vfs.Secret if user.FsConfig.Provider == dataprovider.S3FilesystemProvider { currentS3AccessSecret = user.FsConfig.S3Config.AccessSecret } if user.FsConfig.Provider == dataprovider.AzureBlobFilesystemProvider { currentAzAccountKey = user.FsConfig.AzBlobConfig.AccountKey } + if user.FsConfig.Provider == dataprovider.GCSFilesystemProvider { + currentGCSCredentials = user.FsConfig.GCSConfig.Credentials + } user.Permissions = make(map[string][]string) + user.FsConfig.S3Config = vfs.S3FsConfig{} + user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} err = render.DecodeJSON(r.Body, &user) if err != nil { sendAPIResponse(w, r, err, "", http.StatusBadRequest) @@ -135,7 +161,7 @@ func updateUser(w http.ResponseWriter, r *http.Request) { if len(user.Permissions) == 0 { user.Permissions = currentPermissions } - updateEncryptedSecrets(&user, currentS3AccessSecret, currentAzAccountKey) + updateEncryptedSecrets(&user, currentS3AccessSecret, currentAzAccountKey, currentGCSCredentials) if user.ID != userID { sendAPIResponse(w, r, err, "user ID in request body does not match user ID in path parameter", http.StatusBadRequest) @@ -181,18 +207,21 @@ func disconnectUser(username string) { } } -func updateEncryptedSecrets(user *dataprovider.User, currentS3AccessSecret, currentAzAccountKey string) { - // we use the new access secret if different from the old one and not empty +func updateEncryptedSecrets(user *dataprovider.User, currentS3AccessSecret, currentAzAccountKey, currentGCSCredentials vfs.Secret) { + // we use the new access secret if plain or empty, otherwise the old value if user.FsConfig.Provider == dataprovider.S3FilesystemProvider { - if utils.RemoveDecryptionKey(currentS3AccessSecret) == user.FsConfig.S3Config.AccessSecret || - (user.FsConfig.S3Config.AccessSecret == "" && user.FsConfig.S3Config.AccessKey != "") { + if !user.FsConfig.S3Config.AccessSecret.IsPlain() && !user.FsConfig.S3Config.AccessSecret.IsEmpty() { user.FsConfig.S3Config.AccessSecret = currentS3AccessSecret } } if user.FsConfig.Provider == dataprovider.AzureBlobFilesystemProvider { - if utils.RemoveDecryptionKey(currentAzAccountKey) == user.FsConfig.AzBlobConfig.AccountKey || - (user.FsConfig.AzBlobConfig.AccountKey == "" && user.FsConfig.AzBlobConfig.AccountName != "") { + if !user.FsConfig.AzBlobConfig.AccountKey.IsPlain() && !user.FsConfig.AzBlobConfig.AccountKey.IsEmpty() { user.FsConfig.AzBlobConfig.AccountKey = currentAzAccountKey } } + if user.FsConfig.Provider == dataprovider.GCSFilesystemProvider { + if !user.FsConfig.GCSConfig.Credentials.IsPlain() && !user.FsConfig.GCSConfig.Credentials.IsEmpty() { + user.FsConfig.GCSConfig.Credentials = currentGCSCredentials + } + } } diff --git a/httpd/api_utils.go b/httpd/api_utils.go index 79c6f005..37e6e303 100644 --- a/httpd/api_utils.go +++ b/httpd/api_utils.go @@ -707,28 +707,19 @@ func compareAzBlobConfig(expected *dataprovider.User, actual *dataprovider.User) return nil } -func checkEncryptedSecret(expectedAccessSecret, actualAccessSecret string) error { - if len(expectedAccessSecret) > 0 { - vals := strings.Split(expectedAccessSecret, "$") - if strings.HasPrefix(expectedAccessSecret, "$aes$") && len(vals) == 4 { - expectedAccessSecret = utils.RemoveDecryptionKey(expectedAccessSecret) - if expectedAccessSecret != actualAccessSecret { - return fmt.Errorf("secret mismatch, expected: %v", expectedAccessSecret) - } - } else { - // here we check that actualAccessSecret is aes encrypted without the nonce - parts := strings.Split(actualAccessSecret, "$") - if !strings.HasPrefix(actualAccessSecret, "$aes$") || len(parts) != 3 { - return errors.New("invalid secret") - } - if len(parts) == len(vals) { - if expectedAccessSecret != actualAccessSecret { - return errors.New("encrypted secret mismatch") - } - } +func checkEncryptedSecret(expected, actual vfs.Secret) error { + if expected.IsPlain() && actual.IsEncrypted() { + if actual.Payload == "" { + return errors.New("invalid secret payload") + } + if actual.AdditionalData != "" { + return errors.New("invalid secret additional data") + } + if actual.Key != "" { + return errors.New("invalid secret key") } } else { - if expectedAccessSecret != actualAccessSecret { + if expected.Status != actual.Status || expected.Payload != actual.Payload { return errors.New("secret mismatch") } } diff --git a/httpd/httpd_test.go b/httpd/httpd_test.go index 78c4c70a..ec213028 100644 --- a/httpd/httpd_test.go +++ b/httpd/httpd_test.go @@ -26,6 +26,7 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/drakkan/sftpgo/common" "github.com/drakkan/sftpgo/config" @@ -190,7 +191,7 @@ func TestMain(m *testing.M) { defer testServer.Close() exitCode := m.Run() - os.Remove(logfilePath) //nolint:errcheck + //os.Remove(logfilePath) //nolint:errcheck os.RemoveAll(backupsPath) //nolint:errcheck os.RemoveAll(credentialsPath) //nolint:errcheck os.Remove(certPath) //nolint:errcheck @@ -438,12 +439,16 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u.FsConfig.S3Config.Bucket = "testbucket" u.FsConfig.S3Config.Region = "eu-west-1" u.FsConfig.S3Config.AccessKey = "access-key" - u.FsConfig.S3Config.AccessSecret = "access-secret" + u.FsConfig.S3Config.AccessSecret.Payload = "access-secret" + u.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusRedacted u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" u.FsConfig.S3Config.StorageClass = "Standard" //nolint:goconst u.FsConfig.S3Config.KeyPrefix = "/adir/subdir/" _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) + u.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain + _, _, err = httpd.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err) u.FsConfig.S3Config.KeyPrefix = "" u.FsConfig.S3Config.UploadPartSize = 3 _, _, err = httpd.AddUser(u, http.StatusBadRequest) @@ -463,16 +468,20 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u.FsConfig.GCSConfig.Bucket = "abucket" u.FsConfig.GCSConfig.StorageClass = "Standard" u.FsConfig.GCSConfig.KeyPrefix = "/somedir/subdir/" - u.FsConfig.GCSConfig.Credentials = []byte("test") + u.FsConfig.GCSConfig.Credentials.Payload = "test" //nolint:goconst + u.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusRedacted + _, _, err = httpd.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err) + u.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) u.FsConfig.GCSConfig.KeyPrefix = "somedir/subdir/" //nolint:goconst - u.FsConfig.GCSConfig.Credentials = nil + u.FsConfig.GCSConfig.Credentials = vfs.Secret{} u.FsConfig.GCSConfig.AutomaticCredentials = 0 _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) - - u.FsConfig.GCSConfig.Credentials = invalidBase64{} + u.FsConfig.GCSConfig.Credentials.Payload = "invalid" + u.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusAES256GCM _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) @@ -488,10 +497,14 @@ func TestAddUserInvalidFsConfig(t *testing.T) { u.FsConfig.AzBlobConfig.Container = "container" _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) - u.FsConfig.AzBlobConfig.AccountKey = "key" + u.FsConfig.AzBlobConfig.AccountKey.Payload = "key" + u.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusRedacted u.FsConfig.AzBlobConfig.KeyPrefix = "/amedir/subdir/" _, _, err = httpd.AddUser(u, http.StatusBadRequest) assert.NoError(t, err) + u.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain + _, _, err = httpd.AddUser(u, http.StatusBadRequest) + assert.NoError(t, err) u.FsConfig.AzBlobConfig.KeyPrefix = "amedir/subdir/" u.FsConfig.AzBlobConfig.UploadPartSize = -1 _, _, err = httpd.AddUser(u, http.StatusBadRequest) @@ -1000,19 +1013,35 @@ func TestUserS3Config(t *testing.T) { user.FsConfig.S3Config.Bucket = "test" //nolint:goconst user.FsConfig.S3Config.Region = "us-east-1" //nolint:goconst user.FsConfig.S3Config.AccessKey = "Server-Access-Key" - user.FsConfig.S3Config.AccessSecret = "Server-Access-Secret" + user.FsConfig.S3Config.AccessSecret.Payload = "Server-Access-Secret" + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000" user.FsConfig.S3Config.UploadPartSize = 8 - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") - assert.NoError(t, err) + user, body, err := httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err, string(body)) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, user.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.Key) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 - secret, _ := utils.EncryptData("Server-Access-Secret") + secret := vfs.Secret{ + Payload: "Server-Access-Secret", + Status: vfs.SecretStatusAES256GCM, + } user.FsConfig.S3Config.AccessSecret = secret + _, _, err = httpd.AddUser(user, http.StatusOK) + assert.Error(t, err) + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user, _, err = httpd.AddUser(user, http.StatusOK) assert.NoError(t, err) + initialSecretPayload := user.FsConfig.S3Config.AccessSecret.Payload + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, initialSecretPayload) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.Key) user.FsConfig.Provider = dataprovider.S3FilesystemProvider user.FsConfig.S3Config.Bucket = "test-bucket" user.FsConfig.S3Config.Region = "us-east-1" //nolint:goconst @@ -1022,29 +1051,31 @@ func TestUserS3Config(t *testing.T) { user.FsConfig.S3Config.UploadConcurrency = 5 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) - user.FsConfig.Provider = dataprovider.LocalFilesystemProvider - user.FsConfig.S3Config.Bucket = "" - user.FsConfig.S3Config.Region = "" - user.FsConfig.S3Config.AccessKey = "" - user.FsConfig.S3Config.AccessSecret = "" - user.FsConfig.S3Config.Endpoint = "" - user.FsConfig.S3Config.KeyPrefix = "" - user.FsConfig.S3Config.UploadPartSize = 0 - user.FsConfig.S3Config.UploadConcurrency = 0 - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") - assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.S3Config.AccessSecret.Status) + assert.Equal(t, initialSecretPayload, user.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.Empty(t, user.FsConfig.S3Config.AccessSecret.Key) // test user without access key and access secret (shared config state) user.FsConfig.Provider = dataprovider.S3FilesystemProvider user.FsConfig.S3Config.Bucket = "testbucket" user.FsConfig.S3Config.Region = "us-east-1" user.FsConfig.S3Config.AccessKey = "" - user.FsConfig.S3Config.AccessSecret = "" + user.FsConfig.S3Config.AccessSecret = vfs.Secret{} user.FsConfig.S3Config.Endpoint = "" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir" user.FsConfig.S3Config.UploadPartSize = 6 user.FsConfig.S3Config.UploadConcurrency = 4 - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") + user, body, err = httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err, string(body)) + assert.True(t, user.FsConfig.S3Config.AccessSecret.IsEmpty()) + _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) + user.Password = defaultPassword + user.ID = 0 + // shared credential test for add instead of update + user, _, err = httpd.AddUser(user, http.StatusOK) + assert.NoError(t, err) + assert.True(t, user.FsConfig.S3Config.AccessSecret.IsEmpty()) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) } @@ -1058,36 +1089,69 @@ func TestUserGCSConfig(t *testing.T) { assert.NoError(t, err) user.FsConfig.Provider = dataprovider.GCSFilesystemProvider user.FsConfig.GCSConfig.Bucket = "test" - user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") + user.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" //nolint:goconst + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + credentialFile := filepath.Join(credentialsPath, fmt.Sprintf("%v_gcs_credentials.json", user.Username)) + assert.FileExists(t, credentialFile) + creds, err := ioutil.ReadFile(credentialFile) + assert.NoError(t, err) + secret := &vfs.Secret{} + err = json.Unmarshal(creds, secret) + assert.NoError(t, err) + err = secret.Decrypt() + assert.NoError(t, err) + assert.Equal(t, "fake credentials", secret.Payload) + user.FsConfig.GCSConfig.Credentials.Payload = "fake encrypted credentials" + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusAES256GCM + user, _, err = httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + assert.FileExists(t, credentialFile) + creds, err = ioutil.ReadFile(credentialFile) + assert.NoError(t, err) + secret = &vfs.Secret{} + err = json.Unmarshal(creds, secret) + assert.NoError(t, err) + err = secret.Decrypt() + assert.NoError(t, err) + assert.Equal(t, "fake credentials", secret.Payload) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 - user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") + user.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusAES256GCM + _, _, err = httpd.AddUser(user, http.StatusOK) + assert.Error(t, err) + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain user, body, err := httpd.AddUser(user, http.StatusOK) assert.NoError(t, err, string(body)) err = os.RemoveAll(credentialsPath) assert.NoError(t, err) err = os.MkdirAll(credentialsPath, 0700) assert.NoError(t, err) - user.FsConfig.GCSConfig.Credentials = nil + user.FsConfig.GCSConfig.Credentials = vfs.Secret{} user.FsConfig.GCSConfig.AutomaticCredentials = 1 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + assert.NoFileExists(t, credentialFile) + user.FsConfig.GCSConfig = vfs.GCSFsConfig{} user.FsConfig.Provider = dataprovider.S3FilesystemProvider user.FsConfig.S3Config.Bucket = "test1" user.FsConfig.S3Config.Region = "us-east-1" user.FsConfig.S3Config.AccessKey = "Server-Access-Key1" - user.FsConfig.S3Config.AccessSecret = "secret" + user.FsConfig.S3Config.AccessSecret.Payload = "secret" + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user.FsConfig.S3Config.Endpoint = "http://localhost:9000" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir" user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + user.FsConfig.S3Config = vfs.S3FsConfig{} user.FsConfig.Provider = dataprovider.GCSFilesystemProvider user.FsConfig.GCSConfig.Bucket = "test1" - user.FsConfig.GCSConfig.Credentials = []byte("fake credentials") + user.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" + user.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) @@ -1101,42 +1165,248 @@ func TestUserAzureBlobConfig(t *testing.T) { user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider user.FsConfig.AzBlobConfig.Container = "test" user.FsConfig.AzBlobConfig.AccountName = "Server-Account-Name" - user.FsConfig.AzBlobConfig.AccountKey = "Server-Account-Key" + user.FsConfig.AzBlobConfig.AccountKey.Payload = "Server-Account-Key" + user.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain user.FsConfig.AzBlobConfig.Endpoint = "http://127.0.0.1:9000" user.FsConfig.AzBlobConfig.UploadPartSize = 8 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + initialPayload := user.FsConfig.AzBlobConfig.AccountKey.Payload + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, initialPayload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) + user.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusAES256GCM + user.FsConfig.AzBlobConfig.AccountKey.AdditionalData = "data" + user.FsConfig.AzBlobConfig.AccountKey.Key = "fake key" + user, _, err = httpd.UpdateUser(user, http.StatusOK, "") + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.Equal(t, initialPayload, user.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) + _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) user.Password = defaultPassword user.ID = 0 - secret, _ := utils.EncryptData("Server-Account-Key") + secret := vfs.Secret{ + Payload: "Server-Account-Key", + Status: vfs.SecretStatusAES256GCM, + } user.FsConfig.AzBlobConfig.AccountKey = secret + _, _, err = httpd.AddUser(user, http.StatusOK) + assert.Error(t, err) + user.FsConfig.AzBlobConfig.AccountKey = vfs.Secret{ + Payload: "Server-Account-Key-Test", + Status: vfs.SecretStatusPlain, + } user, _, err = httpd.AddUser(user, http.StatusOK) assert.NoError(t, err) + initialPayload = user.FsConfig.AzBlobConfig.AccountKey.Payload + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, initialPayload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider user.FsConfig.AzBlobConfig.Container = "test-container" - user.FsConfig.AzBlobConfig.AccountKey = "Server-Account-Key1" user.FsConfig.AzBlobConfig.Endpoint = "http://localhost:9001" user.FsConfig.AzBlobConfig.KeyPrefix = "somedir/subdir" user.FsConfig.AzBlobConfig.UploadConcurrency = 5 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) - user.FsConfig.Provider = dataprovider.LocalFilesystemProvider - user.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{} - user, _, err = httpd.UpdateUser(user, http.StatusOK, "") - assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, initialPayload) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.Empty(t, user.FsConfig.AzBlobConfig.AccountKey.Key) // test user without access key and access secret (sas) user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider - user.FsConfig.AzBlobConfig.SASURL = "https://myaccount.blob.core.windows.net/pictures/profile.jpg?sv=2012-02-12&st=2009-02-09&se=2009-02-10&sr=c&sp=r&si=YWJjZGVmZw%3d%3d&sig=dD80ihBh5jfNpymO5Hg1IdiJIEvHcJpCMiCMnN%2fRnbI%3d" user.FsConfig.AzBlobConfig.KeyPrefix = "somedir/subdir" + user.FsConfig.AzBlobConfig.AccountName = "" + user.FsConfig.AzBlobConfig.AccountKey = vfs.Secret{} user.FsConfig.AzBlobConfig.UploadPartSize = 6 user.FsConfig.AzBlobConfig.UploadConcurrency = 4 user, _, err = httpd.UpdateUser(user, http.StatusOK, "") assert.NoError(t, err) + assert.True(t, user.FsConfig.AzBlobConfig.AccountKey.IsEmpty()) _, err = httpd.RemoveUser(user, http.StatusOK) assert.NoError(t, err) + user.Password = defaultPassword + user.ID = 0 + // sas test for add instead of update + user, _, err = httpd.AddUser(user, http.StatusOK) + assert.NoError(t, err) + assert.True(t, user.FsConfig.AzBlobConfig.AccountKey.IsEmpty()) + _, err = httpd.RemoveUser(user, http.StatusOK) + assert.NoError(t, err) +} + +func TestUserHiddenFields(t *testing.T) { + err := dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf := config.GetProviderConf() + providerConf.PreferDatabaseCredentials = true + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) + + // sensitive data must be hidden but not deleted from the dataprovider + usernames := []string{"user1", "user2", "user3"} + u1 := getTestUser() + u1.Username = usernames[0] + u1.FsConfig.Provider = dataprovider.S3FilesystemProvider + u1.FsConfig.S3Config.Bucket = "test" + u1.FsConfig.S3Config.Region = "us-east-1" + u1.FsConfig.S3Config.AccessKey = "S3-Access-Key" + u1.FsConfig.S3Config.AccessSecret.Payload = "S3-Access-Secret" + u1.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain + user1, _, err := httpd.AddUser(u1, http.StatusOK) + assert.NoError(t, err) + + u2 := getTestUser() + u2.Username = usernames[1] + u2.FsConfig.Provider = dataprovider.GCSFilesystemProvider + u2.FsConfig.GCSConfig.Bucket = "test" + u2.FsConfig.GCSConfig.Credentials.Payload = "fake credentials" + u2.FsConfig.GCSConfig.Credentials.Status = vfs.SecretStatusPlain + user2, _, err := httpd.AddUser(u2, http.StatusOK) + assert.NoError(t, err) + + u3 := getTestUser() + u3.Username = usernames[2] + u3.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider + u3.FsConfig.AzBlobConfig.Container = "test" + u3.FsConfig.AzBlobConfig.AccountName = "Server-Account-Name" + u3.FsConfig.AzBlobConfig.AccountKey.Payload = "Server-Account-Key" + u3.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain + user3, _, err := httpd.AddUser(u3, http.StatusOK) + assert.NoError(t, err) + + users, _, err := httpd.GetUsers(0, 0, "", http.StatusOK) + assert.NoError(t, err) + assert.GreaterOrEqual(t, len(users), 3) + for _, username := range usernames { + users, _, err = httpd.GetUsers(0, 0, username, http.StatusOK) + assert.NoError(t, err) + if assert.Len(t, users, 1) { + user := users[0] + assert.Empty(t, user.Password) + } + } + user1, _, err = httpd.GetUserByID(user1.ID, http.StatusOK) + assert.NoError(t, err) + assert.Empty(t, user1.Password) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Payload) + + user2, _, err = httpd.GetUserByID(user2.ID, http.StatusOK) + assert.NoError(t, err) + assert.Empty(t, user2.Password) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.Key) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Payload) + + user3, _, err = httpd.GetUserByID(user3.ID, http.StatusOK) + assert.NoError(t, err) + assert.Empty(t, user3.Password) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Payload) + + // finally check that we have all the data inside the data provider + user1, err = dataprovider.GetUserByID(user1.ID) + assert.NoError(t, err) + assert.NotEmpty(t, user1.Password) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Key) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.AdditionalData) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, user1.FsConfig.S3Config.AccessSecret.Payload) + err = user1.FsConfig.S3Config.AccessSecret.Decrypt() + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusPlain, user1.FsConfig.S3Config.AccessSecret.Status) + assert.Equal(t, u1.FsConfig.S3Config.AccessSecret.Payload, user1.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, user1.FsConfig.S3Config.AccessSecret.AdditionalData) + + user2, err = dataprovider.GetUserByID(user2.ID) + assert.NoError(t, err) + assert.NotEmpty(t, user2.Password) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Key) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user2.FsConfig.GCSConfig.Credentials.Payload) + err = user2.FsConfig.GCSConfig.Credentials.Decrypt() + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusPlain, user2.FsConfig.GCSConfig.Credentials.Status) + assert.Equal(t, u2.FsConfig.GCSConfig.Credentials.Payload, user2.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.Key) + assert.Empty(t, user2.FsConfig.GCSConfig.Credentials.AdditionalData) + + user3, err = dataprovider.GetUserByID(user3.ID) + assert.NoError(t, err) + assert.NotEmpty(t, user3.Password) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Key) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, user3.FsConfig.AzBlobConfig.AccountKey.Payload) + err = user3.FsConfig.AzBlobConfig.AccountKey.Decrypt() + assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusPlain, user3.FsConfig.AzBlobConfig.AccountKey.Status) + assert.Equal(t, u3.FsConfig.AzBlobConfig.AccountKey.Payload, user3.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, user3.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + + _, err = httpd.RemoveUser(user1, http.StatusOK) + assert.NoError(t, err) + _, err = httpd.RemoveUser(user2, http.StatusOK) + assert.NoError(t, err) + _, err = httpd.RemoveUser(user3, http.StatusOK) + assert.NoError(t, err) + + err = dataprovider.Close() + assert.NoError(t, err) + err = config.LoadConfig(configDir, "") + assert.NoError(t, err) + providerConf = config.GetProviderConf() + providerConf.CredentialsPath = credentialsPath + err = os.RemoveAll(credentialsPath) + assert.NoError(t, err) + err = dataprovider.Initialize(providerConf, configDir) + assert.NoError(t, err) +} + +func TestSecretObject(t *testing.T) { + s := vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "test data", + AdditionalData: "username", + } + require.True(t, s.IsValid()) + err := s.Encrypt() + require.NoError(t, err) + require.Equal(t, vfs.SecretStatusAES256GCM, s.Status) + require.NotEmpty(t, s.Payload) + require.NotEmpty(t, s.Key) + require.True(t, s.IsValid()) + err = s.Decrypt() + require.NoError(t, err) + require.Equal(t, vfs.SecretStatusPlain, s.Status) + require.Equal(t, "test data", s.Payload) + require.Empty(t, s.Key) + + oldFormat := "$aes$5b97e3a3324a2f53e2357483383367c0$0ed3132b584742ab217866219da633266782b69b13e50ebc6ddfb7c4fbf2f2a414c6d5f813" + s, err = vfs.GetSecretFromCompatString(oldFormat) + require.NoError(t, err) + require.True(t, s.IsValid()) + require.Equal(t, vfs.SecretStatusPlain, s.Status) + require.Equal(t, "test data", s.Payload) + require.Empty(t, s.Key) } func TestUpdateUserNoCredentials(t *testing.T) { @@ -2727,7 +2997,8 @@ func TestWebUserS3Mock(t *testing.T) { user.FsConfig.S3Config.Bucket = "test" user.FsConfig.S3Config.Region = "eu-west-1" user.FsConfig.S3Config.AccessKey = "access-key" - user.FsConfig.S3Config.AccessSecret = "access-secret" + user.FsConfig.S3Config.AccessSecret.Payload = "access-secret" + user.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain user.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?a=b" user.FsConfig.S3Config.StorageClass = "Standard" user.FsConfig.S3Config.KeyPrefix = "somedir/subdir/" @@ -2753,7 +3024,7 @@ func TestWebUserS3Mock(t *testing.T) { form.Set("s3_bucket", user.FsConfig.S3Config.Bucket) form.Set("s3_region", user.FsConfig.S3Config.Region) form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey) - form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret) + form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.Payload) form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass) form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint) form.Set("s3_key_prefix", user.FsConfig.S3Config.KeyPrefix) @@ -2800,9 +3071,46 @@ func TestWebUserS3Mock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.S3Config.UploadPartSize, user.FsConfig.S3Config.UploadPartSize) assert.Equal(t, updateUser.FsConfig.S3Config.UploadConcurrency, user.FsConfig.S3Config.UploadConcurrency) assert.Equal(t, 2, len(updateUser.Filters.FileExtensions)) - if !strings.HasPrefix(updateUser.FsConfig.S3Config.AccessSecret, "$aes$") { - t.Error("s3 access secret is not encrypted") - } + assert.Equal(t, vfs.SecretStatusAES256GCM, updateUser.FsConfig.S3Config.AccessSecret.Status) + assert.NotEmpty(t, updateUser.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, updateUser.FsConfig.S3Config.AccessSecret.AdditionalData) + // now check that a redacted password is not saved + form.Set("s3_access_secret", "[**redacted**] ") + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), &b) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr.Code) + req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + users = nil + err = render.DecodeJSON(rr.Body, &users) + assert.NoError(t, err) + assert.Equal(t, 1, len(users)) + lastUpdatedUser := users[0] + assert.Equal(t, vfs.SecretStatusAES256GCM, lastUpdatedUser.FsConfig.S3Config.AccessSecret.Status) + assert.Equal(t, updateUser.FsConfig.S3Config.AccessSecret.Payload, lastUpdatedUser.FsConfig.S3Config.AccessSecret.Payload) + assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.Key) + assert.Empty(t, lastUpdatedUser.FsConfig.S3Config.AccessSecret.AdditionalData) + // now clear credentials + form.Set("s3_access_key", "") + form.Set("s3_access_secret", "") + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), &b) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr.Code) + req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + users = nil + err = render.DecodeJSON(rr.Body, &users) + assert.NoError(t, err) + assert.Equal(t, 1, len(users)) + assert.True(t, users[0].FsConfig.S3Config.AccessSecret.IsEmpty()) + req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr.Code) @@ -2908,7 +3216,8 @@ func TestWebUserAzureBlobMock(t *testing.T) { user.FsConfig.Provider = dataprovider.AzureBlobFilesystemProvider user.FsConfig.AzBlobConfig.Container = "container" user.FsConfig.AzBlobConfig.AccountName = "aname" - user.FsConfig.AzBlobConfig.AccountKey = "access-skey" + user.FsConfig.AzBlobConfig.AccountKey.Payload = "access-skey" + user.FsConfig.AzBlobConfig.AccountKey.Status = vfs.SecretStatusPlain user.FsConfig.AzBlobConfig.Endpoint = "http://127.0.0.1:9000/path?b=c" user.FsConfig.AzBlobConfig.KeyPrefix = "somedir/subdir/" user.FsConfig.AzBlobConfig.UploadPartSize = 5 @@ -2933,7 +3242,7 @@ func TestWebUserAzureBlobMock(t *testing.T) { form.Set("fs_provider", "3") form.Set("az_container", user.FsConfig.AzBlobConfig.Container) form.Set("az_account_name", user.FsConfig.AzBlobConfig.AccountName) - form.Set("az_account_key", user.FsConfig.AzBlobConfig.AccountKey) + form.Set("az_account_key", user.FsConfig.AzBlobConfig.AccountKey.Payload) form.Set("az_sas_url", user.FsConfig.AzBlobConfig.SASURL) form.Set("az_endpoint", user.FsConfig.AzBlobConfig.Endpoint) form.Set("az_key_prefix", user.FsConfig.AzBlobConfig.KeyPrefix) @@ -2980,9 +3289,29 @@ func TestWebUserAzureBlobMock(t *testing.T) { assert.Equal(t, updateUser.FsConfig.AzBlobConfig.UploadPartSize, user.FsConfig.AzBlobConfig.UploadPartSize) assert.Equal(t, updateUser.FsConfig.AzBlobConfig.UploadConcurrency, user.FsConfig.AzBlobConfig.UploadConcurrency) assert.Equal(t, 2, len(updateUser.Filters.FileExtensions)) - if !strings.HasPrefix(updateUser.FsConfig.AzBlobConfig.AccountKey, "$aes$") { - t.Error("azure account secret is not encrypted") - } + assert.Equal(t, vfs.SecretStatusAES256GCM, updateUser.FsConfig.AzBlobConfig.AccountKey.Status) + assert.NotEmpty(t, updateUser.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, updateUser.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, updateUser.FsConfig.AzBlobConfig.AccountKey.AdditionalData) + // now check that a redacted password is not saved + form.Set("az_account_key", "[**redacted**] ") + b, contentType, _ = getMultipartFormData(form, "", "") + req, _ = http.NewRequest(http.MethodPost, webUserPath+"/"+strconv.FormatInt(user.ID, 10), &b) + req.Header.Set("Content-Type", contentType) + rr = executeRequest(req) + checkResponseCode(t, http.StatusSeeOther, rr.Code) + req, _ = http.NewRequest(http.MethodGet, userPath+"?limit=1&offset=0&order=ASC&username="+user.Username, nil) + rr = executeRequest(req) + checkResponseCode(t, http.StatusOK, rr.Code) + users = nil + err = render.DecodeJSON(rr.Body, &users) + assert.NoError(t, err) + assert.Equal(t, 1, len(users)) + lastUpdatedUser := users[0] + assert.Equal(t, vfs.SecretStatusAES256GCM, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.Status) + assert.Equal(t, updateUser.FsConfig.AzBlobConfig.AccountKey.Payload, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.Payload) + assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.Key) + assert.Empty(t, lastUpdatedUser.FsConfig.AzBlobConfig.AccountKey.AdditionalData) req, _ = http.NewRequest(http.MethodDelete, userPath+"/"+strconv.FormatInt(user.ID, 10), nil) rr = executeRequest(req) checkResponseCode(t, http.StatusOK, rr.Code) @@ -3211,9 +3540,3 @@ func getMultipartFormData(values url.Values, fileFieldName, filePath string) (by err := w.Close() return b, w.FormDataContentType(), err } - -type invalidBase64 []byte - -func (b invalidBase64) MarshalJSON() ([]byte, error) { - return []byte(`not base64`), nil -} diff --git a/httpd/internal_test.go b/httpd/internal_test.go index d50955a9..5a7b93c5 100644 --- a/httpd/internal_test.go +++ b/httpd/internal_test.go @@ -332,24 +332,36 @@ func TestCompareUserFsConfig(t *testing.T) { err = compareUserFsConfig(expected, actual) assert.Error(t, err) expected.FsConfig.S3Config.AccessKey = "" - actual.FsConfig.S3Config.AccessSecret = "access secret" + actual.FsConfig.S3Config.AccessSecret.Payload = "access secret" err = compareUserFsConfig(expected, actual) assert.Error(t, err) secret, _ := utils.EncryptData("access secret") - actual.FsConfig.S3Config.AccessSecret = "" - expected.FsConfig.S3Config.AccessSecret = secret + actual.FsConfig.S3Config.AccessSecret.Payload = "" + expected.FsConfig.S3Config.AccessSecret.Payload = secret err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) - actual.FsConfig.S3Config.AccessSecret = utils.RemoveDecryptionKey(secret) + "a" + expected.FsConfig.S3Config.AccessSecret.Payload = "test" + actual.FsConfig.S3Config.AccessSecret.Payload = "" err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.S3Config.AccessSecret = "test" - actual.FsConfig.S3Config.AccessSecret = "" + expected.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusPlain + actual.FsConfig.S3Config.AccessSecret.Status = vfs.SecretStatusAES256GCM err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.S3Config.AccessSecret = "" - actual.FsConfig.S3Config.AccessSecret = "" + actual.FsConfig.S3Config.AccessSecret.Payload = "payload" + actual.FsConfig.S3Config.AccessSecret.AdditionalData = "data" + err = compareUserFsConfig(expected, actual) + assert.Error(t, err) + actual.FsConfig.S3Config.AccessSecret.AdditionalData = "" + actual.FsConfig.S3Config.AccessSecret.Key = "key" + err = compareUserFsConfig(expected, actual) + assert.Error(t, err) + expected.FsConfig.S3Config.AccessSecret.Status = "" + expected.FsConfig.S3Config.AccessSecret.Payload = "" + actual.FsConfig.S3Config.AccessSecret.Status = "" + actual.FsConfig.S3Config.AccessSecret.Payload = "" + actual.FsConfig.S3Config.AccessSecret.AdditionalData = "" + actual.FsConfig.S3Config.AccessSecret.Key = "" expected.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/" err = compareUserFsConfig(expected, actual) assert.Error(t, err) @@ -403,10 +415,10 @@ func TestCompareUserAzureConfig(t *testing.T) { err = compareUserFsConfig(expected, actual) assert.Error(t, err) expected.FsConfig.AzBlobConfig.AccountName = "" - expected.FsConfig.AzBlobConfig.AccountKey = "akey" + expected.FsConfig.AzBlobConfig.AccountKey.Payload = "akey" err = compareUserFsConfig(expected, actual) assert.Error(t, err) - expected.FsConfig.AzBlobConfig.AccountKey = "" + expected.FsConfig.AzBlobConfig.AccountKey.Payload = "" expected.FsConfig.AzBlobConfig.Endpoint = "endpt" err = compareUserFsConfig(expected, actual) assert.Error(t, err) diff --git a/httpd/schema/openapi.yaml b/httpd/schema/openapi.yaml index 5237a4e4..45f80854 100644 --- a/httpd/schema/openapi.yaml +++ b/httpd/schema/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.0.3 info: title: SFTPGo description: 'SFTPGo REST API' - version: 2.0.3 + version: 2.1.0 servers: - url: /api/v1 @@ -11,6 +11,7 @@ security: paths: /healthz: get: + security: [] servers: - url : / tags: @@ -956,6 +957,22 @@ components: nullable: true description: maximum allowed size, as bytes, for a single file upload. The upload will be aborted if/when the size of the file being sent exceeds this limit. 0 means unlimited. This restriction does not apply for SSH system commands such as `git` and `rsync` description: Additional restrictions + Secret: + type: object + properties: + status: + type: string + enum: + - Plain + - AES-256-GCM + - Redacted + description: Set to "Plain" to add or update an existing secret, set to "Redacted" to preserve the existing value + payload: + type: string + key: + type: string + additional_data: + type: string S3Config: type: object properties: @@ -968,8 +985,7 @@ components: access_key: type: string access_secret: - type: string - description: the access secret is stored encrypted (AES-256-GCM) + $ref: '#/components/schemas/Secret' endpoint: type: string description: optional endpoint @@ -997,9 +1013,7 @@ components: type: string minLength: 1 credentials: - type: string - format: byte - description: Google Cloud Storage JSON credentials base64 encoded. This field must be populated only when adding/updating a user. It will be always omitted, since there are sensitive data, when you search/get users. The credentials will be stored in the configured "credentials_path" + $ref: '#/components/schemas/Secret' automatic_credentials: type: integer nullable: true @@ -1019,7 +1033,7 @@ components: required: - bucket nullable: true - description: Google Cloud Storage configuration details + description: Google Cloud Storage configuration details. The "credentials" field must be populated only when adding/updating a user. It will be always omitted, since there are sensitive data, when you search/get users AzureBlobFsConfig: type: object properties: @@ -1029,8 +1043,7 @@ components: type: string description: Storage Account Name, leave blank to use SAS URL account_key: - type: string - description: Storage Account Key leave blank to use SAS URL. The access key is stored encrypted (AES-256-GCM) + $ref: '#/components/schemas/Secret' sas_url: type: string description: Shared access signature URL, leave blank if using account/key diff --git a/httpd/web.go b/httpd/web.go index 6a0af1f0..b76d9925 100644 --- a/httpd/web.go +++ b/httpd/web.go @@ -39,6 +39,7 @@ const ( page500Body = "The server is unable to fulfill your request." defaultQueryLimit = 500 webDateTimeFormat = "2006-01-02 15:04:05" // YYYY-MM-DD HH:MM:SS + redactedSecret = "[**redacted**]" ) var ( @@ -81,7 +82,6 @@ type connectionsPage struct { type userPage struct { basePage - IsAdd bool User dataprovider.User RootPerms []string Error string @@ -89,6 +89,10 @@ type userPage struct { ValidSSHLoginMethods []string ValidProtocols []string RootDirPerms []string + RedactedSecret string + IsAdd bool + IsS3SecretEnc bool + IsAzSecretEnc bool } type folderPage struct { @@ -210,6 +214,9 @@ func renderAddUserPage(w http.ResponseWriter, user dataprovider.User, error stri ValidSSHLoginMethods: dataprovider.ValidSSHLoginMethods, ValidProtocols: dataprovider.ValidProtocols, RootDirPerms: user.GetPermissionsForPath("/"), + IsS3SecretEnc: user.FsConfig.S3Config.AccessSecret.IsEncrypted(), + IsAzSecretEnc: user.FsConfig.AzBlobConfig.AccountKey.IsEncrypted(), + RedactedSecret: redactedSecret, } renderTemplate(w, templateUser, data) } @@ -224,6 +231,9 @@ func renderUpdateUserPage(w http.ResponseWriter, user dataprovider.User, error s ValidSSHLoginMethods: dataprovider.ValidSSHLoginMethods, ValidProtocols: dataprovider.ValidProtocols, RootDirPerms: user.GetPermissionsForPath("/"), + IsS3SecretEnc: user.FsConfig.S3Config.AccessSecret.IsEncrypted(), + IsAzSecretEnc: user.FsConfig.AzBlobConfig.AccountKey.IsEncrypted(), + RedactedSecret: redactedSecret, } renderTemplate(w, templateUser, data) } @@ -420,6 +430,20 @@ func getFiltersFromUserPostFields(r *http.Request) dataprovider.UserFilters { return filters } +func getSecretFromFormField(r *http.Request, field string) vfs.Secret { + secret := vfs.Secret{ + Payload: r.Form.Get(field), + Status: vfs.SecretStatusPlain, + } + if strings.TrimSpace(secret.Payload) == redactedSecret { + secret.Status = vfs.SecretStatusRedacted + } + if strings.TrimSpace(secret.Payload) == "" { + secret.Status = "" + } + return secret +} + func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, error) { var fs dataprovider.Filesystem provider, err := strconv.Atoi(r.Form.Get("fs_provider")) @@ -431,7 +455,7 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er fs.S3Config.Bucket = r.Form.Get("s3_bucket") fs.S3Config.Region = r.Form.Get("s3_region") fs.S3Config.AccessKey = r.Form.Get("s3_access_key") - fs.S3Config.AccessSecret = r.Form.Get("s3_access_secret") + fs.S3Config.AccessSecret = getSecretFromFormField(r, "s3_access_secret") fs.S3Config.Endpoint = r.Form.Get("s3_endpoint") fs.S3Config.StorageClass = r.Form.Get("s3_storage_class") fs.S3Config.KeyPrefix = r.Form.Get("s3_key_prefix") @@ -468,12 +492,15 @@ func getFsConfigFromUserPostFields(r *http.Request) (dataprovider.Filesystem, er } return fs, err } - fs.GCSConfig.Credentials = fileBytes + fs.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: string(fileBytes), + } fs.GCSConfig.AutomaticCredentials = 0 } else if fs.Provider == dataprovider.AzureBlobFilesystemProvider { fs.AzBlobConfig.Container = r.Form.Get("az_container") fs.AzBlobConfig.AccountName = r.Form.Get("az_account_name") - fs.AzBlobConfig.AccountKey = r.Form.Get("az_account_key") + fs.AzBlobConfig.AccountKey = getSecretFromFormField(r, "az_account_key") fs.AzBlobConfig.SASURL = r.Form.Get("az_sas_url") fs.AzBlobConfig.Endpoint = r.Form.Get("az_endpoint") fs.AzBlobConfig.KeyPrefix = r.Form.Get("az_key_prefix") @@ -655,6 +682,12 @@ func handleWebUpdateUserPost(w http.ResponseWriter, r *http.Request) { if len(updatedUser.Password) == 0 { updatedUser.Password = user.Password } + if !updatedUser.FsConfig.S3Config.AccessSecret.IsPlain() && !updatedUser.FsConfig.S3Config.AccessSecret.IsEmpty() { + updatedUser.FsConfig.S3Config.AccessSecret = user.FsConfig.S3Config.AccessSecret + } + if !updatedUser.FsConfig.AzBlobConfig.AccountKey.IsPlain() && !updatedUser.FsConfig.AzBlobConfig.AccountKey.IsEmpty() { + updatedUser.FsConfig.AzBlobConfig.AccountKey = user.FsConfig.AzBlobConfig.AccountKey + } err = dataprovider.UpdateUser(updatedUser) if err == nil { if len(r.Form.Get("disconnect")) > 0 { diff --git a/service/service.go b/service/service.go index 8e64c608..61580903 100644 --- a/service/service.go +++ b/service/service.go @@ -2,7 +2,6 @@ package service import ( - "encoding/json" "fmt" "io/ioutil" "os" @@ -207,9 +206,7 @@ func (s *Service) loadInitialData() error { if err != nil { return fmt.Errorf("unable to read input file %#v: %v", s.LoadDataFrom, err) } - var dump dataprovider.BackupData - - err = json.Unmarshal(content, &dump) + dump, err := dataprovider.ParseDumpData(content) if err != nil { return fmt.Errorf("unable to parse file to restore %#v: %v", s.LoadDataFrom, err) } diff --git a/sftpd/sftpd_test.go b/sftpd/sftpd_test.go index b18de796..5460e580 100644 --- a/sftpd/sftpd_test.go +++ b/sftpd/sftpd_test.go @@ -1312,7 +1312,10 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { u := getTestUser(usePubKey) u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "testbucket" - u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: `{ "type": "service_account" }`, + } providerConf := config.GetProviderConf() providerConf.PreferDatabaseCredentials = true @@ -1333,9 +1336,12 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.Key) - _, err = os.Stat(credentialsFile) - assert.Error(t, err) + assert.NoFileExists(t, credentialsFile) client, err := getSftpClient(user, usePubKey) if assert.NoError(t, err) { @@ -1358,7 +1364,10 @@ func TestLoginInvalidFs(t *testing.T) { u := getTestUser(usePubKey) u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "invalid JSON for credentials", + } user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) diff --git a/templates/user.html b/templates/user.html index 93449e9f..dccca447 100644 --- a/templates/user.html +++ b/templates/user.html @@ -337,7 +337,7 @@
+ value="{{if .IsS3SecretEnc}}{{.RedactedSecret}}{{else}}{{.User.FsConfig.S3Config.AccessSecret.Payload}}{{end}}" maxlength="1000">
@@ -345,7 +345,7 @@
+ value="{{.User.FsConfig.S3Config.StorageClass}}" maxlength="255">
@@ -448,7 +448,7 @@
+ value="{{if .IsAzSecretEnc}}{{.RedactedSecret}}{{else}}{{.User.FsConfig.AzBlobConfig.AccountKey.Payload}}{{end}}" maxlength="1000">
diff --git a/utils/utils.go b/utils/utils.go index b4e270bf..5d4f4b16 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -193,6 +193,9 @@ func DecryptData(data string) (string, error) { return result, err } nonceSize := gcm.NonceSize() + if len(encrypted) < nonceSize { + return result, errors.New("malformed ciphertext") + } nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:] plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) if err != nil { diff --git a/vfs/azblobfs.go b/vfs/azblobfs.go index 5a8262fb..61a094ee 100644 --- a/vfs/azblobfs.go +++ b/vfs/azblobfs.go @@ -24,7 +24,6 @@ import ( "github.com/drakkan/sftpgo/logger" "github.com/drakkan/sftpgo/metrics" - "github.com/drakkan/sftpgo/utils" "github.com/drakkan/sftpgo/version" ) @@ -61,12 +60,11 @@ func NewAzBlobFs(connectionID, localTempDir string, config AzBlobFsConfig) (Fs, if err := ValidateAzBlobFsConfig(&fs.config); err != nil { return fs, err } - if fs.config.AccountKey != "" { - accountKey, err := utils.DecryptData(fs.config.AccountKey) + if fs.config.AccountKey.IsEncrypted() { + err := fs.config.AccountKey.Decrypt() if err != nil { return fs, err } - fs.config.AccountKey = accountKey } fs.setConfigDefaults() @@ -106,7 +104,7 @@ func NewAzBlobFs(connectionID, localTempDir string, config AzBlobFsConfig) (Fs, return fs, nil } - credential, err := azblob.NewSharedKeyCredential(fs.config.AccountName, fs.config.AccountKey) + credential, err := azblob.NewSharedKeyCredential(fs.config.AccountName, fs.config.AccountKey.Payload) if err != nil { return fs, fmt.Errorf("invalid credentials: %v", err) } diff --git a/vfs/gcsfs.go b/vfs/gcsfs.go index e5d80c53..d09d24b9 100644 --- a/vfs/gcsfs.go +++ b/vfs/gcsfs.go @@ -4,9 +4,11 @@ package vfs import ( "context" + "encoding/json" "errors" "fmt" "io" + "io/ioutil" "mime" "net/http" "os" @@ -60,10 +62,28 @@ func NewGCSFs(connectionID, localTempDir string, config GCSFsConfig) (Fs, error) ctx := context.Background() if fs.config.AutomaticCredentials > 0 { fs.svc, err = storage.NewClient(ctx) - } else if len(fs.config.Credentials) > 0 { - fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON(fs.config.Credentials)) + } else if fs.config.Credentials.IsEncrypted() { + err = fs.config.Credentials.Decrypt() + if err != nil { + return fs, err + } + fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON([]byte(fs.config.Credentials.Payload))) } else { - fs.svc, err = storage.NewClient(ctx, option.WithCredentialsFile(fs.config.CredentialFile)) + var creds []byte + creds, err = ioutil.ReadFile(fs.config.CredentialFile) + if err != nil { + return fs, err + } + secret := &Secret{} + err = json.Unmarshal(creds, secret) + if err != nil { + return fs, err + } + err = secret.Decrypt() + if err != nil { + return fs, err + } + fs.svc, err = storage.NewClient(ctx, option.WithCredentialsJSON([]byte(secret.Payload))) } return fs, err } diff --git a/vfs/s3fs.go b/vfs/s3fs.go index 6b275306..8bdd309c 100644 --- a/vfs/s3fs.go +++ b/vfs/s3fs.go @@ -60,13 +60,12 @@ func NewS3Fs(connectionID, localTempDir string, config S3FsConfig) (Fs, error) { awsConfig.WithRegion(fs.config.Region) } - if fs.config.AccessSecret != "" { - accessSecret, err := utils.DecryptData(fs.config.AccessSecret) + if fs.config.AccessSecret.IsEncrypted() { + err := fs.config.AccessSecret.Decrypt() if err != nil { return fs, err } - fs.config.AccessSecret = accessSecret - awsConfig.Credentials = credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret, "") + awsConfig.Credentials = credentials.NewStaticCredentials(fs.config.AccessKey, fs.config.AccessSecret.Payload, "") } if fs.config.Endpoint != "" { diff --git a/vfs/secret.go b/vfs/secret.go new file mode 100644 index 00000000..9d16f088 --- /dev/null +++ b/vfs/secret.go @@ -0,0 +1,209 @@ +package vfs + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + + "github.com/drakkan/sftpgo/utils" +) + +// SecretStatus defines the statuses of a Secret object +type SecretStatus = string + +const ( + // SecretStatusPlain means the secret is in plain text and must be encrypted + SecretStatusPlain SecretStatus = "Plain" + // SecretStatusAES256GCM means the secret is encrypted using AES-256-GCM + SecretStatusAES256GCM SecretStatus = "AES-256-GCM" + // SecretStatusRedacted means the secret is redacted + SecretStatusRedacted SecretStatus = "Redacted" +) + +var ( + errWrongSecretStatus = errors.New("wrong secret status") + errMalformedCiphertext = errors.New("malformed ciphertext") + errInvalidSecret = errors.New("invalid secret") + validSecretStatuses = []string{SecretStatusPlain, SecretStatusAES256GCM, SecretStatusRedacted} +) + +// Secret defines the struct used to store confidential data +type Secret struct { + Status SecretStatus `json:"status,omitempty"` + Payload string `json:"payload,omitempty"` + Key string `json:"key,omitempty"` + AdditionalData string `json:"additional_data,omitempty"` +} + +// GetSecretFromCompatString returns a secret from the previous format +func GetSecretFromCompatString(secret string) (Secret, error) { + s := Secret{} + plain, err := utils.DecryptData(secret) + if err != nil { + return s, errMalformedCiphertext + } + s.Status = SecretStatusPlain + s.Payload = plain + return s, nil +} + +// IsEncrypted returns true if the secret is encrypted +// This isn't a pointer receiver because we don't want to pass +// a pointer to html template +func (s *Secret) IsEncrypted() bool { + return s.Status == SecretStatusAES256GCM +} + +// IsPlain returns true if the secret is in plain text +func (s *Secret) IsPlain() bool { + return s.Status == SecretStatusPlain +} + +// IsRedacted returns true if the secret is redacted +func (s *Secret) IsRedacted() bool { + return s.Status == SecretStatusRedacted +} + +// IsEmpty returns true if all fields are empty +func (s *Secret) IsEmpty() bool { + if s.Status != "" { + return false + } + if s.Payload != "" { + return false + } + if s.Key != "" { + return false + } + if s.AdditionalData != "" { + return false + } + return true +} + +// IsValid returns true if the secret is not empty and valid +func (s *Secret) IsValid() bool { + if !s.IsValidInput() { + return false + } + if s.Status == SecretStatusAES256GCM { + if len(s.Key) != 64 { + return false + } + } + return true +} + +// IsValidInput returns true if the secret is a valid user input +func (s *Secret) IsValidInput() bool { + if !utils.IsStringInSlice(s.Status, validSecretStatuses) { + return false + } + if s.Payload == "" { + return false + } + return true +} + +// Hide hides info to decrypt data +func (s *Secret) Hide() { + s.Key = "" + s.AdditionalData = "" +} + +// deriveKey is a weak method of deriving a key but it is still better than using the key as it is. +// We should use a KMS in future +func (s *Secret) deriveKey(key []byte) []byte { + var combined []byte + combined = append(combined, key...) + if s.AdditionalData != "" { + combined = append(combined, []byte(s.AdditionalData)...) + } + combined = append(combined, key...) + hash := sha256.Sum256(combined) + return hash[:] +} + +// Encrypt encrypts a plain text Secret object +func (s *Secret) Encrypt() error { + if s.Payload == "" { + return errInvalidSecret + } + switch s.Status { + case SecretStatusPlain: + key := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, key); err != nil { + return err + } + block, err := aes.NewCipher(s.deriveKey(key)) + if err != nil { + return err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err = io.ReadFull(rand.Reader, nonce); err != nil { + return err + } + var aad []byte + if s.AdditionalData != "" { + aad = []byte(s.AdditionalData) + } + ciphertext := gcm.Seal(nonce, nonce, []byte(s.Payload), aad) + s.Key = hex.EncodeToString(key) + s.Payload = hex.EncodeToString(ciphertext) + s.Status = SecretStatusAES256GCM + return nil + default: + return errWrongSecretStatus + } +} + +// Decrypt decrypts a Secret object +func (s *Secret) Decrypt() error { + switch s.Status { + case SecretStatusAES256GCM: + encrypted, err := hex.DecodeString(s.Payload) + if err != nil { + return err + } + key, err := hex.DecodeString(s.Key) + if err != nil { + return err + } + block, err := aes.NewCipher(s.deriveKey(key)) + if err != nil { + return err + } + gcm, err := cipher.NewGCM(block) + if err != nil { + return err + } + nonceSize := gcm.NonceSize() + if len(encrypted) < nonceSize { + return errMalformedCiphertext + } + nonce, ciphertext := encrypted[:nonceSize], encrypted[nonceSize:] + var aad []byte + if s.AdditionalData != "" { + aad = []byte(s.AdditionalData) + } + plaintext, err := gcm.Open(nil, nonce, ciphertext, aad) + if err != nil { + return err + } + s.Status = SecretStatusPlain + s.Payload = string(plaintext) + s.Key = "" + s.AdditionalData = "" + return nil + default: + return errWrongSecretStatus + } +} diff --git a/vfs/vfs.go b/vfs/vfs.go index e60a68cd..1842955b 100644 --- a/vfs/vfs.go +++ b/vfs/vfs.go @@ -113,7 +113,7 @@ type S3FsConfig struct { KeyPrefix string `json:"key_prefix,omitempty"` Region string `json:"region,omitempty"` AccessKey string `json:"access_key,omitempty"` - AccessSecret string `json:"access_secret,omitempty"` + AccessSecret Secret `json:"access_secret,omitempty"` Endpoint string `json:"endpoint,omitempty"` StorageClass string `json:"storage_class,omitempty"` // The buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB, @@ -137,9 +137,10 @@ type GCSFsConfig struct { // folder. The prefix, if not empty, must not start with "/" and must // end with "/". // If empty the whole bucket contents will be available - KeyPrefix string `json:"key_prefix,omitempty"` - CredentialFile string `json:"-"` - Credentials []byte `json:"credentials,omitempty"` + KeyPrefix string `json:"key_prefix,omitempty"` + CredentialFile string `json:"-"` + Credentials Secret `json:"credentials,omitempty"` + // 0 explicit, 1 automatic AutomaticCredentials int `json:"automatic_credentials,omitempty"` StorageClass string `json:"storage_class,omitempty"` } @@ -151,7 +152,7 @@ type AzBlobFsConfig struct { AccountName string `json:"account_name,omitempty"` // Storage Account Key leave blank to use SAS URL. // The access key is stored encrypted (AES-256-GCM) - AccountKey string `json:"account_key,omitempty"` + AccountKey Secret `json:"account_key,omitempty"` // Optional endpoint. Default is "blob.core.windows.net". // If you use the emulator the endpoint must include the protocol, // for example "http://127.0.0.1:10000" @@ -235,20 +236,33 @@ func IsLocalOsFs(fs Fs) bool { return fs.Name() == osFsName } -// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error -func ValidateS3FsConfig(config *S3FsConfig) error { - if len(config.Bucket) == 0 { - return errors.New("bucket cannot be empty") - } - if len(config.Region) == 0 { - return errors.New("region cannot be empty") - } - if len(config.AccessKey) == 0 && len(config.AccessSecret) > 0 { +func checkS3Credentials(config *S3FsConfig) error { + if config.AccessKey == "" && !config.AccessSecret.IsEmpty() { return errors.New("access_key cannot be empty with access_secret not empty") } - if len(config.AccessSecret) == 0 && len(config.AccessKey) > 0 { + if config.AccessSecret.IsEmpty() && config.AccessKey != "" { return errors.New("access_secret cannot be empty with access_key not empty") } + if config.AccessSecret.IsEncrypted() && !config.AccessSecret.IsValid() { + return errors.New("invalid encrypted access_secret") + } + if !config.AccessSecret.IsEmpty() && !config.AccessSecret.IsValidInput() { + return errors.New("invalid access_secret") + } + return nil +} + +// ValidateS3FsConfig returns nil if the specified s3 config is valid, otherwise an error +func ValidateS3FsConfig(config *S3FsConfig) error { + if config.Bucket == "" { + return errors.New("bucket cannot be empty") + } + if config.Region == "" { + return errors.New("region cannot be empty") + } + if err := checkS3Credentials(config); err != nil { + return err + } if config.KeyPrefix != "" { if strings.HasPrefix(config.KeyPrefix, "/") { return errors.New("key_prefix cannot start with /") @@ -281,7 +295,10 @@ func ValidateGCSFsConfig(config *GCSFsConfig, credentialsFilePath string) error config.KeyPrefix += "/" } } - if len(config.Credentials) == 0 && config.AutomaticCredentials == 0 { + if config.Credentials.IsEncrypted() && !config.Credentials.IsValid() { + return errors.New("invalid encrypted credentials") + } + if !config.Credentials.IsValidInput() && config.AutomaticCredentials == 0 { fi, err := os.Stat(credentialsFilePath) if err != nil { return fmt.Errorf("invalid credentials %v", err) @@ -302,8 +319,11 @@ func ValidateAzBlobFsConfig(config *AzBlobFsConfig) error { if config.Container == "" { return errors.New("container cannot be empty") } - if config.AccountName == "" || config.AccountKey == "" { - return errors.New("credentials cannot be empty") + if config.AccountName == "" || !config.AccountKey.IsValidInput() { + return errors.New("credentials cannot be empty or invalid") + } + if config.AccountKey.IsEncrypted() && !config.AccountKey.IsValid() { + return errors.New("invalid encrypted account_key") } if config.KeyPrefix != "" { if strings.HasPrefix(config.KeyPrefix, "/") { diff --git a/webdavd/webdavd_test.go b/webdavd/webdavd_test.go index 8a6eca21..6f9838d1 100644 --- a/webdavd/webdavd_test.go +++ b/webdavd/webdavd_test.go @@ -861,7 +861,10 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte(`{ "type": "service_account" }`) + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: `{ "type": "service_account" }`, + } providerConf := config.GetProviderConf() providerConf.PreferDatabaseCredentials = true @@ -882,9 +885,12 @@ func TestLoginWithDatabaseCredentials(t *testing.T) { user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err) + assert.Equal(t, vfs.SecretStatusAES256GCM, user.FsConfig.GCSConfig.Credentials.Status) + assert.NotEmpty(t, user.FsConfig.GCSConfig.Credentials.Payload) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.AdditionalData) + assert.Empty(t, user.FsConfig.GCSConfig.Credentials.Key) - _, err = os.Stat(credentialsFile) - assert.Error(t, err) + assert.NoFileExists(t, credentialsFile) client := getWebDavClient(user) @@ -906,7 +912,10 @@ func TestLoginInvalidFs(t *testing.T) { u := getTestUser() u.FsConfig.Provider = dataprovider.GCSFilesystemProvider u.FsConfig.GCSConfig.Bucket = "test" - u.FsConfig.GCSConfig.Credentials = []byte("invalid JSON for credentials") + u.FsConfig.GCSConfig.Credentials = vfs.Secret{ + Status: vfs.SecretStatusPlain, + Payload: "invalid JSON for credentials", + } user, _, err := httpd.AddUser(u, http.StatusOK) assert.NoError(t, err)