Mirrors/sftpgo
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2025-12-07 14:50:55 +03:00
Code Issues Packages Projects Releases Wiki Activity

squash database migrations, remove compat data provider code

Browse Source
This commit is contained in:
Nicola Murino
2021-06-19 09:03:20 +02:00
parent 62744e081b
commit d5b42f72e2
8 changed files with 46 additions and 1046 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/revertprovider.go
View File

@@ -26,8 +26,8 @@ Please take a look at the usage below to customize the options.`,
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
logger.DisableLogger() logger.DisableLogger()
logger.EnableConsoleLogger(zerolog.DebugLevel) logger.EnableConsoleLogger(zerolog.DebugLevel)
if revertProviderTargetVersion != 8 { if revertProviderTargetVersion != 10 {
logger.WarnToConsole("Unsupported target version, 8 is the only supported one") logger.WarnToConsole("Unsupported target version, 10 is the only supported one")
os.Exit(1) os.Exit(1)
} }
configDir = utils.CleanDirInput(configDir) configDir = utils.CleanDirInput(configDir)
@@ -57,7 +57,7 @@ Please take a look at the usage below to customize the options.`,
func init() { func init() {
addConfigFlags(revertProviderCmd) addConfigFlags(revertProviderCmd)
revertProviderCmd.Flags().IntVar(&revertProviderTargetVersion, "to-version", 0, `8 means the version supported in v2.0.x`) revertProviderCmd.Flags().IntVar(&revertProviderTargetVersion, "to-version", 10, `10 means the version supported in v2.1.x`)
revertProviderCmd.MarkFlagRequired("to-version") //nolint:errcheck revertProviderCmd.MarkFlagRequired("to-version") //nolint:errcheck
rootCmd.AddCommand(revertProviderCmd) rootCmd.AddCommand(revertProviderCmd)

329
dataprovider/bolt.go
View File

@@ -855,13 +855,11 @@ func (p *BoltProvider) migrateDatabase() error {
case version == boltDatabaseVersion: case version == boltDatabaseVersion:
providerLog(logger.LevelDebug, "bolt database is up to date, current version: %v", version) providerLog(logger.LevelDebug, "bolt database is up to date, current version: %v", version)
return ErrNoInitRequired return ErrNoInitRequired
case version < 6: case version < 10:
err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version) err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err) providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err) logger.ErrorToConsole("%v", err)
return err return err
case version == 6:
return updateBoltDatabaseFrom6To10(p.dbHandle)
default: default:
if version > boltDatabaseVersion { if version > boltDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version,
@@ -879,15 +877,9 @@ func (p *BoltProvider) revertDatabase(targetVersion int) error {
if err != nil { if err != nil {
return err return err
} }
if targetVersion >= 8 {
targetVersion = 6
}
if dbVersion.Version == targetVersion { if dbVersion.Version == targetVersion {
return errors.New("current version match target version, nothing to do") return errors.New("current version match target version, nothing to do")
} }
if dbVersion.Version == 10 {
return downgradeBoltDatabaseFrom10To6(p.dbHandle)
}
return errors.New("the current version cannot be reverted") return errors.New("the current version cannot be reverted")
} }
@@ -1034,7 +1026,7 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) {
v := bucket.Get(dbVersionKey) v := bucket.Get(dbVersionKey)
if v == nil { if v == nil {
dbVersion = schemaVersion{ dbVersion = schemaVersion{
Version: 6, Version: 10,
} }
return nil return nil
} }
@@ -1043,7 +1035,7 @@ func getBoltDatabaseVersion(dbHandle *bolt.DB) (schemaVersion, error) {
return dbVersion, err return dbVersion, err
} }
func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error { /*func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error {
err := dbHandle.Update(func(tx *bolt.Tx) error { err := dbHandle.Update(func(tx *bolt.Tx) error {
bucket := tx.Bucket(dbVersionBucket) bucket := tx.Bucket(dbVersionBucket)
if bucket == nil { if bucket == nil {
@@ -1059,317 +1051,4 @@ func updateBoltDatabaseVersion(dbHandle *bolt.DB, version int) error {
return bucket.Put(dbVersionKey, buf) return bucket.Put(dbVersionKey, buf)
}) })
return err return err
} }*/
func updateBoltDatabaseFrom6To10(dbHandle *bolt.DB) error {
logger.InfoToConsole("updating database version: 6 -> 10")
providerLog(logger.LevelInfo, "updating database version: 6 -> 10")
if err := boltUpdateV7Folders(dbHandle); err != nil {
return err
}
if err := boltUpdateV7Users(dbHandle); err != nil {
return err
}
return updateBoltDatabaseVersion(dbHandle, 10)
}
func downgradeBoltDatabaseFrom10To6(dbHandle *bolt.DB) error {
logger.InfoToConsole("downgrading database version: 10 -> 6")
providerLog(logger.LevelInfo, "downgrading database version: 10 -> 6")
if err := boltDowngradeV7Folders(dbHandle); err != nil {
return err
}
if err := boltDowngradeV7Users(dbHandle); err != nil {
return err
}
return updateBoltDatabaseVersion(dbHandle, 6)
}
func boltUpdateV7Folders(dbHandle *bolt.DB) error {
var folders []map[string]interface{}
err := dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := getFoldersBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var folderMap map[string]interface{}
err = json.Unmarshal(v, &folderMap)
if err != nil {
return err
}
fsBytes, err := json.Marshal(folderMap["filesystem"])
if err != nil {
continue
}
var compatFsConfig compatFilesystemV9
err = json.Unmarshal(fsBytes, &compatFsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v9 fsconfig for folder %#v, is it already migrated?", folderMap["name"])
continue
}
if compatFsConfig.AzBlobConfig.SASURL != "" {
folder := vfs.BaseVirtualFolder{
Name: folderMap["name"].(string),
}
fsConfig, err := convertFsConfigFromV9(compatFsConfig, folder.GetEncrytionAdditionalData())
if err != nil {
return err
}
folderMap["filesystem"] = fsConfig
folders = append(folders, folderMap)
}
}
return err
})
if err != nil {
return err
}
return dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getFoldersBucket(tx)
if err != nil {
return err
}
for _, folder := range folders {
buf, err := json.Marshal(folder)
if err != nil {
return err
}
err = bucket.Put([]byte(folder["name"].(string)), buf)
if err != nil {
return err
}
}
return nil
})
}
//nolint:gocyclo
func boltUpdateV7Users(dbHandle *bolt.DB) error {
var users []map[string]interface{}
err := dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := getUsersBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var userMap map[string]interface{}
err = json.Unmarshal(v, &userMap)
if err != nil {
return err
}
fsBytes, err := json.Marshal(userMap["filesystem"])
if err != nil {
continue
}
foldersBytes, err := json.Marshal(userMap["virtual_folders"])
if err != nil {
continue
}
var compatFsConfig compatFilesystemV9
err = json.Unmarshal(fsBytes, &compatFsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v9 fsconfig for user %#v, is it already migrated?", userMap["name"])
continue
}
var compatFolders []compatFolderV9
err = json.Unmarshal(foldersBytes, &compatFolders)
if err != nil {
logger.WarnToConsole("failed to unmarshal v9 folders for user %#v, is it already migrated?", userMap["name"])
continue
}
toConvert := false
for idx := range compatFolders {
f := &compatFolders[idx]
if f.FsConfig.AzBlobConfig.SASURL != "" {
f.FsConfig.AzBlobConfig = compatAzBlobFsConfigV9{}
toConvert = true
}
}
if compatFsConfig.AzBlobConfig.SASURL != "" {
user := User{
Username: userMap["username"].(string),
}
fsConfig, err := convertFsConfigFromV9(compatFsConfig, user.GetEncrytionAdditionalData())
if err != nil {
return err
}
userMap["filesystem"] = fsConfig
toConvert = true
}
if toConvert {
userMap["virtual_folders"] = compatFolders
users = append(users, userMap)
}
}
return err
})
if err != nil {
return err
}
return dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getUsersBucket(tx)
if err != nil {
return err
}
for _, user := range users {
buf, err := json.Marshal(user)
if err != nil {
return err
}
err = bucket.Put([]byte(user["username"].(string)), buf)
if err != nil {
return err
}
}
return nil
})
}
//nolint:dupl
func boltDowngradeV7Folders(dbHandle *bolt.DB) error {
var folders []map[string]interface{}
err := dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := getFoldersBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var folderMap map[string]interface{}
err = json.Unmarshal(v, &folderMap)
if err != nil {
return err
}
fsBytes, err := json.Marshal(folderMap["filesystem"])
if err != nil {
continue
}
var fsConfig vfs.Filesystem
err = json.Unmarshal(fsBytes, &fsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v10 fsconfig for folder %#v, is it already migrated?", folderMap["name"])
continue
}
if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() {
fsV9, err := convertFsConfigToV9(fsConfig)
if err != nil {
return err
}
folderMap["filesystem"] = fsV9
folders = append(folders, folderMap)
}
}
return err
})
if err != nil {
return err
}
return dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getFoldersBucket(tx)
if err != nil {
return err
}
for _, folder := range folders {
buf, err := json.Marshal(folder)
if err != nil {
return err
}
err = bucket.Put([]byte(folder["name"].(string)), buf)
if err != nil {
return err
}
}
return nil
})
}
//nolint:dupl,gocyclo
func boltDowngradeV7Users(dbHandle *bolt.DB) error {
var users []map[string]interface{}
err := dbHandle.View(func(tx *bolt.Tx) error {
bucket, err := getUsersBucket(tx)
if err != nil {
return err
}
cursor := bucket.Cursor()
for k, v := cursor.First(); k != nil; k, v = cursor.Next() {
var userMap map[string]interface{}
err = json.Unmarshal(v, &userMap)
if err != nil {
return err
}
fsBytes, err := json.Marshal(userMap["filesystem"])
if err != nil {
continue
}
foldersBytes, err := json.Marshal(userMap["virtual_folders"])
if err != nil {
continue
}
var fsConfig vfs.Filesystem
err = json.Unmarshal(fsBytes, &fsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v10 fsconfig for user %#v, is it already migrated?", userMap["username"])
continue
}
var folders []vfs.VirtualFolder
err = json.Unmarshal(foldersBytes, &folders)
if err != nil {
logger.WarnToConsole("failed to unmarshal v9 folders for user %#v, is it already migrated?", userMap["name"])
continue
}
toConvert := false
for idx := range folders {
f := &folders[idx]
f.FsConfig.AzBlobConfig = vfs.AzBlobFsConfig{}
toConvert = true
}
if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() {
fsV9, err := convertFsConfigToV9(fsConfig)
if err != nil {
return err
}
userMap["filesystem"] = fsV9
toConvert = true
}
if toConvert {
userMap["virtual_folders"] = folders
users = append(users, userMap)
}
}
return err
})
if err != nil {
return err
}
return dbHandle.Update(func(tx *bolt.Tx) error {
bucket, err := getUsersBucket(tx)
if err != nil {
return err
}
for _, user := range users {
buf, err := json.Marshal(user)
if err != nil {
return err
}
err = bucket.Put([]byte(user["username"].(string)), buf)
if err != nil {
return err
}
}
return nil
})
}

118
dataprovider/compat.go
View File

@@ -1,118 +0,0 @@
package dataprovider
import (
"github.com/drakkan/sftpgo/kms"
"github.com/drakkan/sftpgo/vfs"
)
type compatAzBlobFsConfigV9 struct {
Container string `json:"container,omitempty"`
AccountName string `json:"account_name,omitempty"`
AccountKey *kms.Secret `json:"account_key,omitempty"`
Endpoint string `json:"endpoint,omitempty"`
SASURL string `json:"sas_url,omitempty"`
KeyPrefix string `json:"key_prefix,omitempty"`
UploadPartSize int64 `json:"upload_part_size,omitempty"`
UploadConcurrency int `json:"upload_concurrency,omitempty"`
UseEmulator bool `json:"use_emulator,omitempty"`
AccessTier string `json:"access_tier,omitempty"`
}
type compatFilesystemV9 struct {
Provider vfs.FilesystemProvider `json:"provider"`
S3Config vfs.S3FsConfig `json:"s3config,omitempty"`
GCSConfig vfs.GCSFsConfig `json:"gcsconfig,omitempty"`
AzBlobConfig compatAzBlobFsConfigV9 `json:"azblobconfig,omitempty"`
CryptConfig vfs.CryptFsConfig `json:"cryptconfig,omitempty"`
SFTPConfig vfs.SFTPFsConfig `json:"sftpconfig,omitempty"`
}
type compatBaseFolderV9 struct {
ID int64 `json:"id"`
Name string `json:"name"`
MappedPath string `json:"mapped_path,omitempty"`
Description string `json:"description,omitempty"`
UsedQuotaSize int64 `json:"used_quota_size"`
UsedQuotaFiles int `json:"used_quota_files"`
LastQuotaUpdate int64 `json:"last_quota_update"`
Users []string `json:"users,omitempty"`
FsConfig compatFilesystemV9 `json:"filesystem"`
}
type compatFolderV9 struct {
compatBaseFolderV9
VirtualPath string `json:"virtual_path"`
QuotaSize int64 `json:"quota_size"`
QuotaFiles int `json:"quota_files"`
}
type compatUserV9 struct {
ID int64 `json:"id"`
Username string `json:"username"`
FsConfig compatFilesystemV9 `json:"filesystem"`
}
func convertFsConfigFromV9(compatFs compatFilesystemV9, aead string) (vfs.Filesystem, error) {
fsConfig := vfs.Filesystem{
Provider: compatFs.Provider,
S3Config: compatFs.S3Config,
GCSConfig: compatFs.GCSConfig,
CryptConfig: compatFs.CryptConfig,
SFTPConfig: compatFs.SFTPConfig,
}
azSASURL := kms.NewEmptySecret()
if compatFs.Provider == vfs.AzureBlobFilesystemProvider && compatFs.AzBlobConfig.SASURL != "" {
azSASURL = kms.NewPlainSecret(compatFs.AzBlobConfig.SASURL)
}
if compatFs.AzBlobConfig.AccountKey == nil {
compatFs.AzBlobConfig.AccountKey = kms.NewEmptySecret()
}
fsConfig.AzBlobConfig = vfs.AzBlobFsConfig{
Container: compatFs.AzBlobConfig.Container,
AccountName: compatFs.AzBlobConfig.AccountName,
AccountKey: compatFs.AzBlobConfig.AccountKey,
Endpoint: compatFs.AzBlobConfig.Endpoint,
SASURL: azSASURL,
KeyPrefix: compatFs.AzBlobConfig.KeyPrefix,
UploadPartSize: compatFs.AzBlobConfig.UploadPartSize,
UploadConcurrency: compatFs.AzBlobConfig.UploadConcurrency,
UseEmulator: compatFs.AzBlobConfig.UseEmulator,
AccessTier: compatFs.AzBlobConfig.AccessTier,
}
err := fsConfig.AzBlobConfig.EncryptCredentials(aead)
return fsConfig, err
}
func convertFsConfigToV9(fs vfs.Filesystem) (compatFilesystemV9, error) {
azSASURL := ""
if fs.Provider == vfs.AzureBlobFilesystemProvider {
if fs.AzBlobConfig.SASURL != nil && fs.AzBlobConfig.SASURL.IsEncrypted() {
err := fs.AzBlobConfig.SASURL.Decrypt()
if err != nil {
return compatFilesystemV9{}, err
}
azSASURL = fs.AzBlobConfig.SASURL.GetPayload()
}
}
azFsCompat := compatAzBlobFsConfigV9{
Container: fs.AzBlobConfig.Container,
AccountName: fs.AzBlobConfig.AccountName,
AccountKey: fs.AzBlobConfig.AccountKey,
Endpoint: fs.AzBlobConfig.Endpoint,
SASURL: azSASURL,
KeyPrefix: fs.AzBlobConfig.KeyPrefix,
UploadPartSize: fs.AzBlobConfig.UploadPartSize,
UploadConcurrency: fs.AzBlobConfig.UploadConcurrency,
UseEmulator: fs.AzBlobConfig.UseEmulator,
AccessTier: fs.AzBlobConfig.AccessTier,
}
fsV9 := compatFilesystemV9{
Provider: fs.Provider,
S3Config: fs.S3Config,
GCSConfig: fs.GCSConfig,
AzBlobConfig: azFsCompat,
CryptConfig: fs.CryptConfig,
SFTPConfig: fs.SFTPConfig,
}
return fsV9, nil
}

87
dataprovider/mysql.go
View File

@@ -22,13 +22,13 @@ import (
const ( const (
mysqlInitialSQL = "CREATE TABLE `{{schema_version}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `version` integer NOT NULL);" + mysqlInitialSQL = "CREATE TABLE `{{schema_version}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `version` integer NOT NULL);" +
"CREATE TABLE `{{admins}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `username` varchar(255) NOT NULL UNIQUE, " + "CREATE TABLE `{{admins}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `username` varchar(255) NOT NULL UNIQUE, " +
"`password` varchar(255) NOT NULL, `email` varchar(255) NULL, `status` integer NOT NULL, `permissions` longtext NOT NULL, " + "`description` varchar(512) NULL, `password` varchar(255) NOT NULL, `email` varchar(255) NULL, `status` integer NOT NULL, " +
"`filters` longtext NULL, `additional_info` longtext NULL);" + "`permissions` longtext NOT NULL, `filters` longtext NULL, `additional_info` longtext NULL);" +
"CREATE TABLE `{{folders}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `name` varchar(255) NOT NULL UNIQUE, " + "CREATE TABLE `{{folders}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `name` varchar(255) NOT NULL UNIQUE, " +
"`path` varchar(512) NULL, `used_quota_size` bigint NOT NULL, `used_quota_files` integer NOT NULL, " + "`description` varchar(512) NULL, `path` varchar(512) NULL, `used_quota_size` bigint NOT NULL, " +
"`last_quota_update` bigint NOT NULL);" + "`used_quota_files` integer NOT NULL, `last_quota_update` bigint NOT NULL, `filesystem` longtext NULL);" +
"CREATE TABLE `{{users}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `status` integer NOT NULL, " + "CREATE TABLE `{{users}}` (`id` integer AUTO_INCREMENT NOT NULL PRIMARY KEY, `username` varchar(255) NOT NULL UNIQUE, " +
"`expiration_date` bigint NOT NULL, `username` varchar(255) NOT NULL UNIQUE, `password` longtext NULL, " + "`status` integer NOT NULL, `expiration_date` bigint NOT NULL, `description` varchar(512) NULL, `password` longtext NULL, " +
"`public_keys` longtext NULL, `home_dir` varchar(512) NOT NULL, `uid` integer NOT NULL, `gid` integer NOT NULL, " + "`public_keys` longtext NULL, `home_dir` varchar(512) NOT NULL, `uid` integer NOT NULL, `gid` integer NOT NULL, " +
"`max_sessions` integer NOT NULL, `quota_size` bigint NOT NULL, `quota_files` integer NOT NULL, " + "`max_sessions` integer NOT NULL, `quota_size` bigint NOT NULL, `quota_files` integer NOT NULL, " +
"`permissions` longtext NOT NULL, `used_quota_size` bigint NOT NULL, `used_quota_files` integer NOT NULL, " + "`permissions` longtext NOT NULL, `used_quota_size` bigint NOT NULL, `used_quota_files` integer NOT NULL, " +
@@ -39,15 +39,7 @@ const (
"ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `{{prefix}}unique_mapping` UNIQUE (`user_id`, `folder_id`);" + "ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `{{prefix}}unique_mapping` UNIQUE (`user_id`, `folder_id`);" +
"ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `{{prefix}}folders_mapping_folder_id_fk_folders_id` FOREIGN KEY (`folder_id`) REFERENCES `{{folders}}` (`id`) ON DELETE CASCADE;" + "ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `{{prefix}}folders_mapping_folder_id_fk_folders_id` FOREIGN KEY (`folder_id`) REFERENCES `{{folders}}` (`id`) ON DELETE CASCADE;" +
"ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `{{prefix}}folders_mapping_user_id_fk_users_id` FOREIGN KEY (`user_id`) REFERENCES `{{users}}` (`id`) ON DELETE CASCADE;" + "ALTER TABLE `{{folders_mapping}}` ADD CONSTRAINT `{{prefix}}folders_mapping_user_id_fk_users_id` FOREIGN KEY (`user_id`) REFERENCES `{{users}}` (`id`) ON DELETE CASCADE;" +
"INSERT INTO {{schema_version}} (version) VALUES (8);" "INSERT INTO {{schema_version}} (version) VALUES (10);"
mysqlV9SQL = "ALTER TABLE `{{admins}}` ADD COLUMN `description` varchar(512) NULL;" +
"ALTER TABLE `{{folders}}` ADD COLUMN `description` varchar(512) NULL;" +
"ALTER TABLE `{{folders}}` ADD COLUMN `filesystem` longtext NULL;" +
"ALTER TABLE `{{users}}` ADD COLUMN `description` varchar(512) NULL;"
mysqlV9DownSQL = "ALTER TABLE `{{users}}` DROP COLUMN `description`;" +
"ALTER TABLE `{{folders}}` DROP COLUMN `filesystem`;" +
"ALTER TABLE `{{folders}}` DROP COLUMN `description`;" +
"ALTER TABLE `{{admins}}` DROP COLUMN `description`;"
) )
// MySQLProvider auth provider for MySQL/MariaDB database // MySQLProvider auth provider for MySQL/MariaDB database
@@ -230,7 +222,7 @@ func (p *MySQLProvider) initializeDatabase() error {
initialSQL = strings.ReplaceAll(initialSQL, "{{folders_mapping}}", sqlTableFoldersMapping) initialSQL = strings.ReplaceAll(initialSQL, "{{folders_mapping}}", sqlTableFoldersMapping)
initialSQL = strings.ReplaceAll(initialSQL, "{{prefix}}", config.SQLTablesPrefix) initialSQL = strings.ReplaceAll(initialSQL, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, strings.Split(initialSQL, ";"), 8) return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, strings.Split(initialSQL, ";"), 10)
} }
func (p *MySQLProvider) migrateDatabase() error { func (p *MySQLProvider) migrateDatabase() error {
@@ -243,15 +235,11 @@ func (p *MySQLProvider) migrateDatabase() error {
case version == sqlDatabaseVersion: case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version) providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version)
return ErrNoInitRequired return ErrNoInitRequired
case version < 8: case version < 10:
err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version) err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err) providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err) logger.ErrorToConsole("%v", err)
return err return err
case version == 8:
return updateMySQLDatabaseFromV8(p.dbHandle)
case version == 9:
return updateMySQLDatabaseFromV9(p.dbHandle)
default: default:
if version > sqlDatabaseVersion { if version > sqlDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version,
@@ -273,60 +261,5 @@ func (p *MySQLProvider) revertDatabase(targetVersion int) error {
return errors.New("current version match target version, nothing to do") return errors.New("current version match target version, nothing to do")
} }
switch dbVersion.Version { return errors.New("the current version cannot be reverted")
case 9:
return downgradeMySQLDatabaseFromV9(p.dbHandle)
case 10:
return downgradeMySQLDatabaseFromV10(p.dbHandle)
default:
return fmt.Errorf("database version not handled: %v", dbVersion.Version)
}
}
func updateMySQLDatabaseFromV8(dbHandle *sql.DB) error {
if err := updateMySQLDatabaseFrom8To9(dbHandle); err != nil {
return err
}
return updateMySQLDatabaseFromV9(dbHandle)
}
func updateMySQLDatabaseFromV9(dbHandle *sql.DB) error {
return updateMySQLDatabaseFrom9To10(dbHandle)
}
func downgradeMySQLDatabaseFromV9(dbHandle *sql.DB) error {
return downgradeMySQLDatabaseFrom9To8(dbHandle)
}
func downgradeMySQLDatabaseFromV10(dbHandle *sql.DB) error {
if err := downgradeMySQLDatabaseFrom10To9(dbHandle); err != nil {
return err
}
return downgradeMySQLDatabaseFromV9(dbHandle)
}
func updateMySQLDatabaseFrom8To9(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 8 -> 9")
providerLog(logger.LevelInfo, "updating database version: 8 -> 9")
sql := strings.ReplaceAll(mysqlV9SQL, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 9)
}
func downgradeMySQLDatabaseFrom9To8(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 9 -> 8")
providerLog(logger.LevelInfo, "downgrading database version: 9 -> 8")
sql := strings.ReplaceAll(mysqlV9DownSQL, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, strings.Split(sql, ";"), 8)
}
func updateMySQLDatabaseFrom9To10(dbHandle *sql.DB) error {
return sqlCommonUpdateDatabaseFrom9To10(dbHandle)
}
func downgradeMySQLDatabaseFrom10To9(dbHandle *sql.DB) error {
return sqlCommonDowngradeDatabaseFrom10To9(dbHandle)
} }

99
dataprovider/pgsql.go
View File

@@ -21,16 +21,16 @@ import (
const ( const (
pgsqlInitial = `CREATE TABLE "{{schema_version}}" ("id" serial NOT NULL PRIMARY KEY, "version" integer NOT NULL); pgsqlInitial = `CREATE TABLE "{{schema_version}}" ("id" serial NOT NULL PRIMARY KEY, "version" integer NOT NULL);
CREATE TABLE "{{admins}}" ("id" serial NOT NULL PRIMARY KEY, "username" varchar(255) NOT NULL UNIQUE, CREATE TABLE "{{admins}}" ("id" serial NOT NULL PRIMARY KEY, "username" varchar(255) NOT NULL UNIQUE,
"password" varchar(255) NOT NULL, "email" varchar(255) NULL, "status" integer NOT NULL, "permissions" text NOT NULL, "description" varchar(512) NULL, "password" varchar(255) NOT NULL, "email" varchar(255) NULL, "status" integer NOT NULL,
"filters" text NULL, "additional_info" text NULL); "permissions" text NOT NULL, "filters" text NULL, "additional_info" text NULL);
CREATE TABLE "{{folders}}" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL UNIQUE, CREATE TABLE "{{folders}}" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL UNIQUE, "description" varchar(512) NULL,
"path" varchar(512) NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "path" varchar(512) NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL,
"last_quota_update" bigint NOT NULL); "filesystem" text NULL);
CREATE TABLE "{{users}}" ("id" serial NOT NULL PRIMARY KEY, "status" integer NOT NULL, "expiration_date" bigint NOT NULL, CREATE TABLE "{{users}}" ("id" serial NOT NULL PRIMARY KEY, "username" varchar(255) NOT NULL UNIQUE, "status" integer NOT NULL,
"username" varchar(255) NOT NULL UNIQUE, "password" text NULL, "public_keys" text NULL, "home_dir" varchar(512) NOT NULL, "expiration_date" bigint NOT NULL, "description" varchar(512) NULL, "password" text NULL, "public_keys" text NULL,
"uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "home_dir" varchar(512) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL,
"quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL,
"used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL,
"download_bandwidth" integer NOT NULL, "last_login" bigint NOT NULL, "filters" text NULL, "filesystem" text NULL, "download_bandwidth" integer NOT NULL, "last_login" bigint NOT NULL, "filters" text NULL, "filesystem" text NULL,
"additional_info" text NULL); "additional_info" text NULL);
@@ -43,17 +43,7 @@ ALTER TABLE "{{folders_mapping}}" ADD CONSTRAINT "{{prefix}}folders_mapping_user
FOREIGN KEY ("user_id") REFERENCES "{{users}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED; FOREIGN KEY ("user_id") REFERENCES "{{users}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED;
CREATE INDEX "{{prefix}}folders_mapping_folder_id_idx" ON "{{folders_mapping}}" ("folder_id"); CREATE INDEX "{{prefix}}folders_mapping_folder_id_idx" ON "{{folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}folders_mapping_user_id_idx" ON "{{folders_mapping}}" ("user_id"); CREATE INDEX "{{prefix}}folders_mapping_user_id_idx" ON "{{folders_mapping}}" ("user_id");
INSERT INTO {{schema_version}} (version) VALUES (8); INSERT INTO {{schema_version}} (version) VALUES (10);
`
pgsqlV9SQL = `ALTER TABLE "{{admins}}" ADD COLUMN "description" varchar(512) NULL;
ALTER TABLE "{{folders}}" ADD COLUMN "description" varchar(512) NULL;
ALTER TABLE "{{folders}}" ADD COLUMN "filesystem" text NULL;
ALTER TABLE "{{users}}" ADD COLUMN "description" varchar(512) NULL;
`
pgsqlV9DownSQL = `ALTER TABLE "{{users}}" DROP COLUMN "description" CASCADE;
ALTER TABLE "{{folders}}" DROP COLUMN "filesystem" CASCADE;
ALTER TABLE "{{folders}}" DROP COLUMN "description" CASCADE;
ALTER TABLE "{{admins}}" DROP COLUMN "description" CASCADE;
` `
) )
@@ -237,13 +227,13 @@ func (p *PGSQLProvider) initializeDatabase() error {
initialSQL = strings.ReplaceAll(initialSQL, "{{folders_mapping}}", sqlTableFoldersMapping) initialSQL = strings.ReplaceAll(initialSQL, "{{folders_mapping}}", sqlTableFoldersMapping)
initialSQL = strings.ReplaceAll(initialSQL, "{{prefix}}", config.SQLTablesPrefix) initialSQL = strings.ReplaceAll(initialSQL, "{{prefix}}", config.SQLTablesPrefix)
if config.Driver == CockroachDataProviderName { if config.Driver == CockroachDataProviderName {
// Cockroach does not support deferrable constraint validation, we don't need it, // Cockroach does not support deferrable constraint validation, we don't need them,
// we keep these definitions for the PostgreSQL driver to avoid changes for users // we keep these definitions for the PostgreSQL driver to avoid changes for users
// upgrading from old SFTPGo versions // upgrading from old SFTPGo versions
initialSQL = strings.ReplaceAll(initialSQL, "DEFERRABLE INITIALLY DEFERRED", "") initialSQL = strings.ReplaceAll(initialSQL, "DEFERRABLE INITIALLY DEFERRED", "")
} }
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 8) return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 10)
} }
func (p *PGSQLProvider) migrateDatabase() error { func (p *PGSQLProvider) migrateDatabase() error {
@@ -256,15 +246,11 @@ func (p *PGSQLProvider) migrateDatabase() error {
case version == sqlDatabaseVersion: case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version) providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version)
return ErrNoInitRequired return ErrNoInitRequired
case version < 8: case version < 10:
err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version) err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err) providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err) logger.ErrorToConsole("%v", err)
return err return err
case version == 8:
return updatePGSQLDatabaseFromV8(p.dbHandle)
case version == 9:
return updatePGSQLDatabaseFromV9(p.dbHandle)
default: default:
if version > sqlDatabaseVersion { if version > sqlDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version,
@@ -286,60 +272,5 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error {
return errors.New("current version match target version, nothing to do") return errors.New("current version match target version, nothing to do")
} }
switch dbVersion.Version { return errors.New("the current version cannot be reverted")
case 9:
return downgradePGSQLDatabaseFromV9(p.dbHandle)
case 10:
return downgradePGSQLDatabaseFromV10(p.dbHandle)
default:
return fmt.Errorf("database version not handled: %v", dbVersion.Version)
}
}
func updatePGSQLDatabaseFromV8(dbHandle *sql.DB) error {
if err := updatePGSQLDatabaseFrom8To9(dbHandle); err != nil {
return err
}
return updatePGSQLDatabaseFromV9(dbHandle)
}
func updatePGSQLDatabaseFromV9(dbHandle *sql.DB) error {
return updatePGSQLDatabaseFrom9To10(dbHandle)
}
func downgradePGSQLDatabaseFromV9(dbHandle *sql.DB) error {
return downgradePGSQLDatabaseFrom9To8(dbHandle)
}
func downgradePGSQLDatabaseFromV10(dbHandle *sql.DB) error {
if err := downgradePGSQLDatabaseFrom10To9(dbHandle); err != nil {
return err
}
return downgradePGSQLDatabaseFromV9(dbHandle)
}
func updatePGSQLDatabaseFrom8To9(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 8 -> 9")
providerLog(logger.LevelInfo, "updating database version: 8 -> 9")
sql := strings.ReplaceAll(pgsqlV9SQL, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 9)
}
func downgradePGSQLDatabaseFrom9To8(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 9 -> 8")
providerLog(logger.LevelInfo, "downgrading database version: 9 -> 8")
sql := strings.ReplaceAll(pgsqlV9DownSQL, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 8)
}
func updatePGSQLDatabaseFrom9To10(dbHandle *sql.DB) error {
return sqlCommonUpdateDatabaseFrom9To10(dbHandle)
}
func downgradePGSQLDatabaseFrom10To9(dbHandle *sql.DB) error {
return sqlCommonDowngradeDatabaseFrom10To9(dbHandle)
} }

310
dataprovider/sqlcommon.go
View File

@@ -1096,313 +1096,3 @@ func sqlCommonExecuteTx(ctx context.Context, dbHandle *sql.DB, txFn func(*sql.Tx
} }
return tx.Commit() return tx.Commit()
} }
func sqlCommonUpdateDatabaseFrom9To10(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 9 -> 10")
providerLog(logger.LevelInfo, "updating database version: 9 -> 10")
if err := sqlCommonUpdateV10Folders(dbHandle); err != nil {
return err
}
if err := sqlCommonUpdateV10Users(dbHandle); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 10)
}
func sqlCommonDowngradeDatabaseFrom10To9(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 10 -> 9")
providerLog(logger.LevelInfo, "downgrading database version: 10 -> 9")
if err := sqlCommonDowngradeV10Folders(dbHandle); err != nil {
return err
}
if err := sqlCommonDowngradeV10Users(dbHandle); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), defaultSQLQueryTimeout)
defer cancel()
return sqlCommonUpdateDatabaseVersion(ctx, dbHandle, 9)
}
//nolint:dupl
func sqlCommonDowngradeV10Folders(dbHandle *sql.DB) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel()
q := getCompatFolderV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
rows, err := stmt.QueryContext(ctx)
if err != nil {
return err
}
defer rows.Close()
var folders []compatBaseFolderV9
for rows.Next() {
var folder compatBaseFolderV9
var fsConfigString sql.NullString
err = rows.Scan(&folder.ID, &folder.Name, &fsConfigString)
if err != nil {
return err
}
if fsConfigString.Valid {
var fsConfig vfs.Filesystem
err = json.Unmarshal([]byte(fsConfigString.String), &fsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v10 fsconfig for folder %#v, is it already migrated?", folder.Name)
continue
}
if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() {
fsV9, err := convertFsConfigToV9(fsConfig)
if err != nil {
return err
}
folder.FsConfig = fsV9
folders = append(folders, folder)
}
}
}
if err := rows.Err(); err != nil {
return err
}
// update fsconfig for affected folders
for _, folder := range folders {
q := updateCompatFolderV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
cfg, err := json.Marshal(folder.FsConfig)
if err != nil {
return err
}
_, err = stmt.ExecContext(ctx, string(cfg), folder.ID)
if err != nil {
return err
}
}
return nil
}
//nolint:dupl
func sqlCommonDowngradeV10Users(dbHandle *sql.DB) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel()
q := getCompatUserV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
rows, err := stmt.QueryContext(ctx)
if err != nil {
return err
}
defer rows.Close()
var users []compatUserV9
for rows.Next() {
var user compatUserV9
var fsConfigString sql.NullString
err = rows.Scan(&user.ID, &user.Username, &fsConfigString)
if err != nil {
return err
}
if fsConfigString.Valid {
var fsConfig vfs.Filesystem
err = json.Unmarshal([]byte(fsConfigString.String), &fsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v10 fsconfig for user %#v, is it already migrated?", user.Username)
continue
}
if fsConfig.AzBlobConfig.SASURL != nil && !fsConfig.AzBlobConfig.SASURL.IsEmpty() {
fsV9, err := convertFsConfigToV9(fsConfig)
if err != nil {
return err
}
user.FsConfig = fsV9
users = append(users, user)
}
}
}
if err := rows.Err(); err != nil {
return err
}
// update fsconfig for affected users
for _, user := range users {
q := updateCompatUserV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
cfg, err := json.Marshal(user.FsConfig)
if err != nil {
return err
}
_, err = stmt.ExecContext(ctx, string(cfg), user.ID)
if err != nil {
return err
}
}
return nil
}
func sqlCommonUpdateV10Folders(dbHandle *sql.DB) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel()
q := getCompatFolderV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
rows, err := stmt.QueryContext(ctx)
if err != nil {
return err
}
defer rows.Close()
var folders []vfs.BaseVirtualFolder
for rows.Next() {
var folder vfs.BaseVirtualFolder
var fsConfigString sql.NullString
err = rows.Scan(&folder.ID, &folder.Name, &fsConfigString)
if err != nil {
return err
}
if fsConfigString.Valid {
var compatFsConfig compatFilesystemV9
err = json.Unmarshal([]byte(fsConfigString.String), &compatFsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v9 fsconfig for folder %#v, is it already migrated?", folder.Name)
continue
}
if compatFsConfig.AzBlobConfig.SASURL != "" {
fsConfig, err := convertFsConfigFromV9(compatFsConfig, folder.GetEncrytionAdditionalData())
if err != nil {
return err
}
folder.FsConfig = fsConfig
folders = append(folders, folder)
}
}
}
if err := rows.Err(); err != nil {
return err
}
// update fsconfig for affected folders
for _, folder := range folders {
q := updateCompatFolderV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
cfg, err := json.Marshal(folder.FsConfig)
if err != nil {
return err
}
_, err = stmt.ExecContext(ctx, string(cfg), folder.ID)
if err != nil {
return err
}
}
return nil
}
func sqlCommonUpdateV10Users(dbHandle *sql.DB) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel()
q := getCompatUserV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
rows, err := stmt.QueryContext(ctx)
if err != nil {
return err
}
defer rows.Close()
var users []User
for rows.Next() {
var user User
var fsConfigString sql.NullString
err = rows.Scan(&user.ID, &user.Username, &fsConfigString)
if err != nil {
return err
}
if fsConfigString.Valid {
var compatFsConfig compatFilesystemV9
err = json.Unmarshal([]byte(fsConfigString.String), &compatFsConfig)
if err != nil {
logger.WarnToConsole("failed to unmarshal v9 fsconfig for user %#v, is it already migrated?", user.Username)
continue
}
if compatFsConfig.AzBlobConfig.SASURL != "" {
fsConfig, err := convertFsConfigFromV9(compatFsConfig, user.GetEncrytionAdditionalData())
if err != nil {
return err
}
user.FsConfig = fsConfig
users = append(users, user)
}
}
}
if err := rows.Err(); err != nil {
return err
}
// update fsconfig for affected users
for _, user := range users {
q := updateCompatUserV10FsConfigQuery()
stmt, err := dbHandle.PrepareContext(ctx, q)
if err != nil {
providerLog(logger.LevelWarn, "error preparing database query %#v: %v", q, err)
return err
}
defer stmt.Close()
cfg, err := json.Marshal(user.FsConfig)
if err != nil {
return err
}
_, err = stmt.ExecContext(ctx, string(cfg), user.ID)
if err != nil {
return err
}
}
return nil
}

127
dataprovider/sqlite.go
View File

@@ -23,59 +23,25 @@ import (
const ( const (
sqliteInitialSQL = `CREATE TABLE "{{schema_version}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "version" integer NOT NULL); sqliteInitialSQL = `CREATE TABLE "{{schema_version}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "version" integer NOT NULL);
CREATE TABLE "{{admins}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, CREATE TABLE "{{admins}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE,
"password" varchar(255) NOT NULL, "email" varchar(255) NULL, "status" integer NOT NULL, "permissions" text NOT NULL, "description" varchar(512) NULL, "password" varchar(255) NOT NULL, "email" varchar(255) NULL, "status" integer NOT NULL,
"filters" text NULL, "additional_info" text NULL); "permissions" text NOT NULL, "filters" text NULL, "additional_info" text NULL);
CREATE TABLE "{{folders}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(255) NOT NULL UNIQUE, CREATE TABLE "{{folders}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(255) NOT NULL UNIQUE,
"path" varchar(512) NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "description" varchar(512) NULL, "path" varchar(512) NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL,
"last_quota_update" bigint NOT NULL); "last_quota_update" bigint NOT NULL, "filesystem" text NULL);
CREATE TABLE "{{users}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE, CREATE TABLE "{{users}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE,
"password" text NULL, "public_keys" text NULL, "home_dir" varchar(512) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "status" integer NOT NULL, "expiration_date" bigint NOT NULL, "description" varchar(512) NULL, "password" text NULL,
"public_keys" text NULL, "home_dir" varchar(512) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL,
"max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "max_sessions" integer NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL,
"used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL,
"upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "expiration_date" bigint NOT NULL, "upload_bandwidth" integer NOT NULL, "download_bandwidth" integer NOT NULL, "last_login" bigint NOT NULL, "filters" text NULL,
"last_login" bigint NOT NULL, "status" integer NOT NULL, "filters" text NULL, "filesystem" text NULL, "filesystem" text NULL, "additional_info" text NULL);
"additional_info" text NULL);
CREATE TABLE "{{folders_mapping}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "virtual_path" varchar(512) NOT NULL, CREATE TABLE "{{folders_mapping}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "virtual_path" varchar(512) NOT NULL,
"quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "folder_id" integer NOT NULL REFERENCES "{{folders}}" ("id") "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "folder_id" integer NOT NULL REFERENCES "{{folders}}" ("id")
ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, "user_id" integer NOT NULL REFERENCES "{{users}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, "user_id" integer NOT NULL REFERENCES "{{users}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
CONSTRAINT "{{prefix}}unique_mapping" UNIQUE ("user_id", "folder_id")); CONSTRAINT "{{prefix}}unique_mapping" UNIQUE ("user_id", "folder_id"));
CREATE INDEX "{{prefix}}folders_mapping_folder_id_idx" ON "{{folders_mapping}}" ("folder_id"); CREATE INDEX "{{prefix}}folders_mapping_folder_id_idx" ON "{{folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}folders_mapping_user_id_idx" ON "{{folders_mapping}}" ("user_id"); CREATE INDEX "{{prefix}}folders_mapping_user_id_idx" ON "{{folders_mapping}}" ("user_id");
INSERT INTO {{schema_version}} (version) VALUES (8); INSERT INTO {{schema_version}} (version) VALUES (10);
`
sqliteV9SQL = `ALTER TABLE "{{admins}}" ADD COLUMN "description" varchar(512) NULL;
ALTER TABLE "{{folders}}" ADD COLUMN "description" varchar(512) NULL;
ALTER TABLE "{{folders}}" ADD COLUMN "filesystem" text NULL;
ALTER TABLE "{{users}}" ADD COLUMN "description" varchar(512) NULL;
`
sqliteV9DownSQL = `CREATE TABLE "new__users" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "status" integer NOT NULL,
"expiration_date" bigint NOT NULL, "username" varchar(255) NOT NULL UNIQUE, "password" text NULL, "public_keys" text NULL,
"home_dir" varchar(512) NOT NULL, "uid" integer NOT NULL, "gid" integer NOT NULL, "max_sessions" integer NOT NULL,
"quota_size" bigint NOT NULL, "quota_files" integer NOT NULL, "permissions" text NOT NULL, "used_quota_size" bigint NOT NULL,
"used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL, "upload_bandwidth" integer NOT NULL,
"download_bandwidth" integer NOT NULL, "last_login" bigint NOT NULL, "filters" text NULL, "filesystem" text NULL,
"additional_info" text NULL);
INSERT INTO "new__users" ("id", "status", "expiration_date", "username", "password", "public_keys", "home_dir", "uid", "gid",
"max_sessions", "quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update",
"upload_bandwidth", "download_bandwidth", "last_login", "filters", "filesystem", "additional_info")
SELECT "id", "status", "expiration_date", "username", "password", "public_keys", "home_dir", "uid", "gid", "max_sessions",
"quota_size", "quota_files", "permissions", "used_quota_size", "used_quota_files", "last_quota_update", "upload_bandwidth",
"download_bandwidth", "last_login", "filters", "filesystem", "additional_info" FROM "{{users}}";
DROP TABLE "{{users}}";
ALTER TABLE "new__users" RENAME TO "{{users}}";
CREATE TABLE "new__admins" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "username" varchar(255) NOT NULL UNIQUE,
"password" varchar(255) NOT NULL, "email" varchar(255) NULL, "status" integer NOT NULL, "permissions" text NOT NULL,
"filters" text NULL, "additional_info" text NULL);
INSERT INTO "new__admins" ("id", "username", "password", "email", "status", "permissions", "filters", "additional_info")
SELECT "id", "username", "password", "email", "status", "permissions", "filters", "additional_info" FROM "{{admins}}";
DROP TABLE "{{admins}}";
ALTER TABLE "new__admins" RENAME TO "{{admins}}";
CREATE TABLE "new__folders" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(255) NOT NULL UNIQUE,
"path" varchar(512) NULL, "used_quota_size" bigint NOT NULL, "used_quota_files" integer NOT NULL, "last_quota_update" bigint NOT NULL);
INSERT INTO "new__folders" ("id", "name", "path", "used_quota_size", "used_quota_files", "last_quota_update")
SELECT "id", "name", "path", "used_quota_size", "used_quota_files", "last_quota_update" FROM "{{folders}}";
DROP TABLE "{{folders}}";
ALTER TABLE "new__folders" RENAME TO "{{folders}}";
` `
) )
@@ -251,7 +217,7 @@ func (p *SQLiteProvider) initializeDatabase() error {
initialSQL = strings.ReplaceAll(initialSQL, "{{folders_mapping}}", sqlTableFoldersMapping) initialSQL = strings.ReplaceAll(initialSQL, "{{folders_mapping}}", sqlTableFoldersMapping)
initialSQL = strings.ReplaceAll(initialSQL, "{{prefix}}", config.SQLTablesPrefix) initialSQL = strings.ReplaceAll(initialSQL, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 8) return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 10)
} }
func (p *SQLiteProvider) migrateDatabase() error { func (p *SQLiteProvider) migrateDatabase() error {
@@ -264,15 +230,11 @@ func (p *SQLiteProvider) migrateDatabase() error {
case version == sqlDatabaseVersion: case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version) providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version)
return ErrNoInitRequired return ErrNoInitRequired
case version < 8: case version < 10:
err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version) err = fmt.Errorf("database version %v is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err) providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err) logger.ErrorToConsole("%v", err)
return err return err
case version == 8:
return updateSQLiteDatabaseFromV8(p.dbHandle)
case version == 9:
return updateSQLiteDatabaseFromV9(p.dbHandle)
default: default:
if version > sqlDatabaseVersion { if version > sqlDatabaseVersion {
providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version, providerLog(logger.LevelWarn, "database version %v is newer than the supported one: %v", version,
@@ -294,71 +256,10 @@ func (p *SQLiteProvider) revertDatabase(targetVersion int) error {
return errors.New("current version match target version, nothing to do") return errors.New("current version match target version, nothing to do")
} }
switch dbVersion.Version { return errors.New("the current version cannot be reverted")
case 9:
return downgradeSQLiteDatabaseFromV9(p.dbHandle)
case 10:
return downgradeSQLiteDatabaseFromV10(p.dbHandle)
default:
return fmt.Errorf("database version not handled: %v", dbVersion.Version)
}
} }
func updateSQLiteDatabaseFromV8(dbHandle *sql.DB) error { /*func setPragmaFK(dbHandle *sql.DB, value string) error {
if err := updateSQLiteDatabaseFrom8To9(dbHandle); err != nil {
return err
}
return updateSQLiteDatabaseFromV9(dbHandle)
}
func updateSQLiteDatabaseFromV9(dbHandle *sql.DB) error {
return updateSQLiteDatabaseFrom9To10(dbHandle)
}
func downgradeSQLiteDatabaseFromV9(dbHandle *sql.DB) error {
return downgradeSQLiteDatabaseFrom9To8(dbHandle)
}
func downgradeSQLiteDatabaseFromV10(dbHandle *sql.DB) error {
if err := downgradeSQLiteDatabaseFrom10To9(dbHandle); err != nil {
return err
}
return downgradeSQLiteDatabaseFromV9(dbHandle)
}
func updateSQLiteDatabaseFrom8To9(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database version: 8 -> 9")
providerLog(logger.LevelInfo, "updating database version: 8 -> 9")
sql := strings.ReplaceAll(sqliteV9SQL, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 9)
}
func downgradeSQLiteDatabaseFrom9To8(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database version: 9 -> 8")
providerLog(logger.LevelInfo, "downgrading database version: 9 -> 8")
if err := setPragmaFK(dbHandle, "OFF"); err != nil {
return err
}
sql := strings.ReplaceAll(sqliteV9DownSQL, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{folders}}", sqlTableFolders)
if err := sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 8); err != nil {
return err
}
return setPragmaFK(dbHandle, "ON")
}
func updateSQLiteDatabaseFrom9To10(dbHandle *sql.DB) error {
return sqlCommonUpdateDatabaseFrom9To10(dbHandle)
}
func downgradeSQLiteDatabaseFrom10To9(dbHandle *sql.DB) error {
return sqlCommonDowngradeDatabaseFrom10To9(dbHandle)
}
func setPragmaFK(dbHandle *sql.DB, value string) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout) ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel() defer cancel()
@@ -366,4 +267,4 @@ func setPragmaFK(dbHandle *sql.DB, value string) error {
_, err := dbHandle.ExecContext(ctx, sql) _, err := dbHandle.ExecContext(ctx, sql)
return err return err
} }*/

16
dataprovider/sqlqueries.go
View File

@@ -210,19 +210,3 @@ func getDatabaseVersionQuery() string {
func getUpdateDBVersionQuery() string { func getUpdateDBVersionQuery() string {
return fmt.Sprintf(`UPDATE %v SET version=%v`, sqlTableSchemaVersion, sqlPlaceholders[0]) return fmt.Sprintf(`UPDATE %v SET version=%v`, sqlTableSchemaVersion, sqlPlaceholders[0])
} }
func getCompatUserV10FsConfigQuery() string {
return fmt.Sprintf(`SELECT id,username,filesystem FROM %v`, sqlTableUsers)
}
func updateCompatUserV10FsConfigQuery() string {
return fmt.Sprintf(`UPDATE %v SET filesystem=%v WHERE id=%v`, sqlTableUsers, sqlPlaceholders[0], sqlPlaceholders[1])
}
func getCompatFolderV10FsConfigQuery() string {
return fmt.Sprintf(`SELECT id,name,filesystem FROM %v`, sqlTableFolders)
}
func updateCompatFolderV10FsConfigQuery() string {
return fmt.Sprintf(`UPDATE %v SET filesystem=%v WHERE id=%v`, sqlTableFolders, sqlPlaceholders[0], sqlPlaceholders[1])
}