Mirrors/sftpgo
1
0
Fork 0
mirror of https://github.com/drakkan/sftpgo.git synced 2025-12-07 06:40:54 +03:00
Code Issues Packages Projects Releases Wiki Activity

squash database migrations

Browse Source 
Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino
2022-10-28 14:28:37 +02:00
parent 6f422c3d8b
commit 9a9e7d1a7f
6 changed files with 126 additions and 622 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
internal/dataprovider/sqlite.go
View File

@@ -24,7 +24,6 @@ import (
"errors"
"fmt"
"path/filepath"
"strings"
"time"
// we import go-sqlite3 here to be able to disable SQLite support using a build tag
@@ -89,7 +88,8 @@ CREATE TABLE "{{users}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "user
"filters" text NULL, "filesystem" text NULL, "additional_info" text NULL, "created_at" bigint NOT NULL,
"updated_at" bigint NOT NULL, "email" varchar(255) NULL, "upload_data_transfer" integer NOT NULL,
"download_data_transfer" integer NOT NULL, "total_data_transfer" integer NOT NULL, "used_upload_data_transfer" integer NOT NULL,
"used_download_data_transfer" integer NOT NULL);
"used_download_data_transfer" integer NOT NULL, "deleted_at" bigint NOT NULL, "first_download" bigint NOT NULL,
"first_upload" bigint NOT NULL);
CREATE TABLE "{{groups_folders_mapping}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"folder_id" integer NOT NULL REFERENCES "{{folders}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
"group_id" integer NOT NULL REFERENCES "{{groups}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
@@ -114,29 +114,7 @@ CREATE TABLE "{{api_keys}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "n
"created_at" bigint NOT NULL, "updated_at" bigint NOT NULL, "last_use_at" bigint NOT NULL, "expires_at" bigint NOT NULL,
"description" text NULL, "admin_id" integer NULL REFERENCES "{{admins}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
"user_id" integer NULL REFERENCES "{{users}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED);
CREATE INDEX "{{prefix}}groups_updated_at_idx" ON "{{groups}}" ("updated_at");
CREATE INDEX "{{prefix}}users_folders_mapping_folder_id_idx" ON "{{users_folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}users_folders_mapping_user_id_idx" ON "{{users_folders_mapping}}" ("user_id");
CREATE INDEX "{{prefix}}users_groups_mapping_group_id_idx" ON "{{users_groups_mapping}}" ("group_id");
CREATE INDEX "{{prefix}}users_groups_mapping_user_id_idx" ON "{{users_groups_mapping}}" ("user_id");
CREATE INDEX "{{prefix}}groups_folders_mapping_folder_id_idx" ON "{{groups_folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}groups_folders_mapping_group_id_idx" ON "{{groups_folders_mapping}}" ("group_id");
CREATE INDEX "{{prefix}}api_keys_admin_id_idx" ON "{{api_keys}}" ("admin_id");
CREATE INDEX "{{prefix}}api_keys_user_id_idx" ON "{{api_keys}}" ("user_id");
CREATE INDEX "{{prefix}}users_updated_at_idx" ON "{{users}}" ("updated_at");
CREATE INDEX "{{prefix}}shares_user_id_idx" ON "{{shares}}" ("user_id");
CREATE INDEX "{{prefix}}defender_hosts_updated_at_idx" ON "{{defender_hosts}}" ("updated_at");
CREATE INDEX "{{prefix}}defender_hosts_ban_time_idx" ON "{{defender_hosts}}" ("ban_time");
CREATE INDEX "{{prefix}}defender_events_date_time_idx" ON "{{defender_events}}" ("date_time");
CREATE INDEX "{{prefix}}defender_events_host_id_idx" ON "{{defender_events}}" ("host_id");
CREATE INDEX "{{prefix}}active_transfers_connection_id_idx" ON "{{active_transfers}}" ("connection_id");
CREATE INDEX "{{prefix}}active_transfers_transfer_id_idx" ON "{{active_transfers}}" ("transfer_id");
CREATE INDEX "{{prefix}}active_transfers_updated_at_idx" ON "{{active_transfers}}" ("updated_at");
CREATE INDEX "{{prefix}}shared_sessions_type_idx" ON "{{shared_sessions}}" ("type");
CREATE INDEX "{{prefix}}shared_sessions_timestamp_idx" ON "{{shared_sessions}}" ("timestamp");
INSERT INTO {{schema_version}} (version) VALUES (19);
`
sqliteV20SQL = `CREATE TABLE "{{events_rules}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
CREATE TABLE "{{events_rules}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"name" varchar(255) NOT NULL UNIQUE, "description" varchar(512) NULL, "created_at" bigint NOT NULL,
"updated_at" bigint NOT NULL, "trigger" integer NOT NULL, "conditions" text NOT NULL, "deleted_at" bigint NOT NULL);
CREATE TABLE "{{events_actions}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(255) NOT NULL UNIQUE,
@@ -148,35 +126,40 @@ CREATE TABLE "{{rules_actions_mapping}}" ("id" integer NOT NULL PRIMARY KEY AUTO
CONSTRAINT "{{prefix}}unique_rule_action_mapping" UNIQUE ("rule_id", "action_id"));
CREATE TABLE "{{tasks}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar(255) NOT NULL UNIQUE,
"updated_at" bigint NOT NULL, "version" bigint NOT NULL);
ALTER TABLE "{{users}}" ADD COLUMN "deleted_at" bigint DEFAULT 0 NOT NULL;
CREATE TABLE "{{admins_groups_mapping}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"admin_id" integer NOT NULL REFERENCES "{{admins}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
"group_id" integer NOT NULL REFERENCES "{{groups}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
"options" text NOT NULL, CONSTRAINT "{{prefix}}unique_admin_group_mapping" UNIQUE ("admin_id", "group_id"));
CREATE INDEX "{{prefix}}users_folders_mapping_folder_id_idx" ON "{{users_folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}users_folders_mapping_user_id_idx" ON "{{users_folders_mapping}}" ("user_id");
CREATE INDEX "{{prefix}}users_groups_mapping_group_id_idx" ON "{{users_groups_mapping}}" ("group_id");
CREATE INDEX "{{prefix}}users_groups_mapping_user_id_idx" ON "{{users_groups_mapping}}" ("user_id");
CREATE INDEX "{{prefix}}groups_folders_mapping_folder_id_idx" ON "{{groups_folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}groups_folders_mapping_group_id_idx" ON "{{groups_folders_mapping}}" ("group_id");
CREATE INDEX "{{prefix}}api_keys_admin_id_idx" ON "{{api_keys}}" ("admin_id");
CREATE INDEX "{{prefix}}api_keys_user_id_idx" ON "{{api_keys}}" ("user_id");
CREATE INDEX "{{prefix}}users_updated_at_idx" ON "{{users}}" ("updated_at");
CREATE INDEX "{{prefix}}users_deleted_at_idx" ON "{{users}}" ("deleted_at");
CREATE INDEX "{{prefix}}shares_user_id_idx" ON "{{shares}}" ("user_id");
CREATE INDEX "{{prefix}}defender_hosts_updated_at_idx" ON "{{defender_hosts}}" ("updated_at");
CREATE INDEX "{{prefix}}defender_hosts_ban_time_idx" ON "{{defender_hosts}}" ("ban_time");
CREATE INDEX "{{prefix}}defender_events_date_time_idx" ON "{{defender_events}}" ("date_time");
CREATE INDEX "{{prefix}}defender_events_host_id_idx" ON "{{defender_events}}" ("host_id");
CREATE INDEX "{{prefix}}active_transfers_connection_id_idx" ON "{{active_transfers}}" ("connection_id");
CREATE INDEX "{{prefix}}active_transfers_transfer_id_idx" ON "{{active_transfers}}" ("transfer_id");
CREATE INDEX "{{prefix}}active_transfers_updated_at_idx" ON "{{active_transfers}}" ("updated_at");
CREATE INDEX "{{prefix}}shared_sessions_type_idx" ON "{{shared_sessions}}" ("type");
CREATE INDEX "{{prefix}}shared_sessions_timestamp_idx" ON "{{shared_sessions}}" ("timestamp");
CREATE INDEX "{{prefix}}events_rules_updated_at_idx" ON "{{events_rules}}" ("updated_at");
CREATE INDEX "{{prefix}}events_rules_deleted_at_idx" ON "{{events_rules}}" ("deleted_at");
CREATE INDEX "{{prefix}}events_rules_trigger_idx" ON "{{events_rules}}" ("trigger");
CREATE INDEX "{{prefix}}rules_actions_mapping_rule_id_idx" ON "{{rules_actions_mapping}}" ("rule_id");
CREATE INDEX "{{prefix}}rules_actions_mapping_action_id_idx" ON "{{rules_actions_mapping}}" ("action_id");
CREATE INDEX "{{prefix}}rules_actions_mapping_order_idx" ON "{{rules_actions_mapping}}" ("order");
CREATE INDEX "{{prefix}}users_deleted_at_idx" ON "{{users}}" ("deleted_at");
`
sqliteV20DownSQL = `DROP TABLE "{{rules_actions_mapping}}";
DROP TABLE "{{events_rules}}";
DROP TABLE "{{events_actions}}";
DROP TABLE "{{tasks}}";
DROP INDEX IF EXISTS "{{prefix}}users_deleted_at_idx";
ALTER TABLE "{{users}}" DROP COLUMN "deleted_at";
`
sqliteV21SQL = `ALTER TABLE "{{users}}" ADD COLUMN "first_download" bigint DEFAULT 0 NOT NULL;
ALTER TABLE "{{users}}" ADD COLUMN "first_upload" bigint DEFAULT 0 NOT NULL;`
sqliteV21DownSQL = `ALTER TABLE "{{users}}" DROP COLUMN "first_upload";
ALTER TABLE "{{users}}" DROP COLUMN "first_download";
`
sqliteV22SQL = `CREATE TABLE "{{admins_groups_mapping}}" ("id" integer NOT NULL PRIMARY KEY AUTOINCREMENT,
"admin_id" integer NOT NULL REFERENCES "{{admins}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
"group_id" integer NOT NULL REFERENCES "{{groups}}" ("id") ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED,
"options" text NOT NULL, CONSTRAINT "{{prefix}}unique_admin_group_mapping" UNIQUE ("admin_id", "group_id"));
CREATE INDEX "{{prefix}}admins_groups_mapping_admin_id_idx" ON "{{admins_groups_mapping}}" ("admin_id");
CREATE INDEX "{{prefix}}admins_groups_mapping_group_id_idx" ON "{{admins_groups_mapping}}" ("group_id");
INSERT INTO {{schema_version}} (version) VALUES (23);
`
sqliteV22DownSQL = `DROP TABLE "{{admins_groups_mapping}}";`
)
// SQLiteProvider defines the auth provider for SQLite database
@@ -623,11 +606,11 @@ func (p *SQLiteProvider) initializeDatabase() error {
if errors.Is(err, sql.ErrNoRows) {
return errSchemaVersionEmpty
}
logger.InfoToConsole("creating initial database schema, version 19")
providerLog(logger.LevelInfo, "creating initial database schema, version 19")
logger.InfoToConsole("creating initial database schema, version 23")
providerLog(logger.LevelInfo, "creating initial database schema, version 23")
sql := sqlReplaceAll(sqliteInitialSQL)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 19, true)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 23, true)
}
func (p *SQLiteProvider) migrateDatabase() error { //nolint:dupl
@@ -638,30 +621,22 @@ func (p *SQLiteProvider) migrateDatabase() error { //nolint:dupl
switch version := dbVersion.Version; {
case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version)
providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version)
return ErrNoInitRequired
case version < 19:
err = fmt.Errorf("database schema version %v is too old, please see the upgrading docs", version)
case version < 23:
err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err)
return err
case version == 19:
return updateSQLiteDatabaseFromV19(p.dbHandle)
case version == 20:
return updateSQLiteDatabaseFromV20(p.dbHandle)
case version == 21:
return updateSQLiteDatabaseFromV21(p.dbHandle)
case version == 22:
return updateSQLiteDatabaseFromV22(p.dbHandle)
default:
if version > sqlDatabaseVersion {
providerLog(logger.LevelError, "database schema version %v is newer than the supported one: %v", version,
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
sqlDatabaseVersion)
logger.WarnToConsole("database schema version %v is newer than the supported one: %v", version,
logger.WarnToConsole("database schema version %d is newer than the supported one: %d", version,
sqlDatabaseVersion)
return nil
}
return fmt.Errorf("database schema version not handled: %v", version)
return fmt.Errorf("database schema version not handled: %d", version)
}
}
@@ -675,16 +650,8 @@ func (p *SQLiteProvider) revertDatabase(targetVersion int) error {
}
switch dbVersion.Version {
case 20:
return downgradeSQLiteDatabaseFromV20(p.dbHandle)
case 21:
return downgradeSQLiteDatabaseFromV21(p.dbHandle)
case 22:
return downgradeSQLiteDatabaseFromV22(p.dbHandle)
case 23:
return downgradeSQLiteDatabaseFromV23(p.dbHandle)
default:
return fmt.Errorf("database schema version not handled: %v", dbVersion.Version)
return fmt.Errorf("database schema version not handled: %d", dbVersion.Version)
}
}
@@ -693,124 +660,6 @@ func (p *SQLiteProvider) resetDatabase() error {
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 0, false)
}
func updateSQLiteDatabaseFromV19(dbHandle *sql.DB) error {
if err := updateSQLiteDatabaseFrom19To20(dbHandle); err != nil {
return err
}
return updateSQLiteDatabaseFromV20(dbHandle)
}
func updateSQLiteDatabaseFromV20(dbHandle *sql.DB) error {
if err := updateSQLiteDatabaseFrom20To21(dbHandle); err != nil {
return err
}
return updateSQLiteDatabaseFromV21(dbHandle)
}
func updateSQLiteDatabaseFromV21(dbHandle *sql.DB) error {
if err := updateSQLiteDatabaseFrom21To22(dbHandle); err != nil {
return err
}
return updateSQLiteDatabaseFromV22(dbHandle)
}
func updateSQLiteDatabaseFromV22(dbHandle *sql.DB) error {
return updateSQLiteDatabaseFrom22To23(dbHandle)
}
func downgradeSQLiteDatabaseFromV20(dbHandle *sql.DB) error {
return downgradeSQLiteDatabaseFrom20To19(dbHandle)
}
func downgradeSQLiteDatabaseFromV21(dbHandle *sql.DB) error {
if err := downgradeSQLiteDatabaseFrom21To20(dbHandle); err != nil {
return err
}
return downgradeSQLiteDatabaseFromV20(dbHandle)
}
func downgradeSQLiteDatabaseFromV22(dbHandle *sql.DB) error {
if err := downgradeSQLiteDatabaseFrom22To21(dbHandle); err != nil {
return err
}
return downgradeSQLiteDatabaseFromV21(dbHandle)
}
func downgradeSQLiteDatabaseFromV23(dbHandle *sql.DB) error {
if err := downgradeSQLiteDatabaseFrom23To22(dbHandle); err != nil {
return err
}
return downgradeSQLiteDatabaseFromV22(dbHandle)
}
func updateSQLiteDatabaseFrom19To20(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 19 -> 20")
providerLog(logger.LevelInfo, "updating database schema version: 19 -> 20")
sql := strings.ReplaceAll(sqliteV20SQL, "{{events_actions}}", sqlTableEventsActions)
sql = strings.ReplaceAll(sql, "{{events_rules}}", sqlTableEventsRules)
sql = strings.ReplaceAll(sql, "{{rules_actions_mapping}}", sqlTableRulesActionsMapping)
sql = strings.ReplaceAll(sql, "{{tasks}}", sqlTableTasks)
sql = strings.ReplaceAll(sql, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 20, true)
}
func updateSQLiteDatabaseFrom20To21(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 20 -> 21")
providerLog(logger.LevelInfo, "updating database schema version: 20 -> 21")
sql := strings.ReplaceAll(sqliteV21SQL, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 21, true)
}
func updateSQLiteDatabaseFrom21To22(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 21 -> 22")
providerLog(logger.LevelInfo, "updating database schema version: 21 -> 22")
sql := strings.ReplaceAll(sqliteV22SQL, "{{admins_groups_mapping}}", sqlTableAdminsGroupsMapping)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{groups}}", sqlTableGroups)
sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 22, true)
}
func updateSQLiteDatabaseFrom22To23(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 22 -> 23")
providerLog(logger.LevelInfo, "updating database schema version: 22 -> 23")
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{`SELECT 1`}, 23, true)
}
func downgradeSQLiteDatabaseFrom20To19(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 20 -> 19")
providerLog(logger.LevelInfo, "downgrading database schema version: 20 -> 19")
sql := strings.ReplaceAll(sqliteV20DownSQL, "{{events_actions}}", sqlTableEventsActions)
sql = strings.ReplaceAll(sql, "{{events_rules}}", sqlTableEventsRules)
sql = strings.ReplaceAll(sql, "{{rules_actions_mapping}}", sqlTableRulesActionsMapping)
sql = strings.ReplaceAll(sql, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{tasks}}", sqlTableTasks)
sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 19, false)
}
func downgradeSQLiteDatabaseFrom21To20(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 21 -> 20")
providerLog(logger.LevelInfo, "downgrading database schema version: 21 -> 20")
sql := strings.ReplaceAll(sqliteV21DownSQL, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 20, false)
}
func downgradeSQLiteDatabaseFrom22To21(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 22 -> 21")
providerLog(logger.LevelInfo, "downgrading database schema version: 22 -> 21")
sql := strings.ReplaceAll(sqliteV22DownSQL, "{{admins_groups_mapping}}", sqlTableAdminsGroupsMapping)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 21, false)
}
func downgradeSQLiteDatabaseFrom23To22(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 23 -> 22")
providerLog(logger.LevelInfo, "downgrading database schema version: 23 -> 22")
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{`SELECT 1`}, 22, false)
}
/*func setPragmaFK(dbHandle *sql.DB, value string) error {
ctx, cancel := context.WithTimeout(context.Background(), longSQLQueryTimeout)
defer cancel()