squash database migrations

Signed-off-by: Nicola Murino <nicola.murino@gmail.com>
This commit is contained in:
Nicola Murino
2022-10-28 14:28:37 +02:00
parent 6f422c3d8b
commit 9a9e7d1a7f
6 changed files with 126 additions and 622 deletions

View File

@@ -23,7 +23,6 @@ import (
"database/sql"
"errors"
"fmt"
"strings"
"time"
// we import pgx here to be able to disable PostgreSQL support using a build tag
@@ -88,7 +87,8 @@ CREATE TABLE "{{users}}" ("id" serial NOT NULL PRIMARY KEY, "username" varchar(2
"download_bandwidth" integer NOT NULL, "last_login" bigint NOT NULL, "filters" text NULL, "filesystem" text NULL,
"additional_info" text NULL, "created_at" bigint NOT NULL, "updated_at" bigint NOT NULL, "email" varchar(255) NULL,
"upload_data_transfer" integer NOT NULL, "download_data_transfer" integer NOT NULL, "total_data_transfer" integer NOT NULL,
"used_upload_data_transfer" integer NOT NULL, "used_download_data_transfer" integer NOT NULL);
"used_upload_data_transfer" integer NOT NULL, "used_download_data_transfer" integer NOT NULL, "deleted_at" bigint NOT NULL,
"first_download" bigint NOT NULL, "first_upload" bigint NOT NULL);
CREATE TABLE "{{groups_folders_mapping}}" ("id" serial NOT NULL PRIMARY KEY, "group_id" integer NOT NULL,
"folder_id" integer NOT NULL, "virtual_path" text NOT NULL, "quota_size" bigint NOT NULL, "quota_files" integer NOT NULL);
CREATE TABLE "{{users_groups_mapping}}" ("id" serial NOT NULL PRIMARY KEY, "user_id" integer NOT NULL,
@@ -130,25 +130,7 @@ FOREIGN KEY ("folder_id") REFERENCES "{{folders}}" ("id") MATCH SIMPLE ON UPDATE
CREATE INDEX "{{prefix}}groups_folders_mapping_group_id_idx" ON "{{groups_folders_mapping}}" ("group_id");
ALTER TABLE "{{groups_folders_mapping}}" ADD CONSTRAINT "{{prefix}}groups_folders_mapping_group_id_fk_groups_id"
FOREIGN KEY ("group_id") REFERENCES "{{groups}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE;
CREATE INDEX "{{prefix}}groups_updated_at_idx" ON "{{groups}}" ("updated_at");
CREATE INDEX "{{prefix}}users_folders_mapping_folder_id_idx" ON "{{users_folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}users_folders_mapping_user_id_idx" ON "{{users_folders_mapping}}" ("user_id");
CREATE INDEX "{{prefix}}api_keys_admin_id_idx" ON "{{api_keys}}" ("admin_id");
CREATE INDEX "{{prefix}}api_keys_user_id_idx" ON "{{api_keys}}" ("user_id");
CREATE INDEX "{{prefix}}users_updated_at_idx" ON "{{users}}" ("updated_at");
CREATE INDEX "{{prefix}}shares_user_id_idx" ON "{{shares}}" ("user_id");
CREATE INDEX "{{prefix}}defender_hosts_updated_at_idx" ON "{{defender_hosts}}" ("updated_at");
CREATE INDEX "{{prefix}}defender_hosts_ban_time_idx" ON "{{defender_hosts}}" ("ban_time");
CREATE INDEX "{{prefix}}defender_events_date_time_idx" ON "{{defender_events}}" ("date_time");
CREATE INDEX "{{prefix}}defender_events_host_id_idx" ON "{{defender_events}}" ("host_id");
CREATE INDEX "{{prefix}}active_transfers_connection_id_idx" ON "{{active_transfers}}" ("connection_id");
CREATE INDEX "{{prefix}}active_transfers_transfer_id_idx" ON "{{active_transfers}}" ("transfer_id");
CREATE INDEX "{{prefix}}active_transfers_updated_at_idx" ON "{{active_transfers}}" ("updated_at");
CREATE INDEX "{{prefix}}shared_sessions_type_idx" ON "{{shared_sessions}}" ("type");
CREATE INDEX "{{prefix}}shared_sessions_timestamp_idx" ON "{{shared_sessions}}" ("timestamp");
INSERT INTO {{schema_version}} (version) VALUES (19);
`
pgsqlV20SQL = `CREATE TABLE "{{events_rules}}" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL UNIQUE,
CREATE TABLE "{{events_rules}}" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL UNIQUE,
"description" varchar(512) NULL, "created_at" bigint NOT NULL, "updated_at" bigint NOT NULL, "trigger" integer NOT NULL,
"conditions" text NOT NULL, "deleted_at" bigint NOT NULL);
CREATE TABLE "{{events_actions}}" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL UNIQUE,
@@ -162,46 +144,41 @@ ALTER TABLE "{{rules_actions_mapping}}" ADD CONSTRAINT "{{prefix}}rules_actions_
FOREIGN KEY ("rule_id") REFERENCES "{{events_rules}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE;
ALTER TABLE "{{rules_actions_mapping}}" ADD CONSTRAINT "{{prefix}}rules_actions_mapping_action_id_fk_events_targets_id"
FOREIGN KEY ("action_id") REFERENCES "{{events_actions}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE NO ACTION;
ALTER TABLE "{{users}}" ADD COLUMN "deleted_at" bigint DEFAULT 0 NOT NULL;
ALTER TABLE "{{users}}" ALTER COLUMN "deleted_at" DROP DEFAULT;
CREATE INDEX "{{prefix}}events_rules_updated_at_idx" ON "{{events_rules}}" ("updated_at");
CREATE INDEX "{{prefix}}events_rules_deleted_at_idx" ON "{{events_rules}}" ("deleted_at");
CREATE INDEX "{{prefix}}events_rules_trigger_idx" ON "{{events_rules}}" ("trigger");
CREATE INDEX "{{prefix}}rules_actions_mapping_rule_id_idx" ON "{{rules_actions_mapping}}" ("rule_id");
CREATE INDEX "{{prefix}}rules_actions_mapping_action_id_idx" ON "{{rules_actions_mapping}}" ("action_id");
CREATE INDEX "{{prefix}}rules_actions_mapping_order_idx" ON "{{rules_actions_mapping}}" ("order");
CREATE INDEX "{{prefix}}users_deleted_at_idx" ON "{{users}}" ("deleted_at");
`
pgsqlV20DownSQL = `DROP TABLE "{{rules_actions_mapping}}" CASCADE;
DROP TABLE "{{events_rules}}" CASCADE;
DROP TABLE "{{events_actions}}" CASCADE;
DROP TABLE "{{tasks}}" CASCADE;
ALTER TABLE "{{users}}" DROP COLUMN "deleted_at" CASCADE;
`
pgsqlV21SQL = `ALTER TABLE "{{users}}" ADD COLUMN "first_download" bigint DEFAULT 0 NOT NULL;
ALTER TABLE "{{users}}" ALTER COLUMN "first_download" DROP DEFAULT;
ALTER TABLE "{{users}}" ADD COLUMN "first_upload" bigint DEFAULT 0 NOT NULL;
ALTER TABLE "{{users}}" ALTER COLUMN "first_upload" DROP DEFAULT;
`
pgsqlV21DownSQL = `ALTER TABLE "{{users}}" DROP COLUMN "first_upload" CASCADE;
ALTER TABLE "{{users}}" DROP COLUMN "first_download" CASCADE;
`
pgsqlV22SQL = `CREATE TABLE "{{admins_groups_mapping}}" ("id" serial NOT NULL PRIMARY KEY,
CREATE TABLE "{{admins_groups_mapping}}" ("id" serial NOT NULL PRIMARY KEY,
"admin_id" integer NOT NULL, "group_id" integer NOT NULL, "options" text NOT NULL);
ALTER TABLE "{{admins_groups_mapping}}" ADD CONSTRAINT "{{prefix}}unique_admin_group_mapping" UNIQUE ("admin_id", "group_id");
ALTER TABLE "{{admins_groups_mapping}}" ADD CONSTRAINT "{{prefix}}admins_groups_mapping_admin_id_fk_admins_id"
FOREIGN KEY ("admin_id") REFERENCES "{{admins}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE;
ALTER TABLE "{{admins_groups_mapping}}" ADD CONSTRAINT "{{prefix}}admins_groups_mapping_group_id_fk_groups_id"
FOREIGN KEY ("group_id") REFERENCES "{{groups}}" ("id") MATCH SIMPLE ON UPDATE NO ACTION ON DELETE CASCADE;
CREATE TABLE "{{nodes}}" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL UNIQUE,
"data" text NOT NULL, "created_at" bigint NOT NULL, "updated_at" bigint NOT NULL);
CREATE INDEX "{{prefix}}users_folders_mapping_folder_id_idx" ON "{{users_folders_mapping}}" ("folder_id");
CREATE INDEX "{{prefix}}users_folders_mapping_user_id_idx" ON "{{users_folders_mapping}}" ("user_id");
CREATE INDEX "{{prefix}}api_keys_admin_id_idx" ON "{{api_keys}}" ("admin_id");
CREATE INDEX "{{prefix}}api_keys_user_id_idx" ON "{{api_keys}}" ("user_id");
CREATE INDEX "{{prefix}}users_updated_at_idx" ON "{{users}}" ("updated_at");
CREATE INDEX "{{prefix}}users_deleted_at_idx" ON "{{users}}" ("deleted_at");
CREATE INDEX "{{prefix}}shares_user_id_idx" ON "{{shares}}" ("user_id");
CREATE INDEX "{{prefix}}defender_hosts_updated_at_idx" ON "{{defender_hosts}}" ("updated_at");
CREATE INDEX "{{prefix}}defender_hosts_ban_time_idx" ON "{{defender_hosts}}" ("ban_time");
CREATE INDEX "{{prefix}}defender_events_date_time_idx" ON "{{defender_events}}" ("date_time");
CREATE INDEX "{{prefix}}defender_events_host_id_idx" ON "{{defender_events}}" ("host_id");
CREATE INDEX "{{prefix}}active_transfers_connection_id_idx" ON "{{active_transfers}}" ("connection_id");
CREATE INDEX "{{prefix}}active_transfers_transfer_id_idx" ON "{{active_transfers}}" ("transfer_id");
CREATE INDEX "{{prefix}}active_transfers_updated_at_idx" ON "{{active_transfers}}" ("updated_at");
CREATE INDEX "{{prefix}}shared_sessions_type_idx" ON "{{shared_sessions}}" ("type");
CREATE INDEX "{{prefix}}shared_sessions_timestamp_idx" ON "{{shared_sessions}}" ("timestamp");
CREATE INDEX "{{prefix}}events_rules_updated_at_idx" ON "{{events_rules}}" ("updated_at");
CREATE INDEX "{{prefix}}events_rules_deleted_at_idx" ON "{{events_rules}}" ("deleted_at");
CREATE INDEX "{{prefix}}events_rules_trigger_idx" ON "{{events_rules}}" ("trigger");
CREATE INDEX "{{prefix}}rules_actions_mapping_rule_id_idx" ON "{{rules_actions_mapping}}" ("rule_id");
CREATE INDEX "{{prefix}}rules_actions_mapping_action_id_idx" ON "{{rules_actions_mapping}}" ("action_id");
CREATE INDEX "{{prefix}}rules_actions_mapping_order_idx" ON "{{rules_actions_mapping}}" ("order");
CREATE INDEX "{{prefix}}admins_groups_mapping_admin_id_idx" ON "{{admins_groups_mapping}}" ("admin_id");
CREATE INDEX "{{prefix}}admins_groups_mapping_group_id_idx" ON "{{admins_groups_mapping}}" ("group_id");
INSERT INTO {{schema_version}} (version) VALUES (23);
`
pgsqlV22DownSQL = `ALTER TABLE "{{admins_groups_mapping}}" DROP CONSTRAINT "{{prefix}}unique_admin_group_mapping";
DROP TABLE "{{admins_groups_mapping}}" CASCADE;
`
pgsqlV23SQL = `CREATE TABLE "{{nodes}}" ("id" serial NOT NULL PRIMARY KEY, "name" varchar(255) NOT NULL UNIQUE,
"data" text NOT NULL, "created_at" bigint NOT NULL, "updated_at" bigint NOT NULL);`
pgsqlV23DownSQL = `DROP TABLE "{{nodes}}" CASCADE;`
)
// PGSQLProvider defines the auth provider for PostgreSQL database
@@ -669,11 +646,11 @@ func (p *PGSQLProvider) initializeDatabase() error {
if errors.Is(err, sql.ErrNoRows) {
return errSchemaVersionEmpty
}
logger.InfoToConsole("creating initial database schema, version 19")
providerLog(logger.LevelInfo, "creating initial database schema, version 19")
logger.InfoToConsole("creating initial database schema, version 23")
providerLog(logger.LevelInfo, "creating initial database schema, version 23")
initialSQL := sqlReplaceAll(pgsqlInitial)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 19, true)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{initialSQL}, 23, true)
}
func (p *PGSQLProvider) migrateDatabase() error { //nolint:dupl
@@ -684,30 +661,22 @@ func (p *PGSQLProvider) migrateDatabase() error { //nolint:dupl
switch version := dbVersion.Version; {
case version == sqlDatabaseVersion:
providerLog(logger.LevelDebug, "sql database is up to date, current version: %v", version)
providerLog(logger.LevelDebug, "sql database is up to date, current version: %d", version)
return ErrNoInitRequired
case version < 19:
err = fmt.Errorf("database schema version %v is too old, please see the upgrading docs", version)
case version < 23:
err = fmt.Errorf("database schema version %d is too old, please see the upgrading docs", version)
providerLog(logger.LevelError, "%v", err)
logger.ErrorToConsole("%v", err)
return err
case version == 19:
return updatePgSQLDatabaseFromV19(p.dbHandle)
case version == 20:
return updatePgSQLDatabaseFromV20(p.dbHandle)
case version == 21:
return updatePgSQLDatabaseFromV21(p.dbHandle)
case version == 22:
return updatePgSQLDatabaseFromV21(p.dbHandle)
default:
if version > sqlDatabaseVersion {
providerLog(logger.LevelError, "database schema version %v is newer than the supported one: %v", version,
providerLog(logger.LevelError, "database schema version %d is newer than the supported one: %d", version,
sqlDatabaseVersion)
logger.WarnToConsole("database schema version %v is newer than the supported one: %v", version,
logger.WarnToConsole("database schema version %d is newer than the supported one: %d", version,
sqlDatabaseVersion)
return nil
}
return fmt.Errorf("database schema version not handled: %v", version)
return fmt.Errorf("database schema version not handled: %d", version)
}
}
@@ -721,16 +690,8 @@ func (p *PGSQLProvider) revertDatabase(targetVersion int) error {
}
switch dbVersion.Version {
case 20:
return downgradePgSQLDatabaseFromV20(p.dbHandle)
case 21:
return downgradePgSQLDatabaseFromV21(p.dbHandle)
case 22:
return downgradePgSQLDatabaseFromV22(p.dbHandle)
case 23:
return downgradePgSQLDatabaseFromV23(p.dbHandle)
default:
return fmt.Errorf("database schema version not handled: %v", dbVersion.Version)
return fmt.Errorf("database schema version not handled: %d", dbVersion.Version)
}
}
@@ -738,136 +699,3 @@ func (p *PGSQLProvider) resetDatabase() error {
sql := sqlReplaceAll(pgsqlResetSQL)
return sqlCommonExecSQLAndUpdateDBVersion(p.dbHandle, []string{sql}, 0, false)
}
func updatePgSQLDatabaseFromV19(dbHandle *sql.DB) error {
if err := updatePgSQLDatabaseFrom19To20(dbHandle); err != nil {
return err
}
return updatePgSQLDatabaseFromV20(dbHandle)
}
func updatePgSQLDatabaseFromV20(dbHandle *sql.DB) error {
if err := updatePgSQLDatabaseFrom20To21(dbHandle); err != nil {
return err
}
return updatePgSQLDatabaseFromV21(dbHandle)
}
func updatePgSQLDatabaseFromV21(dbHandle *sql.DB) error {
if err := updatePgSQLDatabaseFrom21To22(dbHandle); err != nil {
return err
}
return updatePgSQLDatabaseFromV22(dbHandle)
}
func updatePgSQLDatabaseFromV22(dbHandle *sql.DB) error {
return updatePgSQLDatabaseFrom22To23(dbHandle)
}
func downgradePgSQLDatabaseFromV20(dbHandle *sql.DB) error {
return downgradePgSQLDatabaseFrom20To19(dbHandle)
}
func downgradePgSQLDatabaseFromV21(dbHandle *sql.DB) error {
if err := downgradePgSQLDatabaseFrom21To20(dbHandle); err != nil {
return err
}
return downgradePgSQLDatabaseFromV20(dbHandle)
}
func downgradePgSQLDatabaseFromV22(dbHandle *sql.DB) error {
if err := downgradePgSQLDatabaseFrom22To21(dbHandle); err != nil {
return err
}
return downgradePgSQLDatabaseFromV21(dbHandle)
}
func downgradePgSQLDatabaseFromV23(dbHandle *sql.DB) error {
if err := downgradePgSQLDatabaseFrom23To22(dbHandle); err != nil {
return err
}
return downgradePgSQLDatabaseFromV22(dbHandle)
}
func updatePgSQLDatabaseFrom19To20(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 19 -> 20")
providerLog(logger.LevelInfo, "updating database schema version: 19 -> 20")
sql := pgsqlV20SQL
if config.Driver == CockroachDataProviderName {
sql = strings.ReplaceAll(sql, `ALTER TABLE "{{users}}" ALTER COLUMN "deleted_at" DROP DEFAULT;`, "")
}
sql = strings.ReplaceAll(sql, "{{events_actions}}", sqlTableEventsActions)
sql = strings.ReplaceAll(sql, "{{events_rules}}", sqlTableEventsRules)
sql = strings.ReplaceAll(sql, "{{rules_actions_mapping}}", sqlTableRulesActionsMapping)
sql = strings.ReplaceAll(sql, "{{tasks}}", sqlTableTasks)
sql = strings.ReplaceAll(sql, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 20, true)
}
func updatePgSQLDatabaseFrom20To21(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 20 -> 21")
providerLog(logger.LevelInfo, "updating database schema version: 20 -> 21")
sql := pgsqlV21SQL
if config.Driver == CockroachDataProviderName {
sql = strings.ReplaceAll(sql, `ALTER TABLE "{{users}}" ALTER COLUMN "first_download" DROP DEFAULT;`, "")
sql = strings.ReplaceAll(sql, `ALTER TABLE "{{users}}" ALTER COLUMN "first_upload" DROP DEFAULT;`, "")
}
sql = strings.ReplaceAll(sql, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 21, true)
}
func updatePgSQLDatabaseFrom21To22(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 21 -> 22")
providerLog(logger.LevelInfo, "updating database schema version: 21 -> 22")
sql := strings.ReplaceAll(pgsqlV22SQL, "{{admins_groups_mapping}}", sqlTableAdminsGroupsMapping)
sql = strings.ReplaceAll(sql, "{{admins}}", sqlTableAdmins)
sql = strings.ReplaceAll(sql, "{{groups}}", sqlTableGroups)
sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 22, true)
}
func updatePgSQLDatabaseFrom22To23(dbHandle *sql.DB) error {
logger.InfoToConsole("updating database schema version: 22 -> 23")
providerLog(logger.LevelInfo, "updating database schema version: 22 -> 23")
sql := strings.ReplaceAll(pgsqlV23SQL, "{{nodes}}", sqlTableNodes)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 23, true)
}
func downgradePgSQLDatabaseFrom20To19(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 20 -> 19")
providerLog(logger.LevelInfo, "downgrading database schema version: 20 -> 19")
sql := strings.ReplaceAll(pgsqlV20DownSQL, "{{events_actions}}", sqlTableEventsActions)
sql = strings.ReplaceAll(sql, "{{events_rules}}", sqlTableEventsRules)
sql = strings.ReplaceAll(sql, "{{rules_actions_mapping}}", sqlTableRulesActionsMapping)
sql = strings.ReplaceAll(sql, "{{users}}", sqlTableUsers)
sql = strings.ReplaceAll(sql, "{{tasks}}", sqlTableTasks)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 19, false)
}
func downgradePgSQLDatabaseFrom21To20(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 21 -> 20")
providerLog(logger.LevelInfo, "downgrading database schema version: 21 -> 20")
sql := strings.ReplaceAll(pgsqlV21DownSQL, "{{users}}", sqlTableUsers)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 20, false)
}
func downgradePgSQLDatabaseFrom22To21(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 22 -> 21")
providerLog(logger.LevelInfo, "downgrading database schema version: 22 -> 21")
sql := pgsqlV22DownSQL
if config.Driver == CockroachDataProviderName {
sql = strings.ReplaceAll(sql, `ALTER TABLE "{{admins_groups_mapping}}" DROP CONSTRAINT "{{prefix}}unique_admin_group_mapping";`,
`DROP INDEX "{{prefix}}unique_admin_group_mapping" CASCADE;`)
}
sql = strings.ReplaceAll(sql, "{{admins_groups_mapping}}", sqlTableAdminsGroupsMapping)
sql = strings.ReplaceAll(sql, "{{prefix}}", config.SQLTablesPrefix)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 21, false)
}
func downgradePgSQLDatabaseFrom23To22(dbHandle *sql.DB) error {
logger.InfoToConsole("downgrading database schema version: 23 -> 22")
providerLog(logger.LevelInfo, "downgrading database schema version: 23 -> 22")
sql := strings.ReplaceAll(pgsqlV23DownSQL, "{{nodes}}", sqlTableNodes)
return sqlCommonExecSQLAndUpdateDBVersion(dbHandle, []string{sql}, 22, false)
}