From e7eb3476b7a38e567a31ebc0fd14e100fbcad17e Mon Sep 17 00:00:00 2001 From: Nicola Murino Date: Fri, 13 Sep 2019 08:14:07 +0200 Subject: [PATCH] dataprovider: remove transaction for quota update The update is atomic so no transaction is needed. Addionally a transaction will ask for a new connection to the pool and this can deadlock if the pool has a max connection limit too low. Also make configurable the pool size instead of hard code to the cpu number. Fixes #47 --- README.md | 1 + config/config.go | 1 + dataprovider/dataprovider.go | 3 +++ dataprovider/mysql.go | 23 +++-------------------- dataprovider/pgsql.go | 23 +++-------------------- sftpgo.json | 3 ++- 6 files changed, 13 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 6f12566e..eebc206e 100644 --- a/README.md +++ b/README.md @@ -153,6 +153,7 @@ The `sftpgo` configuration file contains the following sections: - 0, disable quota tracking. REST API to scan user dir and update quota will do nothing - 1, quota is updated each time a user upload or delete a file even if the user has no quota restrictions - 2, quota is updated each time a user upload or delete a file but only for users with quota restrictions. With this configuration the "quota scan" REST API can still be used to periodically update space usage for users without quota restrictions + - `pool_size`, integer. Sets the maximum number of open connections for mysql and postgresql driver. Default 0 (unlimited) - **"httpd"**, the configuration for the HTTP server used to serve REST API - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080 - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1" diff --git a/config/config.go b/config/config.go index 40d47ebe..91af7f4e 100644 --- a/config/config.go +++ b/config/config.go @@ -73,6 +73,7 @@ func init() { ManageUsers: 1, SSLMode: 0, TrackQuota: 1, + PoolSize: 0, }, HTTPDConfig: api.HTTPDConf{ BindPort: 8080, diff --git a/dataprovider/dataprovider.go b/dataprovider/dataprovider.go index f9077a17..d0aef535 100644 --- a/dataprovider/dataprovider.go +++ b/dataprovider/dataprovider.go @@ -91,6 +91,9 @@ type Config struct { // With this configuration the "quota scan" REST API can still be used to periodically update space usage // for users without quota restrictions TrackQuota int `json:"track_quota" mapstructure:"track_quota"` + // Sets the maximum number of open connections for mysql and postgresql driver. + // Default 0 (unlimited) + PoolSize int `json:"pool_size" mapstructure:"pool_size"` } // ValidationError raised if input data is not valid diff --git a/dataprovider/mysql.go b/dataprovider/mysql.go index 47bdeb91..e8f7f7df 100644 --- a/dataprovider/mysql.go +++ b/dataprovider/mysql.go @@ -3,7 +3,6 @@ package dataprovider import ( "database/sql" "fmt" - "runtime" "time" "github.com/drakkan/sftpgo/logger" @@ -26,10 +25,8 @@ func initializeMySQLProvider() error { } dbHandle, err := sql.Open("mysql", connectionString) if err == nil { - numCPU := runtime.NumCPU() - providerLog(logger.LevelDebug, "mysql database handle created, connection string: %#v, pool size: %v", connectionString, numCPU) - dbHandle.SetMaxIdleConns(numCPU) - dbHandle.SetMaxOpenConns(numCPU) + providerLog(logger.LevelDebug, "mysql database handle created, connection string: %#v, pool size: %v", connectionString, config.PoolSize) + dbHandle.SetMaxOpenConns(config.PoolSize) dbHandle.SetConnMaxLifetime(1800 * time.Second) provider = MySQLProvider{dbHandle: dbHandle} } else { @@ -51,21 +48,7 @@ func (p MySQLProvider) getUserByID(ID int64) (User, error) { } func (p MySQLProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error { - tx, err := p.dbHandle.Begin() - if err != nil { - providerLog(logger.LevelWarn, "error starting transaction to update quota for user %v: %v", username, err) - return err - } - err = sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle) - if err == nil { - err = tx.Commit() - } else { - err = tx.Rollback() - } - if err != nil { - providerLog(logger.LevelWarn, "error closing transaction to update quota for user %v: %v", username, err) - } - return err + return sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle) } func (p MySQLProvider) getUsedQuota(username string) (int, int64, error) { diff --git a/dataprovider/pgsql.go b/dataprovider/pgsql.go index 94a92de0..bbdc23d3 100644 --- a/dataprovider/pgsql.go +++ b/dataprovider/pgsql.go @@ -3,7 +3,6 @@ package dataprovider import ( "database/sql" "fmt" - "runtime" "github.com/drakkan/sftpgo/logger" ) @@ -25,10 +24,8 @@ func initializePGSQLProvider() error { } dbHandle, err := sql.Open("postgres", connectionString) if err == nil { - numCPU := runtime.NumCPU() - providerLog(logger.LevelDebug, "postgres database handle created, connection string: %#v, pool size: %v", connectionString, numCPU) - dbHandle.SetMaxIdleConns(numCPU) - dbHandle.SetMaxOpenConns(numCPU) + providerLog(logger.LevelDebug, "postgres database handle created, connection string: %#v, pool size: %v", connectionString, config.PoolSize) + dbHandle.SetMaxOpenConns(config.PoolSize) provider = PGSQLProvider{dbHandle: dbHandle} } else { providerLog(logger.LevelWarn, "error creating postgres database handler, connection string: %#v, error: %v", connectionString, err) @@ -49,21 +46,7 @@ func (p PGSQLProvider) getUserByID(ID int64) (User, error) { } func (p PGSQLProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error { - tx, err := p.dbHandle.Begin() - if err != nil { - providerLog(logger.LevelWarn, "error starting transaction to update quota for user %v: %v", username, err) - return err - } - err = sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle) - if err == nil { - err = tx.Commit() - } else { - err = tx.Rollback() - } - if err != nil { - providerLog(logger.LevelWarn, "error closing transaction to update quota for user %v: %v", username, err) - } - return err + return sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle) } func (p PGSQLProvider) getUsedQuota(username string) (int, int64, error) { diff --git a/sftpgo.json b/sftpgo.json index 125397af..addac2cf 100644 --- a/sftpgo.json +++ b/sftpgo.json @@ -30,7 +30,8 @@ "connection_string": "", "users_table": "users", "manage_users": 1, - "track_quota": 2 + "track_quota": 2, + "pool_size": 0 }, "httpd": { "bind_port": 8080,