dataprovider: remove transaction for quota update

The update is atomic so no transaction is needed.
Addionally a transaction will ask for a new connection to the pool
and this can deadlock if the pool has a max connection limit too low.

Also make configurable the pool size instead of hard code to the cpu number.

Fixes #47

README.md

@@ -153,6 +153,7 @@ The `sftpgo` configuration file contains the following sections:
         - 0, disable quota tracking. REST API to scan user dir and update quota will do nothing
         - 1, quota is updated each time a user upload or delete a file even if the user has no quota restrictions
         - 2, quota is updated each time a user upload or delete a file but only for users with quota restrictions. With this configuration the "quota scan" REST API can still be used to periodically update space usage for users without quota restrictions
+    - `pool_size`, integer. Sets the maximum number of open connections for mysql and postgresql driver. Default 0 (unlimited)
 - **"httpd"**, the configuration for the HTTP server used to serve REST API
     - `bind_port`, integer. The port used for serving HTTP requests. Set to 0 to disable HTTP server. Default: 8080
     - `bind_address`, string. Leave blank to listen on all available network interfaces. Default: "127.0.0.1"

config/config.go

@@ -73,6 +73,7 @@ func init() {
 			ManageUsers:      1,
 			SSLMode:          0,
 			TrackQuota:       1,
+			PoolSize:         0,
 		},
 		HTTPDConfig: api.HTTPDConf{
 			BindPort:    8080,

dataprovider/dataprovider.go

@@ -91,6 +91,9 @@ type Config struct {
 	//    With this configuration the "quota scan" REST API can still be used to periodically update space usage
 	//    for users without quota restrictions
 	TrackQuota int `json:"track_quota" mapstructure:"track_quota"`
+	// Sets the maximum number of open connections for mysql and postgresql driver.
+	// Default 0 (unlimited)
+	PoolSize int `json:"pool_size" mapstructure:"pool_size"`
 }
 
 // ValidationError raised if input data is not valid

dataprovider/mysql.go

@@ -3,7 +3,6 @@ package dataprovider
 import (
 	"database/sql"
 	"fmt"
-	"runtime"
 	"time"
 
 	"github.com/drakkan/sftpgo/logger"
@@ -26,10 +25,8 @@ func initializeMySQLProvider() error {
 	}
 	dbHandle, err := sql.Open("mysql", connectionString)
 	if err == nil {
-		numCPU := runtime.NumCPU()
+		providerLog(logger.LevelDebug, "mysql database handle created, connection string: %#v, pool size: %v", connectionString, config.PoolSize)
-		providerLog(logger.LevelDebug, "mysql database handle created, connection string: %#v, pool size: %v", connectionString, numCPU)
+		dbHandle.SetMaxOpenConns(config.PoolSize)
-		dbHandle.SetMaxIdleConns(numCPU)
-		dbHandle.SetMaxOpenConns(numCPU)
 		dbHandle.SetConnMaxLifetime(1800 * time.Second)
 		provider = MySQLProvider{dbHandle: dbHandle}
 	} else {
@@ -51,21 +48,7 @@ func (p MySQLProvider) getUserByID(ID int64) (User, error) {
 }
 
 func (p MySQLProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error {
-	tx, err := p.dbHandle.Begin()
+	return sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle)
-	if err != nil {
-		providerLog(logger.LevelWarn, "error starting transaction to update quota for user %v: %v", username, err)
-		return err
-	}
-	err = sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle)
-	if err == nil {
-		err = tx.Commit()
-	} else {
-		err = tx.Rollback()
-	}
-	if err != nil {
-		providerLog(logger.LevelWarn, "error closing transaction to update quota for user %v: %v", username, err)
-	}
-	return err
 }
 
 func (p MySQLProvider) getUsedQuota(username string) (int, int64, error) {

dataprovider/pgsql.go

@@ -3,7 +3,6 @@ package dataprovider
 import (
 	"database/sql"
 	"fmt"
-	"runtime"
 
 	"github.com/drakkan/sftpgo/logger"
 )
@@ -25,10 +24,8 @@ func initializePGSQLProvider() error {
 	}
 	dbHandle, err := sql.Open("postgres", connectionString)
 	if err == nil {
-		numCPU := runtime.NumCPU()
+		providerLog(logger.LevelDebug, "postgres database handle created, connection string: %#v, pool size: %v", connectionString, config.PoolSize)
-		providerLog(logger.LevelDebug, "postgres database handle created, connection string: %#v, pool size: %v", connectionString, numCPU)
+		dbHandle.SetMaxOpenConns(config.PoolSize)
-		dbHandle.SetMaxIdleConns(numCPU)
-		dbHandle.SetMaxOpenConns(numCPU)
 		provider = PGSQLProvider{dbHandle: dbHandle}
 	} else {
 		providerLog(logger.LevelWarn, "error creating postgres database handler, connection string: %#v, error: %v", connectionString, err)
@@ -49,21 +46,7 @@ func (p PGSQLProvider) getUserByID(ID int64) (User, error) {
 }
 
 func (p PGSQLProvider) updateQuota(username string, filesAdd int, sizeAdd int64, reset bool) error {
-	tx, err := p.dbHandle.Begin()
+	return sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle)
-	if err != nil {
-		providerLog(logger.LevelWarn, "error starting transaction to update quota for user %v: %v", username, err)
-		return err
-	}
-	err = sqlCommonUpdateQuota(username, filesAdd, sizeAdd, reset, p.dbHandle)
-	if err == nil {
-		err = tx.Commit()
-	} else {
-		err = tx.Rollback()
-	}
-	if err != nil {
-		providerLog(logger.LevelWarn, "error closing transaction to update quota for user %v: %v", username, err)
-	}
-	return err
 }
 
 func (p PGSQLProvider) getUsedQuota(username string) (int, int64, error) {

sftpgo.json

@@ -30,7 +30,8 @@
     "connection_string": "",
     "users_table": "users",
     "manage_users": 1,
-    "track_quota": 2
+    "track_quota": 2,
+    "pool_size": 0
   },
   "httpd": {
     "bind_port": 8080,