Merge pull request #8616 from heww/db-connection-pool

feat(configuration,db): connection pool configs for db

make/harbor.yml

@@ -30,6 +30,11 @@ harbor_admin_password: Harbor12345
 database:
   # The password for the root user of Harbor DB. Change this before any production use.
   password: root123
+  # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
+  max_idle_conns: 50
+  # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
+  # Note: the default number of connections is 100 for postgres.
+  max_open_conns: 100
 
 # The default data volume
 data_volume: /data
@@ -50,12 +55,12 @@ data_volume: /data
 #     disabled: false
 
 # Clair configuration
-clair: 
+clair:
   # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
   updaters_interval: 12
 
 jobservice:
-  # Maximum number of job workers in job service  
+  # Maximum number of job workers in job service
   max_job_workers: 10
 
 notification:
@@ -74,8 +79,8 @@ log:
   local:
     # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
     rotate_count: 50
-    # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
+    # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
-    # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
+    # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
     # are all valid.
     rotate_size: 200M
     # The directory on your host that store log

make/photon/prepare/templates/core/env.jinja

@@ -15,6 +15,8 @@ POSTGRESQL_USERNAME={{harbor_db_username}}
 POSTGRESQL_PASSWORD={{harbor_db_password}}
 POSTGRESQL_DATABASE={{harbor_db_name}}
 POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
+POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
+POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
 REGISTRY_URL={{registry_url}}
 TOKEN_SERVICE_URL={{token_service_url}}
 HARBOR_ADMIN_PASSWORD={{harbor_admin_password}}

make/photon/prepare/utils/configs.py

@@ -112,6 +112,11 @@ def parse_yaml_config(config_file_path):
         config_dict['harbor_db_username'] = 'postgres'
         config_dict['harbor_db_password'] = db_configs.get("password") or ''
         config_dict['harbor_db_sslmode'] = 'disable'
+
+        default_max_idle_conns = 2  # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
+        default_max_open_conns = 0  # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
+        config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_max_idle_conns
+        config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_max_open_conns
         # clari db
         config_dict['clair_db_host'] = 'postgresql'
         config_dict['clair_db_port'] = 5432

src/common/config/manager.go

@@ -210,12 +210,14 @@ func (c *CfgManager) GetDatabaseCfg() *models.Database {
 	return &models.Database{
 		Type: c.Get(common.DatabaseType).GetString(),
 		PostGreSQL: &models.PostGreSQL{
-			Host:     c.Get(common.PostGreSQLHOST).GetString(),
+			Host:         c.Get(common.PostGreSQLHOST).GetString(),
-			Port:     c.Get(common.PostGreSQLPort).GetInt(),
+			Port:         c.Get(common.PostGreSQLPort).GetInt(),
-			Username: c.Get(common.PostGreSQLUsername).GetString(),
+			Username:     c.Get(common.PostGreSQLUsername).GetString(),
-			Password: c.Get(common.PostGreSQLPassword).GetString(),
+			Password:     c.Get(common.PostGreSQLPassword).GetString(),
-			Database: c.Get(common.PostGreSQLDatabase).GetString(),
+			Database:     c.Get(common.PostGreSQLDatabase).GetString(),
-			SSLMode:  c.Get(common.PostGreSQLSSLMode).GetString(),
+			SSLMode:      c.Get(common.PostGreSQLSSLMode).GetString(),
+			MaxIdleConns: c.Get(common.PostGreSQLMaxIdleConns).GetInt(),
+			MaxOpenConns: c.Get(common.PostGreSQLMaxOpenConns).GetInt(),
 		},
 	}
 }

src/common/config/metadata/metadatalist.go

@@ -116,6 +116,8 @@ var (
 		{Name: common.PostGreSQLPort, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_PORT", DefaultValue: "5432", ItemType: &PortType{}, Editable: false},
 		{Name: common.PostGreSQLSSLMode, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false},
 		{Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
+		{Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false},
+		{Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false},
 
 		{Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false},
 		{Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},

src/common/const.go

@@ -53,6 +53,8 @@ const (
 	PostGreSQLPassword               = "postgresql_password"
 	PostGreSQLDatabase               = "postgresql_database"
 	PostGreSQLSSLMode                = "postgresql_sslmode"
+	PostGreSQLMaxIdleConns           = "postgresql_max_idle_conns"
+	PostGreSQLMaxOpenConns           = "postgresql_max_open_conns"
 	SelfRegistration                 = "self_registration"
 	CoreURL                          = "core_url"
 	CoreLocalURL                     = "core_local_url"

src/common/dao/base.go

@@ -121,12 +121,16 @@ func getDatabase(database *models.Database) (db Database, err error) {
 
 	switch database.Type {
 	case "", "postgresql":
-		db = NewPGSQL(database.PostGreSQL.Host,
+		db = NewPGSQL(
+			database.PostGreSQL.Host,
 			strconv.Itoa(database.PostGreSQL.Port),
 			database.PostGreSQL.Username,
 			database.PostGreSQL.Password,
 			database.PostGreSQL.Database,
-			database.PostGreSQL.SSLMode)
+			database.PostGreSQL.SSLMode,
+			database.PostGreSQL.MaxIdleConns,
+			database.PostGreSQL.MaxOpenConns,
+		)
 	default:
 		err = fmt.Errorf("invalid database: %s", database.Type)
 	}

src/common/dao/pgsql.go

@@ -31,12 +31,14 @@ import (
 const defaultMigrationPath = "migrations/postgresql/"
 
 type pgsql struct {
-	host     string
+	host         string
-	port     string
+	port         string
-	usr      string
+	usr          string
-	pwd      string
+	pwd          string
-	database string
+	database     string
-	sslmode  string
+	sslmode      string
+	maxIdleConns int
+	maxOpenConns int
 }
 
 // Name returns the name of PostgreSQL
@@ -51,17 +53,19 @@ func (p *pgsql) String() string {
 }
 
 // NewPGSQL returns an instance of postgres
-func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string) Database {
+func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int) Database {
 	if len(sslmode) == 0 {
 		sslmode = "disable"
 	}
 	return &pgsql{
-		host:     host,
+		host:         host,
-		port:     port,
+		port:         port,
-		usr:      usr,
+		usr:          usr,
-		pwd:      pwd,
+		pwd:          pwd,
-		database: database,
+		database:     database,
-		sslmode:  sslmode,
+		sslmode:      sslmode,
+		maxIdleConns: maxIdleConns,
+		maxOpenConns: maxOpenConns,
 	}
 }
 
@@ -82,7 +86,7 @@ func (p *pgsql) Register(alias ...string) error {
 	info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
 		p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
 
-	return orm.RegisterDataBase(an, "postgres", info)
+	return orm.RegisterDataBase(an, "postgres", info, p.maxIdleConns, p.maxOpenConns)
 }
 
 // UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts.

src/common/models/config.go

@@ -45,12 +45,14 @@ type SQLite struct {
 
 // PostGreSQL ...
 type PostGreSQL struct {
-	Host     string `json:"host"`
+	Host         string `json:"host"`
-	Port     int    `json:"port"`
+	Port         int    `json:"port"`
-	Username string `json:"username"`
+	Username     string `json:"username"`
-	Password string `json:"password,omitempty"`
+	Password     string `json:"password,omitempty"`
-	Database string `json:"database"`
+	Database     string `json:"database"`
-	SSLMode  string `json:"sslmode"`
+	SSLMode      string `json:"sslmode"`
+	MaxIdleConns int    `json:"max_idle_conns"`
+	MaxOpenConns int    `json:"max_open_conns"`
 }
 
 // Email ...

src/core/config/config.go

@@ -331,12 +331,14 @@ func Database() (*models.Database, error) {
 	database := &models.Database{}
 	database.Type = cfgMgr.Get(common.DatabaseType).GetString()
 	postgresql := &models.PostGreSQL{
-		Host:     cfgMgr.Get(common.PostGreSQLHOST).GetString(),
+		Host:         cfgMgr.Get(common.PostGreSQLHOST).GetString(),
-		Port:     cfgMgr.Get(common.PostGreSQLPort).GetInt(),
+		Port:         cfgMgr.Get(common.PostGreSQLPort).GetInt(),
-		Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(),
+		Username:     cfgMgr.Get(common.PostGreSQLUsername).GetString(),
-		Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(),
+		Password:     cfgMgr.Get(common.PostGreSQLPassword).GetString(),
-		Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(),
+		Database:     cfgMgr.Get(common.PostGreSQLDatabase).GetString(),
-		SSLMode:  cfgMgr.Get(common.PostGreSQLSSLMode).GetString(),
+		SSLMode:      cfgMgr.Get(common.PostGreSQLSSLMode).GetString(),
+		MaxIdleConns: cfgMgr.Get(common.PostGreSQLMaxIdleConns).GetInt(),
+		MaxOpenConns: cfgMgr.Get(common.PostGreSQLMaxOpenConns).GetInt(),
 	}
 	database.PostGreSQL = postgresql