Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(configuration,db): connection pool configs for db 

Signed-off-by: He Weiwei <hweiwei@vmware.com>
This commit is contained in:
He Weiwei 2019-08-12 17:14:57 +08:00
parent a545b460ed
commit 98e1f68468
10 changed files with 68 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
make/harbor.yml
View File

@ -30,6 +30,11 @@ harbor_admin_password: Harbor12345
database:
# The password for the root user of Harbor DB. Change this before any production use.
password: root123
# The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
max_idle_conns: 50
# The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.
# Note: the default number of connections is 100 for postgres.
max_open_conns: 100
# The default data volume
data_volume: /data
@ -50,12 +55,12 @@ data_volume: /data
# disabled: false
# Clair configuration
clair:
clair:
# The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
updaters_interval: 12
jobservice:
# Maximum number of job workers in job service
# Maximum number of job workers in job service
max_job_workers: 10
notification:
@ -74,8 +79,8 @@ log:
local:
# Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
rotate_count: 50
# Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
# If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
# Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
# If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
# are all valid.
rotate_size: 200M
# The directory on your host that store log

2
make/photon/prepare/templates/core/env.jinja
View File

@ -15,6 +15,8 @@ POSTGRESQL_USERNAME={{harbor_db_username}}
POSTGRESQL_PASSWORD={{harbor_db_password}}
POSTGRESQL_DATABASE={{harbor_db_name}}
POSTGRESQL_SSLMODE={{harbor_db_sslmode}}
POSTGRESQL_MAX_IDLE_CONNS={{harbor_db_max_idle_conns}}
POSTGRESQL_MAX_OPEN_CONNS={{harbor_db_max_open_conns}}
REGISTRY_URL={{registry_url}}
TOKEN_SERVICE_URL={{token_service_url}}
HARBOR_ADMIN_PASSWORD={{harbor_admin_password}}

5
make/photon/prepare/utils/configs.py
View File

@ -112,6 +112,11 @@ def parse_yaml_config(config_file_path):
config_dict['harbor_db_username'] = 'postgres'
config_dict['harbor_db_password'] = db_configs.get("password") or ''
config_dict['harbor_db_sslmode'] = 'disable'
default_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
default_max_open_conns = 0 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxOpenConns
config_dict['harbor_db_max_idle_conns'] = db_configs.get("max_idle_conns") or default_max_idle_conns
config_dict['harbor_db_max_open_conns'] = db_configs.get("max_open_conns") or default_max_open_conns
# clari db
config_dict['clair_db_host'] = 'postgresql'
config_dict['clair_db_port'] = 5432

14
src/common/config/manager.go
View File

@ -210,12 +210,14 @@ func (c *CfgManager) GetDatabaseCfg() *models.Database {
return &models.Database{
Type: c.Get(common.DatabaseType).GetString(),
PostGreSQL: &models.PostGreSQL{
Host: c.Get(common.PostGreSQLHOST).GetString(),
Port: c.Get(common.PostGreSQLPort).GetInt(),
Username: c.Get(common.PostGreSQLUsername).GetString(),
Password: c.Get(common.PostGreSQLPassword).GetString(),
Database: c.Get(common.PostGreSQLDatabase).GetString(),
SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(),
Host: c.Get(common.PostGreSQLHOST).GetString(),
Port: c.Get(common.PostGreSQLPort).GetInt(),
Username: c.Get(common.PostGreSQLUsername).GetString(),
Password: c.Get(common.PostGreSQLPassword).GetString(),
Database: c.Get(common.PostGreSQLDatabase).GetString(),
SSLMode: c.Get(common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: c.Get(common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: c.Get(common.PostGreSQLMaxOpenConns).GetInt(),
},
}
}

2
src/common/config/metadata/metadatalist.go
View File

@ -116,6 +116,8 @@ var (
{Name: common.PostGreSQLPort, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_PORT", DefaultValue: "5432", ItemType: &PortType{}, Editable: false},
{Name: common.PostGreSQLSSLMode, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_SSLMODE", DefaultValue: "disable", ItemType: &StringType{}, Editable: false},
{Name: common.PostGreSQLUsername, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_USERNAME", DefaultValue: "postgres", ItemType: &StringType{}, Editable: false},
{Name: common.PostGreSQLMaxIdleConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_IDLE_CONNS", DefaultValue: "2", ItemType: &IntType{}, Editable: false},
{Name: common.PostGreSQLMaxOpenConns, Scope: SystemScope, Group: DatabaseGroup, EnvKey: "POSTGRESQL_MAX_OPEN_CONNS", DefaultValue: "0", ItemType: &IntType{}, Editable: false},
{Name: common.ProjectCreationRestriction, Scope: UserScope, Group: BasicGroup, EnvKey: "PROJECT_CREATION_RESTRICTION", DefaultValue: common.ProCrtRestrEveryone, ItemType: &ProjectCreationRestrictionType{}, Editable: false},
{Name: common.ReadOnly, Scope: UserScope, Group: BasicGroup, EnvKey: "READ_ONLY", DefaultValue: "false", ItemType: &BoolType{}, Editable: false},

2
src/common/const.go
View File

@ -53,6 +53,8 @@ const (
PostGreSQLPassword = "postgresql_password"
PostGreSQLDatabase = "postgresql_database"
PostGreSQLSSLMode = "postgresql_sslmode"
PostGreSQLMaxIdleConns = "postgresql_max_idle_conns"
PostGreSQLMaxOpenConns = "postgresql_max_open_conns"
SelfRegistration = "self_registration"
CoreURL = "core_url"
CoreLocalURL = "core_local_url"

8
src/common/dao/base.go
View File

@ -121,12 +121,16 @@ func getDatabase(database *models.Database) (db Database, err error) {
switch database.Type {
case "", "postgresql":
db = NewPGSQL(database.PostGreSQL.Host,
db = NewPGSQL(
database.PostGreSQL.Host,
strconv.Itoa(database.PostGreSQL.Port),
database.PostGreSQL.Username,
database.PostGreSQL.Password,
database.PostGreSQL.Database,
database.PostGreSQL.SSLMode)
database.PostGreSQL.SSLMode,
database.PostGreSQL.MaxIdleConns,
database.PostGreSQL.MaxOpenConns,
)
default:
err = fmt.Errorf("invalid database: %s", database.Type)
}

32
src/common/dao/pgsql.go
View File

@ -31,12 +31,14 @@ import (
const defaultMigrationPath = "migrations/postgresql/"
type pgsql struct {
host string
port string
usr string
pwd string
database string
sslmode string
host string
port string
usr string
pwd string
database string
sslmode string
maxIdleConns int
maxOpenConns int
}
// Name returns the name of PostgreSQL
@ -51,17 +53,19 @@ func (p *pgsql) String() string {
}
// NewPGSQL returns an instance of postgres
func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string) Database {
func NewPGSQL(host string, port string, usr string, pwd string, database string, sslmode string, maxIdleConns int, maxOpenConns int) Database {
if len(sslmode) == 0 {
sslmode = "disable"
}
return &pgsql{
host: host,
port: port,
usr: usr,
pwd: pwd,
database: database,
sslmode: sslmode,
host: host,
port: port,
usr: usr,
pwd: pwd,
database: database,
sslmode: sslmode,
maxIdleConns: maxIdleConns,
maxOpenConns: maxOpenConns,
}
}
@ -82,7 +86,7 @@ func (p *pgsql) Register(alias ...string) error {
info := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
p.host, p.port, p.usr, p.pwd, p.database, p.sslmode)
return orm.RegisterDataBase(an, "postgres", info)
return orm.RegisterDataBase(an, "postgres", info, p.maxIdleConns, p.maxOpenConns)
}
// UpgradeSchema calls migrate tool to upgrade schema to the latest based on the SQL scripts.

14
src/common/models/config.go
View File

@ -45,12 +45,14 @@ type SQLite struct {
// PostGreSQL ...
type PostGreSQL struct {
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password,omitempty"`
Database string `json:"database"`
SSLMode string `json:"sslmode"`
Host string `json:"host"`
Port int `json:"port"`
Username string `json:"username"`
Password string `json:"password,omitempty"`
Database string `json:"database"`
SSLMode string `json:"sslmode"`
MaxIdleConns int `json:"max_idle_conns"`
MaxOpenConns int `json:"max_open_conns"`
}
// Email ...

14
src/core/config/config.go
View File

@ -331,12 +331,14 @@ func Database() (*models.Database, error) {
database := &models.Database{}
database.Type = cfgMgr.Get(common.DatabaseType).GetString()
postgresql := &models.PostGreSQL{
Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(),
Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(),
Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(),
Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(),
Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(),
SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(),
Host: cfgMgr.Get(common.PostGreSQLHOST).GetString(),
Port: cfgMgr.Get(common.PostGreSQLPort).GetInt(),
Username: cfgMgr.Get(common.PostGreSQLUsername).GetString(),
Password: cfgMgr.Get(common.PostGreSQLPassword).GetString(),
Database: cfgMgr.Get(common.PostGreSQLDatabase).GetString(),
SSLMode: cfgMgr.Get(common.PostGreSQLSSLMode).GetString(),
MaxIdleConns: cfgMgr.Get(common.PostGreSQLMaxIdleConns).GetInt(),
MaxOpenConns: cfgMgr.Get(common.PostGreSQLMaxOpenConns).GetInt(),
}
database.PostGreSQL = postgresql