From f63588855f8d3b1b138d3be63ca165bb52ab930c Mon Sep 17 00:00:00 2001 From: yixingj Date: Tue, 19 Dec 2017 15:25:12 +0800 Subject: [PATCH] Make Clair DB configurable Make the HOST,PORT,USERNAME,DB configurable for Clair --- make/common/templates/adminserver/env | 6 ++++- make/common/templates/clair/config.yaml | 2 +- make/harbor.cfg | 17 +++++++++++++- make/prepare | 22 +++++++++++++++---- .../systemcfg/store/database/driver_db.go | 1 + src/adminserver/systemcfg/systemcfg.go | 6 ++++- src/common/const.go | 4 ++++ src/common/dao/base.go | 14 ++++++------ src/common/models/config.go | 9 ++++++++ src/common/utils/test/adminserver.go | 5 +++++ src/ui/config/config.go | 16 +++++++++----- src/ui/config/config_test.go | 4 ++++ src/ui/main.go | 4 ++-- 13 files changed, 88 insertions(+), 22 deletions(-) diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index 58394cf88..20eceb4fa 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -39,7 +39,11 @@ GODEBUG=netdns=cgo ADMIRAL_URL=$admiral_url WITH_NOTARY=$with_notary WITH_CLAIR=$with_clair -CLAIR_DB_PASSWORD=$pg_password +CLAIR_DB_PASSWORD=$clair_db_password +CLAIR_DB_HOST=$clair_db_host +CLAIR_DB_PORT=$clair_db_port +CLAIR_DB_USERNAME=$clair_db_username +CLAIR_DB=$clair_db RESET=false UAA_ENDPOINT=$uaa_endpoint UAA_CLIENTID=$uaa_clientid diff --git a/make/common/templates/clair/config.yaml b/make/common/templates/clair/config.yaml index c09dd2585..c5219afd0 100644 --- a/make/common/templates/clair/config.yaml +++ b/make/common/templates/clair/config.yaml @@ -2,7 +2,7 @@ clair: database: type: pgsql options: - source: postgresql://postgres:$password@postgres:5432?sslmode=disable + source: postgresql://$username:$password@$host:$port?sslmode=disable # Number of elements kept in the cache # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database. diff --git a/make/harbor.cfg b/make/harbor.cfg index a39efe774..6e22b8fe6 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -107,7 +107,7 @@ token_expiration = 30 project_creation_restriction = everyone #The follow configurations are for Harbor HA mode only - +##################################################### #the address of the mysql database. db_host = mysql @@ -118,6 +118,21 @@ db_port = 3306 db_user = root #The redis server address redis_url = + +#Clair DB host address +clair_db_host = postgres + +#Clair DB connect port +clair_db_port = 5432 + +#Clair DB username +clair_db_username = postgres + +#Clair default database +clair_db = postgres + + +################### end of HA section ##################### #************************END INITIAL PROPERTIES************************ #The following attributes only need to be set when auth mode is uaa_auth uaa_endpoint = uaa.mydomain.org diff --git a/make/prepare b/make/prepare index 1b8b5fa43..1546f3113 100755 --- a/make/prepare +++ b/make/prepare @@ -234,7 +234,12 @@ if rcp.has_option("configuration", "admiral_url"): admiral_url = rcp.get("configuration", "admiral_url") else: admiral_url = "" -pg_password = rcp.get("configuration", "clair_db_password") +clair_db_password = rcp.get("configuration", "clair_db_password") +clair_db_host = rcp.get("configuration", "clair_db_host") +clair_db_port = rcp.get("configuration", "clair_db_port") +clair_db_username = rcp.get("configuration", "clair_db_username") +clair_db = rcp.get("configuration", "clair_db") + uaa_endpoint = rcp.get("configuration", "uaa_endpoint") uaa_clientid = rcp.get("configuration", "uaa_clientid") uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret") @@ -326,7 +331,11 @@ render(os.path.join(templates_dir, "adminserver", "env"), admiral_url=admiral_url, with_notary=args.notary_mode, with_clair=args.clair_mode, - pg_password=pg_password, + clair_db_password=clair_db_password, + clair_db_host=clair_db_host, + clair_db_port=clair_db_port, + clair_db_username=clair_db_username, + clair_db=clair_db, uaa_endpoint=uaa_endpoint, uaa_clientid=uaa_clientid, uaa_clientsecret=uaa_clientsecret, @@ -495,9 +504,14 @@ if args.clair_mode: shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d")) shutil.copytree(os.path.join(clair_temp_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d")) postgres_env = os.path.join(clair_config_dir, "postgres_env") - render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = pg_password) + render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = clair_db_password) clair_conf = os.path.join(clair_config_dir, "config.yaml") - render(os.path.join(clair_temp_dir, "config.yaml"), clair_conf, password = pg_password) + render(os.path.join(clair_temp_dir, "config.yaml"), + clair_conf, + password = clair_db_password, + username = clair_db_username, + host = clair_db_host, + port = clair_db_port) if args.ha_mode: prepare_ha(rcp, args) diff --git a/src/adminserver/systemcfg/store/database/driver_db.go b/src/adminserver/systemcfg/store/database/driver_db.go index db97ab90b..3a6377e7d 100644 --- a/src/adminserver/systemcfg/store/database/driver_db.go +++ b/src/adminserver/systemcfg/store/database/driver_db.go @@ -35,6 +35,7 @@ var( common.MySQLPort:true, common.MaxJobWorkers:true, common.CfgExpiration:true, + common.ClairDBPort:true, } boolKeys = map[string]bool{ common.WithClair:true, diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 5ba116718..7bfe4d47f 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -130,6 +130,10 @@ var ( parse: parseStringToBool, }, common.ClairDBPassword: "CLAIR_DB_PASSWORD", + common.ClairDB: "CLAIR_DB", + common.ClairDBUsername: "CLAIR_DB_USERNAME", + common.ClairDBHost: "CLAIR_DB_HOST", + common.ClairDBPort: "CLAIR_DB_PORT", common.UAAEndpoint: "UAA_ENDPOINT", common.UAAClientID: "UAA_CLIENTID", common.UAAClientSecret: "UAA_CLIENTSECRET", @@ -267,7 +271,7 @@ func initCfgStore() (err error) { } err = CfgStore.Write(jsonconfig) if err != nil { - log.Error("Failed to update old configuration to dattabase") + log.Error("Failed to update old configuration to database") return err } } diff --git a/src/common/const.go b/src/common/const.go index de9702103..ca5f317c4 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -70,6 +70,10 @@ const ( WithClair = "with_clair" ScanAllPolicy = "scan_all_policy" ClairDBPassword = "clair_db_password" + ClairDBHost = "clair_db_host" + ClairDBPort = "clair_db_port" + ClairDB = "clair_db" + ClairDBUsername = "clair_db_username" UAAEndpoint = "uaa_endpoint" UAAClientID = "uaa_client_id" UAAClientSecret = "uaa_client_secret" diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 1da44e0f2..395342cbc 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -43,20 +43,20 @@ type Database interface { } // InitClairDB ... -func InitClairDB(password string) error { +func InitClairDB(clairDB *models.PostGreSQL) error { //Except for password other information will not be configurable, so keep it hard coded for 1.2.0. p := &pgsql{ - host: "postgres", - port: 5432, - usr: "postgres", - pwd: password, - database: "postgres", + host: clairDB.Host, + port: clairDB.Port, + usr: clairDB.Username, + pwd: clairDB.Password, + database: clairDB.Database, sslmode: false, } if err := p.Register(ClairDBAlias); err != nil { return err } - log.Info("initialized clair databas") + log.Info("initialized clair database") return nil } diff --git a/src/common/models/config.go b/src/common/models/config.go index 52de3e6e8..42caf3be8 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -57,6 +57,15 @@ type SQLite struct { File string `json:"file"` } +// PostGreSQL ... +type PostGreSQL struct { + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password,omitempty"` + Database string `json:"database"` +} + // Email ... type Email struct { Host string `json:"host"` diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 9c09436df..8c4397275 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -60,6 +60,11 @@ var adminServerDefaultConfig = map[string]interface{}{ common.AdmiralEndpoint: "http://www.vmware.com", common.WithNotary: false, common.WithClair: false, + common.ClairDBUsername: "postgres", + common.ClairDBHost: "postgres", + common.ClairDB: "postgres", + common.ClairDBPort: 5432, + common.ClairDBPassword: "password", common.UAAClientID: "testid", common.UAAClientSecret: "testsecret", common.UAAEndpoint: "10.192.168.5", diff --git a/src/ui/config/config.go b/src/ui/config/config.go index a510b2ca4..6809eb568 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -379,15 +379,21 @@ func ClairEndpoint() string { return common.DefaultClairEndpoint } -// ClairDBPassword returns the password for accessing Clair's DB. -func ClairDBPassword() (string, error) { +// ClairDB return Clair db info +func ClairDB() (*models.PostGreSQL, error){ cfg, err := mg.Get() if err != nil { - return "", err + log.Errorf("Failed to get configuration of Clair DB, Error detail %v", err) + return nil, err } - return cfg[common.ClairDBPassword].(string), nil + clairDB := &models.PostGreSQL{} + clairDB.Host = cfg[common.ClairDBHost].(string) + clairDB.Port = int(cfg[common.ClairDBPort].(float64)) + clairDB.Username = cfg[common.ClairDBUsername].(string) + clairDB.Password = cfg[common.ClairDBPassword].(string) + clairDB.Database = cfg[common.ClairDB].(string) + return clairDB, nil } - // AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string. func AdmiralEndpoint() string { cfg, err := mg.Get() diff --git a/src/ui/config/config_test.go b/src/ui/config/config_test.go index fc9426fdc..6055ed5ec 100644 --- a/src/ui/config/config_test.go +++ b/src/ui/config/config_test.go @@ -117,6 +117,10 @@ func TestConfig(t *testing.T) { if _, err := Database(); err != nil { t.Fatalf("failed to get database: %v", err) } + + if _, err := ClairDB(); err != nil { + t.Fatalf("failed to get clair DB %v", err) + } if InternalNotaryEndpoint() != "http://notary-server:4443" { t.Errorf("Unexpected notary endpoint: %s", InternalNotaryEndpoint()) } diff --git a/src/ui/main.go b/src/ui/main.go index 42d967315..ede844b02 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -93,11 +93,11 @@ func main() { log.Fatalf("failed to initialize database: %v", err) } if config.WithClair() { - clairDBPassword, err := config.ClairDBPassword() + clairDB, err := config.ClairDB() if err != nil { log.Fatalf("failed to load clair database information: %v", err) } - if err := dao.InitClairDB(clairDBPassword); err != nil { + if err := dao.InitClairDB(clairDB); err != nil { log.Fatalf("failed to initialize clair database: %v", err) } }