Merge pull request #3831 from yixingjia/HA_Clair

Make Clair DB configurable

make/common/templates/adminserver/env

@@ -39,7 +39,11 @@ GODEBUG=netdns=cgo
 ADMIRAL_URL=$admiral_url
 WITH_NOTARY=$with_notary
 WITH_CLAIR=$with_clair
-CLAIR_DB_PASSWORD=$pg_password
+CLAIR_DB_PASSWORD=$clair_db_password
+CLAIR_DB_HOST=$clair_db_host
+CLAIR_DB_PORT=$clair_db_port
+CLAIR_DB_USERNAME=$clair_db_username
+CLAIR_DB=$clair_db
 RESET=false
 UAA_ENDPOINT=$uaa_endpoint
 UAA_CLIENTID=$uaa_clientid

make/common/templates/clair/config.yaml

@@ -2,7 +2,7 @@ clair:
   database:
     type: pgsql
     options:
-      source: postgresql://postgres:$password@postgres:5432?sslmode=disable
+      source: postgresql://$username:$password@$host:$port?sslmode=disable
 
       # Number of elements kept in the cache
       # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.

make/harbor.cfg

@@ -107,7 +107,7 @@ token_expiration = 30
 project_creation_restriction = everyone
 
 #The follow configurations are for Harbor HA mode only
-
+#####################################################
 #the address of the mysql database.
 db_host = mysql
 
@@ -118,6 +118,21 @@ db_port = 3306
 db_user = root
 #The redis server address
 redis_url =
+
+#Clair DB host address
+clair_db_host = postgres
+
+#Clair DB connect port
+clair_db_port = 5432
+
+#Clair DB username
+clair_db_username = postgres
+
+#Clair default database
+clair_db = postgres
+
+
+################### end of HA section #####################
 #************************END INITIAL PROPERTIES************************
 #The following attributes only need to be set when auth mode is uaa_auth
 uaa_endpoint = uaa.mydomain.org

make/prepare

@@ -234,7 +234,12 @@ if rcp.has_option("configuration", "admiral_url"):
     admiral_url = rcp.get("configuration", "admiral_url")
 else:
     admiral_url = ""
-pg_password = rcp.get("configuration", "clair_db_password")
+clair_db_password = rcp.get("configuration", "clair_db_password")
+clair_db_host = rcp.get("configuration", "clair_db_host")
+clair_db_port = rcp.get("configuration", "clair_db_port")
+clair_db_username = rcp.get("configuration", "clair_db_username")
+clair_db = rcp.get("configuration", "clair_db")
+
 uaa_endpoint = rcp.get("configuration", "uaa_endpoint")
 uaa_clientid = rcp.get("configuration", "uaa_clientid")
 uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret")
@@ -326,7 +331,11 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         admiral_url=admiral_url,
         with_notary=args.notary_mode,
         with_clair=args.clair_mode,
-        pg_password=pg_password,
+        clair_db_password=clair_db_password,
+        clair_db_host=clair_db_host,
+        clair_db_port=clair_db_port,
+        clair_db_username=clair_db_username,
+        clair_db=clair_db,
         uaa_endpoint=uaa_endpoint,
         uaa_clientid=uaa_clientid,
         uaa_clientsecret=uaa_clientsecret,
@@ -495,9 +504,14 @@ if args.clair_mode:
         shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d"))
     shutil.copytree(os.path.join(clair_temp_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d"))
     postgres_env = os.path.join(clair_config_dir, "postgres_env") 
-    render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = pg_password)
+    render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = clair_db_password)
     clair_conf = os.path.join(clair_config_dir, "config.yaml")
-    render(os.path.join(clair_temp_dir, "config.yaml"), clair_conf, password = pg_password)
+    render(os.path.join(clair_temp_dir, "config.yaml"),
+            clair_conf,
+            password = clair_db_password,
+            username = clair_db_username,
+            host = clair_db_host,
+            port = clair_db_port)
 
 if args.ha_mode:
     prepare_ha(rcp, args)

src/adminserver/systemcfg/store/database/driver_db.go

@@ -35,6 +35,7 @@ var(
 		common.MySQLPort:true,
 		common.MaxJobWorkers:true,
 		common.CfgExpiration:true,
+		common.ClairDBPort:true,
 		}
 	boolKeys = map[string]bool{
 		common.WithClair:true,

src/adminserver/systemcfg/systemcfg.go

@@ -130,6 +130,10 @@ var (
 			parse: parseStringToBool,
 		},
 		common.ClairDBPassword: "CLAIR_DB_PASSWORD",
+		common.ClairDB: "CLAIR_DB",
+		common.ClairDBUsername: "CLAIR_DB_USERNAME",
+		common.ClairDBHost: "CLAIR_DB_HOST",
+		common.ClairDBPort: "CLAIR_DB_PORT",
 		common.UAAEndpoint:     "UAA_ENDPOINT",
 		common.UAAClientID:     "UAA_CLIENTID",
 		common.UAAClientSecret: "UAA_CLIENTSECRET",
@@ -267,7 +271,7 @@ func initCfgStore() (err error) {
 				}
 				err = CfgStore.Write(jsonconfig)
 				if err != nil {
-					log.Error("Failed to update old configuration to dattabase")
+					log.Error("Failed to update old configuration to database")
 					return err
 				}
 			}

src/common/const.go

@@ -70,6 +70,10 @@ const (
 	WithClair                  = "with_clair"
 	ScanAllPolicy              = "scan_all_policy"
 	ClairDBPassword            = "clair_db_password"
+	ClairDBHost                = "clair_db_host"
+	ClairDBPort                = "clair_db_port"
+	ClairDB                    = "clair_db"
+	ClairDBUsername            = "clair_db_username"
 	UAAEndpoint                = "uaa_endpoint"
 	UAAClientID                = "uaa_client_id"
 	UAAClientSecret            = "uaa_client_secret"

src/common/dao/base.go

@@ -43,20 +43,20 @@ type Database interface {
 }
 
 // InitClairDB ...
-func InitClairDB(password string) error {
+func InitClairDB(clairDB *models.PostGreSQL) error {
 	//Except for password other information will not be configurable, so keep it hard coded for 1.2.0.
 	p := &pgsql{
-		host:     "postgres",
-		port:     5432,
-		usr:      "postgres",
-		pwd:      password,
-		database: "postgres",
+		host:     clairDB.Host,
+		port:     clairDB.Port,
+		usr:      clairDB.Username,
+		pwd:      clairDB.Password,
+		database: clairDB.Database,
 		sslmode:  false,
 	}
 	if err := p.Register(ClairDBAlias); err != nil {
 		return err
 	}
-	log.Info("initialized clair databas")
+	log.Info("initialized clair database")
 	return nil
 }
 

src/common/models/config.go

@@ -57,6 +57,15 @@ type SQLite struct {
 	File string `json:"file"`
 }
 
+// PostGreSQL ...
+type PostGreSQL struct {
+	Host string `json:"host"`
+	Port int `json:"port"`
+	Username string `json:"username"`
+	Password string `json:"password,omitempty"`
+	Database string `json:"database"`
+}
+
 // Email ...
 type Email struct {
 	Host     string `json:"host"`

src/common/utils/test/adminserver.go

@@ -60,6 +60,11 @@ var adminServerDefaultConfig = map[string]interface{}{
 	common.AdmiralEndpoint:            "http://www.vmware.com",
 	common.WithNotary:                 false,
 	common.WithClair:                  false,
+	common.ClairDBUsername:            "postgres",
+	common.ClairDBHost:                "postgres",
+	common.ClairDB:                    "postgres",
+	common.ClairDBPort:                5432,
+	common.ClairDBPassword:            "password",
 	common.UAAClientID:                "testid",
 	common.UAAClientSecret:            "testsecret",
 	common.UAAEndpoint:                "10.192.168.5",

src/ui/config/config.go

@@ -379,15 +379,21 @@ func ClairEndpoint() string {
 	return common.DefaultClairEndpoint
 }
 
-// ClairDBPassword returns the password for accessing Clair's DB.
-func ClairDBPassword() (string, error) {
+// ClairDB return Clair db info
+func ClairDB() (*models.PostGreSQL, error){
 	cfg, err := mg.Get()
 	if err != nil {
-		return "", err
+		log.Errorf("Failed to get configuration of Clair DB, Error detail %v", err)
+		return nil, err
 	}
-	return cfg[common.ClairDBPassword].(string), nil
+	clairDB := &models.PostGreSQL{}
+	clairDB.Host = cfg[common.ClairDBHost].(string)
+	clairDB.Port = int(cfg[common.ClairDBPort].(float64))
+	clairDB.Username = cfg[common.ClairDBUsername].(string)
+	clairDB.Password = cfg[common.ClairDBPassword].(string)
+	clairDB.Database = cfg[common.ClairDB].(string)
+	return clairDB, nil
 }
-
 // AdmiralEndpoint returns the URL of admiral, if Harbor is not deployed with admiral it should return an empty string.
 func AdmiralEndpoint() string {
 	cfg, err := mg.Get()

src/ui/config/config_test.go

@@ -117,6 +117,10 @@ func TestConfig(t *testing.T) {
 	if _, err := Database(); err != nil {
 		t.Fatalf("failed to get database: %v", err)
 	}
+
+	if _, err := ClairDB(); err != nil {
+		t.Fatalf("failed to get clair DB %v", err)
+	}
 	if InternalNotaryEndpoint() != "http://notary-server:4443" {
 		t.Errorf("Unexpected notary endpoint: %s", InternalNotaryEndpoint())
 	}

src/ui/main.go

@@ -93,11 +93,11 @@ func main() {
 		log.Fatalf("failed to initialize database: %v", err)
 	}
 	if config.WithClair() {
-		clairDBPassword, err := config.ClairDBPassword()
+		clairDB, err := config.ClairDB()
 		if err != nil {
 			log.Fatalf("failed to load clair database information: %v", err)
 		}
-		if err := dao.InitClairDB(clairDBPassword); err != nil {
+		if err := dao.InitClairDB(clairDB); err != nil {
 			log.Fatalf("failed to initialize clair database: %v", err)
 		}
 	}