Merge pull request #1434 from ywk253100/170222_config_all

Support changing all configurations through API

.travis.yml

@@ -89,6 +89,7 @@ script:
 
   - docker-compose -f make/docker-compose.test.yml down
   - sudo make/prepare
+  - sudo rm -rf /data/config/*
   - docker-compose -f make/dev/docker-compose.yml up -d
   
   - docker ps

src/adminserver/systemcfg/systemcfg.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/vmware/harbor/src/adminserver/systemcfg/store"
 	"github.com/vmware/harbor/src/adminserver/systemcfg/store/json"
@@ -34,6 +35,9 @@ const (
 )
 
 var (
+	cfgStore    store.Driver
+	keyProvider comcfg.KeyProvider
+
 	// attrs need to be encrypted or decrypted
 	attrs = []string{
 		comcfg.EmailPassword,
@@ -41,11 +45,98 @@ var (
 		comcfg.MySQLPassword,
 		comcfg.AdminInitialPassword,
 	}
-	cfgStore    store.Driver
+
-	keyProvider comcfg.KeyProvider
+	// envs are configurations need read from environment variables
+	envs = map[string]interface{}{
+		comcfg.ExtEndpoint: "EXT_ENDPOINT",
+		comcfg.AUTHMode:    "AUTH_MODE",
+		comcfg.SelfRegistration: &parser{
+			env:   "SELF_REGISTRATION",
+			parse: parseStringToBool,
+		},
+		comcfg.DatabaseType: "DATABASE_TYPE",
+		comcfg.MySQLHost:    "MYSQL_HOST",
+		comcfg.MySQLPort: &parser{
+			env:   "MYSQL_PORT",
+			parse: parseStringToInt,
+		},
+		comcfg.MySQLUsername: "MYSQL_USR",
+		comcfg.MySQLPassword: "MYSQL_PWD",
+		comcfg.MySQLDatabase: "MYSQL_DATABASE",
+		comcfg.SQLiteFile:    "SQLITE_FILE",
+		comcfg.LDAPURL:       "LDAP_URL",
+		comcfg.LDAPSearchDN:  "LDAP_SEARCH_DN",
+		comcfg.LDAPSearchPwd: "LDAP_SEARCH_PWD",
+		comcfg.LDAPBaseDN:    "LDAP_BASE_DN",
+		comcfg.LDAPFilter:    "LDAP_FILTER",
+		comcfg.LDAPUID:       "LDAP_UID",
+		comcfg.LDAPScope: &parser{
+			env:   "LDAP_SCOPE",
+			parse: parseStringToInt,
+		},
+		comcfg.LDAPTimeout: &parser{
+			env:   "LDAP_TIMEOUT",
+			parse: parseStringToInt,
+		},
+		comcfg.EmailHost: "EMAIL_HOST",
+		comcfg.EmailPort: &parser{
+			env:   "EMAIL_PORT",
+			parse: parseStringToInt,
+		},
+		comcfg.EmailUsername: "EMAIL_USR",
+		comcfg.EmailPassword: "EMAIL_PWD",
+		comcfg.EmailSSL: &parser{
+			env:   "EMAIL_SSL",
+			parse: parseStringToBool,
+		},
+		comcfg.EmailFrom:     "EMAIL_FROM",
+		comcfg.EmailIdentity: "EMAIL_IDENTITY",
+		comcfg.RegistryURL:   "REGISTRY_URL",
+		comcfg.TokenExpiration: &parser{
+			env:   "TOKEN_EXPIRATION",
+			parse: parseStringToInt,
+		},
+		comcfg.JobLogDir: "LOG_DIR",
+		comcfg.UseCompressedJS: &parser{
+			env:   "USE_COMPRESSED_JS",
+			parse: parseStringToBool,
+		},
+		comcfg.CfgExpiration: &parser{
+			env:   "CFG_EXPIRATION",
+			parse: parseStringToInt,
+		},
+		comcfg.MaxJobWorkers: &parser{
+			env:   "MAX_JOB_WORKERS",
+			parse: parseStringToInt,
+		},
+		comcfg.VerifyRemoteCert: &parser{
+			env:   "VERIFY_REMOTE_CERT",
+			parse: parseStringToBool,
+		},
+		comcfg.ProjectCreationRestriction: "PROJECT_CREATION_RESTRICTION",
+		comcfg.AdminInitialPassword:       "HARBOR_ADMIN_PASSWORD",
+	}
 )
 
-// Init system configurations. Read from config store first, if null read from env
+type parser struct {
+	// the name of env
+	env string
+	// parse the value of env, e.g. parse string to int or
+	// parse string to bool
+	parse func(string) (interface{}, error)
+}
+
+func parseStringToInt(str string) (interface{}, error) {
+	return strconv.Atoi(str)
+}
+
+func parseStringToBool(str string) (interface{}, error) {
+	return strings.ToLower(str) == "true" ||
+		strings.ToLower(str) == "on", nil
+}
+
+// Init system configurations. Read from config store first,
+// if null read from env
 func Init() (err error) {
 	//init configuation store
 	if err = initCfgStore(); err != nil {
@@ -60,24 +151,18 @@ func Init() (err error) {
 		return err
 	}
 
-	if cfg == nil {
+	if cfg != nil {
-		log.Info("configurations read from store driver are null, initializing system from environment variables...")
+		return nil
-		cfg, err = initFromEnv()
-		if err != nil {
-			return err
-		}
-	} else {
-		if err := readFromEnv(cfg); err != nil {
-			return err
-		}
 	}
 
-	//sync configurations into cfg store
+	log.Info("configurations read from store driver are null, initializing system from environment variables...")
-	if err = UpdateSystemCfg(cfg); err != nil {
+	cfg, err = loadFromEnv()
+	if err != nil {
 		return err
 	}
 
-	return nil
+	//sync configurations into cfg store
+	return UpdateSystemCfg(cfg)
 }
 
 func initCfgStore() (err error) {
@@ -113,85 +198,27 @@ func initKeyProvider() {
 	keyProvider = comcfg.NewFileKeyProvider(path)
 }
 
-//read the following attrs from env every time boots up
+//load the configurations from env
-func readFromEnv(cfg map[string]interface{}) error {
+func loadFromEnv() (map[string]interface{}, error) {
-	cfg[comcfg.ExtEndpoint] = os.Getenv("EXT_ENDPOINT")
-
-	cfg[comcfg.DatabaseType] = os.Getenv("DATABASE_TYPE")
-	cfg[comcfg.MySQLHost] = os.Getenv("MYSQL_HOST")
-	port, err := strconv.Atoi(os.Getenv("MYSQL_PORT"))
-	if err != nil {
-		return err
-	}
-	cfg[comcfg.MySQLPort] = port
-	cfg[comcfg.MySQLUsername] = os.Getenv("MYSQL_USR")
-	cfg[comcfg.MySQLPassword] = os.Getenv("MYSQL_PWD")
-	cfg[comcfg.MySQLDatabase] = os.Getenv("MYSQL_DATABASE")
-	cfg[comcfg.SQLiteFile] = os.Getenv("SQLITE_FILE")
-	cfg[comcfg.TokenServiceURL] = os.Getenv("TOKEN_SERVICE_URL")
-	tokenExpi, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION"))
-	if err != nil {
-		return err
-	}
-	cfg[comcfg.TokenExpiration] = tokenExpi
-	cfg[comcfg.RegistryURL] = os.Getenv("REGISTRY_URL")
-	//TODO remove
-	cfg[comcfg.JobLogDir] = os.Getenv("LOG_DIR")
-	//TODO remove
-	cfg[comcfg.UseCompressedJS] = os.Getenv("USE_COMPRESSED_JS") == "on"
-	cfgExpi, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION"))
-	if err != nil {
-		return err
-	}
-	cfg[comcfg.CfgExpiration] = cfgExpi
-	workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS"))
-	if err != nil {
-		return err
-	}
-	cfg[comcfg.MaxJobWorkers] = workers
-
-	return nil
-}
-
-func initFromEnv() (map[string]interface{}, error) {
 	cfg := map[string]interface{}{}
 
-	if err := readFromEnv(cfg); err != nil {
+	for k, v := range envs {
-		return nil, err
+		if str, ok := v.(string); ok {
-	}
+			cfg[k] = os.Getenv(str)
+			continue
+		}
 
-	cfg[comcfg.AUTHMode] = os.Getenv("AUTH_MODE")
+		if parser, ok := v.(*parser); ok {
-	cfg[comcfg.SelfRegistration] = os.Getenv("SELF_REGISTRATION") == "on"
+			i, err := parser.parse(os.Getenv(parser.env))
-	cfg[comcfg.LDAPURL] = os.Getenv("LDAP_URL")
+			if err != nil {
-	cfg[comcfg.LDAPSearchDN] = os.Getenv("LDAP_SEARCH_DN")
+				return nil, err
-	cfg[comcfg.LDAPSearchPwd] = os.Getenv("LDAP_SEARCH_PWD")
+			}
-	cfg[comcfg.LDAPBaseDN] = os.Getenv("LDAP_BASE_DN")
+			cfg[k] = i
-	cfg[comcfg.LDAPFilter] = os.Getenv("LDAP_FILTER")
+			continue
-	cfg[comcfg.LDAPUID] = os.Getenv("LDAP_UID")
+		}
-	scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE"))
+
-	if err != nil {
+		return nil, fmt.Errorf("%v is not string or parse type", v)
-		return nil, err
 	}
-	cfg[comcfg.LDAPScope] = scope
-	timeout, err := strconv.Atoi(os.Getenv("LDAP_TIMEOUT"))
-	if err != nil {
-		return nil, err
-	}
-	cfg[comcfg.LDAPTimeout] = timeout
-	cfg[comcfg.EmailHost] = os.Getenv("EMAIL_HOST")
-	port, err := strconv.Atoi(os.Getenv("EMAIL_PORT"))
-	if err != nil {
-		return nil, err
-	}
-	cfg[comcfg.EmailPort] = port
-	cfg[comcfg.EmailUsername] = os.Getenv("EMAIL_USR")
-	cfg[comcfg.EmailPassword] = os.Getenv("EMAIL_PWD")
-	cfg[comcfg.EmailSSL] = os.Getenv("EMAIL_SSL") == "true"
-	cfg[comcfg.EmailFrom] = os.Getenv("EMAIL_FROM")
-	cfg[comcfg.EmailIdentity] = os.Getenv("EMAIL_IDENTITY")
-	cfg[comcfg.VerifyRemoteCert] = os.Getenv("VERIFY_REMOTE_CERT") == "on"
-	cfg[comcfg.ProjectCreationRestriction] = os.Getenv("PROJECT_CREATION_RESTRICTION")
-	cfg[comcfg.AdminInitialPassword] = os.Getenv("HARBOR_ADMIN_PASSWORD")
 
 	return cfg, nil
 }

src/ui/api/config.go

@@ -19,51 +19,78 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
-	//"strings"
 
 	"github.com/vmware/harbor/src/common/api"
 	comcfg "github.com/vmware/harbor/src/common/config"
 	"github.com/vmware/harbor/src/common/dao"
-	//"github.com/vmware/harbor/src/common/models"
-	//"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
 )
 
-// keys of attrs which user can modify
+var (
-var validKeys = []string{
+	// valid keys of configurations which user can modify
-	comcfg.AUTHMode,
+	validKeys = []string{
-	comcfg.EmailFrom,
+		comcfg.ExtEndpoint,
-	comcfg.EmailHost,
+		comcfg.AUTHMode,
-	comcfg.EmailIdentity,
+		comcfg.DatabaseType,
-	comcfg.EmailPassword,
+		comcfg.MySQLHost,
-	comcfg.EmailPort,
+		comcfg.MySQLPort,
-	comcfg.EmailSSL,
+		comcfg.MySQLUsername,
-	comcfg.EmailUsername,
+		comcfg.MySQLPassword,
-	comcfg.LDAPBaseDN,
+		comcfg.MySQLDatabase,
-	comcfg.LDAPFilter,
+		comcfg.SQLiteFile,
-	comcfg.LDAPScope,
+		comcfg.SelfRegistration,
-	comcfg.LDAPSearchDN,
+		comcfg.LDAPURL,
-	comcfg.LDAPSearchPwd,
+		comcfg.LDAPSearchDN,
-	comcfg.LDAPTimeout,
+		comcfg.LDAPSearchPwd,
-	comcfg.LDAPUID,
+		comcfg.LDAPBaseDN,
-	comcfg.LDAPURL,
+		comcfg.LDAPUID,
-	comcfg.ProjectCreationRestriction,
+		comcfg.LDAPFilter,
-	comcfg.SelfRegistration,
+		comcfg.LDAPScope,
-	comcfg.VerifyRemoteCert,
+		comcfg.LDAPTimeout,
-}
+		comcfg.TokenServiceURL,
+		comcfg.RegistryURL,
+		comcfg.EmailHost,
+		comcfg.EmailPort,
+		comcfg.EmailUsername,
+		comcfg.EmailPassword,
+		comcfg.EmailFrom,
+		comcfg.EmailSSL,
+		comcfg.EmailIdentity,
+		comcfg.ProjectCreationRestriction,
+		comcfg.VerifyRemoteCert,
+		comcfg.MaxJobWorkers,
+		comcfg.TokenExpiration,
+		comcfg.CfgExpiration,
+		comcfg.JobLogDir,
+		comcfg.UseCompressedJS,
+		comcfg.AdminInitialPassword,
+	}
 
-var numKeys = []string{
+	numKeys = []string{
-	comcfg.EmailPort,
+		comcfg.EmailPort,
-	comcfg.LDAPScope,
+		comcfg.LDAPScope,
-	comcfg.LDAPTimeout,
+		comcfg.LDAPTimeout,
-}
+		comcfg.MySQLPort,
+		comcfg.MaxJobWorkers,
+		comcfg.TokenExpiration,
+		comcfg.CfgExpiration,
+	}
 
-var boolKeys = []string{
+	boolKeys = []string{
-	comcfg.EmailSSL,
+		comcfg.EmailSSL,
-	comcfg.SelfRegistration,
+		comcfg.SelfRegistration,
-	comcfg.VerifyRemoteCert,
+		comcfg.VerifyRemoteCert,
-}
+		comcfg.UseCompressedJS,
+	}
+
+	passwordKeys = []string{
+		comcfg.AdminInitialPassword,
+		comcfg.EmailPassword,
+		comcfg.LDAPSearchPwd,
+		comcfg.MySQLPassword,
+	}
+)
 
 // ConfigAPI ...
 type ConfigAPI struct {
@@ -234,26 +261,34 @@ func validateCfg(c map[string]string) (bool, error) {
 			comcfg.LDAPScopeOnelevel,
 			comcfg.LDAPScopeSubtree)
 	}
-	if timeout, ok := c[comcfg.LDAPTimeout]; ok {
+
-		if t, err := strconv.Atoi(timeout); err != nil || t < 0 {
+	for _, k := range boolKeys {
-			return isSysErr, fmt.Errorf("invalid %s", comcfg.LDAPTimeout)
+		v, ok := c[k]
+		if !ok {
+			continue
+		}
+
+		if v != "0" && v != "1" {
+			return isSysErr, fmt.Errorf("%s should be %s or %s",
+				k, "0", "1")
 		}
 	}
 
-	if self, ok := c[comcfg.SelfRegistration]; ok &&
+	for _, k := range numKeys {
-		self != "0" && self != "1" {
+		v, ok := c[k]
-		return isSysErr, fmt.Errorf("%s should be %s or %s",
+		if !ok {
-			comcfg.SelfRegistration, "0", "1")
+			continue
-	}
-
-	if port, ok := c[comcfg.EmailPort]; ok {
-		if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 {
-			return isSysErr, fmt.Errorf("invalid %s", comcfg.EmailPort)
 		}
-	}
 
-	if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" {
+		n, err := strconv.Atoi(v)
-		return isSysErr, fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1")
+		if err != nil || n < 0 {
+			return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
+		}
+
+		if (k == comcfg.EmailPort ||
+			k == comcfg.MySQLPort) && n > 65535 {
+			return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
+		}
 	}
 
 	if crt, ok := c[comcfg.ProjectCreationRestriction]; ok &&
@@ -265,29 +300,13 @@ func validateCfg(c map[string]string) (bool, error) {
 			comcfg.ProCrtRestrEveryone)
 	}
 
-	if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" {
-		return isSysErr, fmt.Errorf("invalid %s, should be %s or %s",
-			comcfg.VerifyRemoteCert, "0", "1")
-	}
-
 	return isSysErr, nil
 }
 
-//encode passwords and convert map[string]string to map[string]interface{}
+//convert map[string]string to map[string]interface{}
 func convertForPut(m map[string]string) (map[string]interface{}, error) {
 	cfg := map[string]interface{}{}
 
-	/*
-		pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD}
-		for _, pwdKey := range pwdKeys {
-			if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 {
-				c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey())
-				if err != nil {
-					return nil, err
-				}
-			}
-		}
-	*/
 	for k, v := range m {
 		cfg[k] = v
 	}
@@ -319,14 +338,9 @@ func convertForPut(m map[string]string) (map[string]interface{}, error) {
 func convertForGet(cfg map[string]interface{}) (map[string]*value, error) {
 	result := map[string]*value{}
 
-	dels := []string{
+	for _, k := range passwordKeys {
-		comcfg.AdminInitialPassword,
+		if _, ok := cfg[k]; ok {
-		comcfg.EmailPassword,
+			delete(cfg, k)
-		comcfg.LDAPSearchPwd,
-		comcfg.MySQLPassword}
-	for _, del := range dels {
-		if _, ok := cfg[del]; ok {
-			delete(cfg, del)
 		}
 	}