From e2c7cfc0adee842be8f161255ac3b59c381cb556 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 23 Feb 2017 14:24:49 +0800 Subject: [PATCH] support changing all configurations through API --- .travis.yml | 1 + src/adminserver/systemcfg/systemcfg.go | 209 ++++++++++++++----------- src/ui/api/config.go | 164 ++++++++++--------- 3 files changed, 208 insertions(+), 166 deletions(-) diff --git a/.travis.yml b/.travis.yml index c7b40e1bc..170581ecf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,7 @@ script: - docker-compose -f make/docker-compose.test.yml down - sudo make/prepare + - sudo rm -rf /data/config/* - docker-compose -f make/dev/docker-compose.yml up -d - docker ps diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index efef094e7..24d13ba84 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -19,6 +19,7 @@ import ( "fmt" "os" "strconv" + "strings" "github.com/vmware/harbor/src/adminserver/systemcfg/store" "github.com/vmware/harbor/src/adminserver/systemcfg/store/json" @@ -34,6 +35,9 @@ const ( ) var ( + cfgStore store.Driver + keyProvider comcfg.KeyProvider + // attrs need to be encrypted or decrypted attrs = []string{ comcfg.EmailPassword, @@ -41,11 +45,98 @@ var ( comcfg.MySQLPassword, comcfg.AdminInitialPassword, } - cfgStore store.Driver - keyProvider comcfg.KeyProvider + + // envs are configurations need read from environment variables + envs = map[string]interface{}{ + comcfg.ExtEndpoint: "EXT_ENDPOINT", + comcfg.AUTHMode: "AUTH_MODE", + comcfg.SelfRegistration: &parser{ + env: "SELF_REGISTRATION", + parse: parseStringToBool, + }, + comcfg.DatabaseType: "DATABASE_TYPE", + comcfg.MySQLHost: "MYSQL_HOST", + comcfg.MySQLPort: &parser{ + env: "MYSQL_PORT", + parse: parseStringToInt, + }, + comcfg.MySQLUsername: "MYSQL_USR", + comcfg.MySQLPassword: "MYSQL_PWD", + comcfg.MySQLDatabase: "MYSQL_DATABASE", + comcfg.SQLiteFile: "SQLITE_FILE", + comcfg.LDAPURL: "LDAP_URL", + comcfg.LDAPSearchDN: "LDAP_SEARCH_DN", + comcfg.LDAPSearchPwd: "LDAP_SEARCH_PWD", + comcfg.LDAPBaseDN: "LDAP_BASE_DN", + comcfg.LDAPFilter: "LDAP_FILTER", + comcfg.LDAPUID: "LDAP_UID", + comcfg.LDAPScope: &parser{ + env: "LDAP_SCOPE", + parse: parseStringToInt, + }, + comcfg.LDAPTimeout: &parser{ + env: "LDAP_TIMEOUT", + parse: parseStringToInt, + }, + comcfg.EmailHost: "EMAIL_HOST", + comcfg.EmailPort: &parser{ + env: "EMAIL_PORT", + parse: parseStringToInt, + }, + comcfg.EmailUsername: "EMAIL_USR", + comcfg.EmailPassword: "EMAIL_PWD", + comcfg.EmailSSL: &parser{ + env: "EMAIL_SSL", + parse: parseStringToBool, + }, + comcfg.EmailFrom: "EMAIL_FROM", + comcfg.EmailIdentity: "EMAIL_IDENTITY", + comcfg.RegistryURL: "REGISTRY_URL", + comcfg.TokenExpiration: &parser{ + env: "TOKEN_EXPIRATION", + parse: parseStringToInt, + }, + comcfg.JobLogDir: "LOG_DIR", + comcfg.UseCompressedJS: &parser{ + env: "USE_COMPRESSED_JS", + parse: parseStringToBool, + }, + comcfg.CfgExpiration: &parser{ + env: "CFG_EXPIRATION", + parse: parseStringToInt, + }, + comcfg.MaxJobWorkers: &parser{ + env: "MAX_JOB_WORKERS", + parse: parseStringToInt, + }, + comcfg.VerifyRemoteCert: &parser{ + env: "VERIFY_REMOTE_CERT", + parse: parseStringToBool, + }, + comcfg.ProjectCreationRestriction: "PROJECT_CREATION_RESTRICTION", + comcfg.AdminInitialPassword: "HARBOR_ADMIN_PASSWORD", + } ) -// Init system configurations. Read from config store first, if null read from env +type parser struct { + // the name of env + env string + // parse the value of env, e.g. parse string to int or + // parse string to bool + parse func(string) (interface{}, error) +} + +func parseStringToInt(str string) (interface{}, error) { + return strconv.Atoi(str) +} + +func parseStringToBool(str string) (interface{}, error) { + return strings.ToLower(str) == "true" || + strings.ToLower(str) == "on", nil +} + +// Init system configurations. Read from config store first, +// if null read from env func Init() (err error) { //init configuation store if err = initCfgStore(); err != nil { @@ -60,24 +151,18 @@ func Init() (err error) { return err } - if cfg == nil { - log.Info("configurations read from store driver are null, initializing system from environment variables...") - cfg, err = initFromEnv() - if err != nil { - return err - } - } else { - if err := readFromEnv(cfg); err != nil { - return err - } + if cfg != nil { + return nil } - //sync configurations into cfg store - if err = UpdateSystemCfg(cfg); err != nil { + log.Info("configurations read from store driver are null, initializing system from environment variables...") + cfg, err = loadFromEnv() + if err != nil { return err } - return nil + //sync configurations into cfg store + return UpdateSystemCfg(cfg) } func initCfgStore() (err error) { @@ -113,85 +198,27 @@ func initKeyProvider() { keyProvider = comcfg.NewFileKeyProvider(path) } -//read the following attrs from env every time boots up -func readFromEnv(cfg map[string]interface{}) error { - cfg[comcfg.ExtEndpoint] = os.Getenv("EXT_ENDPOINT") - - cfg[comcfg.DatabaseType] = os.Getenv("DATABASE_TYPE") - cfg[comcfg.MySQLHost] = os.Getenv("MYSQL_HOST") - port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) - if err != nil { - return err - } - cfg[comcfg.MySQLPort] = port - cfg[comcfg.MySQLUsername] = os.Getenv("MYSQL_USR") - cfg[comcfg.MySQLPassword] = os.Getenv("MYSQL_PWD") - cfg[comcfg.MySQLDatabase] = os.Getenv("MYSQL_DATABASE") - cfg[comcfg.SQLiteFile] = os.Getenv("SQLITE_FILE") - cfg[comcfg.TokenServiceURL] = os.Getenv("TOKEN_SERVICE_URL") - tokenExpi, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) - if err != nil { - return err - } - cfg[comcfg.TokenExpiration] = tokenExpi - cfg[comcfg.RegistryURL] = os.Getenv("REGISTRY_URL") - //TODO remove - cfg[comcfg.JobLogDir] = os.Getenv("LOG_DIR") - //TODO remove - cfg[comcfg.UseCompressedJS] = os.Getenv("USE_COMPRESSED_JS") == "on" - cfgExpi, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) - if err != nil { - return err - } - cfg[comcfg.CfgExpiration] = cfgExpi - workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) - if err != nil { - return err - } - cfg[comcfg.MaxJobWorkers] = workers - - return nil -} - -func initFromEnv() (map[string]interface{}, error) { +//load the configurations from env +func loadFromEnv() (map[string]interface{}, error) { cfg := map[string]interface{}{} - if err := readFromEnv(cfg); err != nil { - return nil, err - } + for k, v := range envs { + if str, ok := v.(string); ok { + cfg[k] = os.Getenv(str) + continue + } - cfg[comcfg.AUTHMode] = os.Getenv("AUTH_MODE") - cfg[comcfg.SelfRegistration] = os.Getenv("SELF_REGISTRATION") == "on" - cfg[comcfg.LDAPURL] = os.Getenv("LDAP_URL") - cfg[comcfg.LDAPSearchDN] = os.Getenv("LDAP_SEARCH_DN") - cfg[comcfg.LDAPSearchPwd] = os.Getenv("LDAP_SEARCH_PWD") - cfg[comcfg.LDAPBaseDN] = os.Getenv("LDAP_BASE_DN") - cfg[comcfg.LDAPFilter] = os.Getenv("LDAP_FILTER") - cfg[comcfg.LDAPUID] = os.Getenv("LDAP_UID") - scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE")) - if err != nil { - return nil, err + if parser, ok := v.(*parser); ok { + i, err := parser.parse(os.Getenv(parser.env)) + if err != nil { + return nil, err + } + cfg[k] = i + continue + } + + return nil, fmt.Errorf("%v is not string or parse type", v) } - cfg[comcfg.LDAPScope] = scope - timeout, err := strconv.Atoi(os.Getenv("LDAP_TIMEOUT")) - if err != nil { - return nil, err - } - cfg[comcfg.LDAPTimeout] = timeout - cfg[comcfg.EmailHost] = os.Getenv("EMAIL_HOST") - port, err := strconv.Atoi(os.Getenv("EMAIL_PORT")) - if err != nil { - return nil, err - } - cfg[comcfg.EmailPort] = port - cfg[comcfg.EmailUsername] = os.Getenv("EMAIL_USR") - cfg[comcfg.EmailPassword] = os.Getenv("EMAIL_PWD") - cfg[comcfg.EmailSSL] = os.Getenv("EMAIL_SSL") == "true" - cfg[comcfg.EmailFrom] = os.Getenv("EMAIL_FROM") - cfg[comcfg.EmailIdentity] = os.Getenv("EMAIL_IDENTITY") - cfg[comcfg.VerifyRemoteCert] = os.Getenv("VERIFY_REMOTE_CERT") == "on" - cfg[comcfg.ProjectCreationRestriction] = os.Getenv("PROJECT_CREATION_RESTRICTION") - cfg[comcfg.AdminInitialPassword] = os.Getenv("HARBOR_ADMIN_PASSWORD") return cfg, nil } diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 05c4c5c44..ab833c961 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -19,51 +19,78 @@ import ( "fmt" "net/http" "strconv" - //"strings" "github.com/vmware/harbor/src/common/api" comcfg "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" - //"github.com/vmware/harbor/src/common/models" - //"github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) -// keys of attrs which user can modify -var validKeys = []string{ - comcfg.AUTHMode, - comcfg.EmailFrom, - comcfg.EmailHost, - comcfg.EmailIdentity, - comcfg.EmailPassword, - comcfg.EmailPort, - comcfg.EmailSSL, - comcfg.EmailUsername, - comcfg.LDAPBaseDN, - comcfg.LDAPFilter, - comcfg.LDAPScope, - comcfg.LDAPSearchDN, - comcfg.LDAPSearchPwd, - comcfg.LDAPTimeout, - comcfg.LDAPUID, - comcfg.LDAPURL, - comcfg.ProjectCreationRestriction, - comcfg.SelfRegistration, - comcfg.VerifyRemoteCert, -} +var ( + // valid keys of configurations which user can modify + validKeys = []string{ + comcfg.ExtEndpoint, + comcfg.AUTHMode, + comcfg.DatabaseType, + comcfg.MySQLHost, + comcfg.MySQLPort, + comcfg.MySQLUsername, + comcfg.MySQLPassword, + comcfg.MySQLDatabase, + comcfg.SQLiteFile, + comcfg.SelfRegistration, + comcfg.LDAPURL, + comcfg.LDAPSearchDN, + comcfg.LDAPSearchPwd, + comcfg.LDAPBaseDN, + comcfg.LDAPUID, + comcfg.LDAPFilter, + comcfg.LDAPScope, + comcfg.LDAPTimeout, + comcfg.TokenServiceURL, + comcfg.RegistryURL, + comcfg.EmailHost, + comcfg.EmailPort, + comcfg.EmailUsername, + comcfg.EmailPassword, + comcfg.EmailFrom, + comcfg.EmailSSL, + comcfg.EmailIdentity, + comcfg.ProjectCreationRestriction, + comcfg.VerifyRemoteCert, + comcfg.MaxJobWorkers, + comcfg.TokenExpiration, + comcfg.CfgExpiration, + comcfg.JobLogDir, + comcfg.UseCompressedJS, + comcfg.AdminInitialPassword, + } -var numKeys = []string{ - comcfg.EmailPort, - comcfg.LDAPScope, - comcfg.LDAPTimeout, -} + numKeys = []string{ + comcfg.EmailPort, + comcfg.LDAPScope, + comcfg.LDAPTimeout, + comcfg.MySQLPort, + comcfg.MaxJobWorkers, + comcfg.TokenExpiration, + comcfg.CfgExpiration, + } -var boolKeys = []string{ - comcfg.EmailSSL, - comcfg.SelfRegistration, - comcfg.VerifyRemoteCert, -} + boolKeys = []string{ + comcfg.EmailSSL, + comcfg.SelfRegistration, + comcfg.VerifyRemoteCert, + comcfg.UseCompressedJS, + } + + passwordKeys = []string{ + comcfg.AdminInitialPassword, + comcfg.EmailPassword, + comcfg.LDAPSearchPwd, + comcfg.MySQLPassword, + } +) // ConfigAPI ... type ConfigAPI struct { @@ -234,26 +261,34 @@ func validateCfg(c map[string]string) (bool, error) { comcfg.LDAPScopeOnelevel, comcfg.LDAPScopeSubtree) } - if timeout, ok := c[comcfg.LDAPTimeout]; ok { - if t, err := strconv.Atoi(timeout); err != nil || t < 0 { - return isSysErr, fmt.Errorf("invalid %s", comcfg.LDAPTimeout) + + for _, k := range boolKeys { + v, ok := c[k] + if !ok { + continue + } + + if v != "0" && v != "1" { + return isSysErr, fmt.Errorf("%s should be %s or %s", + k, "0", "1") } } - if self, ok := c[comcfg.SelfRegistration]; ok && - self != "0" && self != "1" { - return isSysErr, fmt.Errorf("%s should be %s or %s", - comcfg.SelfRegistration, "0", "1") - } - - if port, ok := c[comcfg.EmailPort]; ok { - if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { - return isSysErr, fmt.Errorf("invalid %s", comcfg.EmailPort) + for _, k := range numKeys { + v, ok := c[k] + if !ok { + continue } - } - if ssl, ok := c[comcfg.EmailSSL]; ok && ssl != "0" && ssl != "1" { - return isSysErr, fmt.Errorf("%s should be %s or %s", comcfg.EmailSSL, "0", "1") + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return isSysErr, fmt.Errorf("invalid %s: %s", k, v) + } + + if (k == comcfg.EmailPort || + k == comcfg.MySQLPort) && n > 65535 { + return isSysErr, fmt.Errorf("invalid %s: %s", k, v) + } } if crt, ok := c[comcfg.ProjectCreationRestriction]; ok && @@ -265,29 +300,13 @@ func validateCfg(c map[string]string) (bool, error) { comcfg.ProCrtRestrEveryone) } - if verify, ok := c[comcfg.VerifyRemoteCert]; ok && verify != "0" && verify != "1" { - return isSysErr, fmt.Errorf("invalid %s, should be %s or %s", - comcfg.VerifyRemoteCert, "0", "1") - } - return isSysErr, nil } -//encode passwords and convert map[string]string to map[string]interface{} +//convert map[string]string to map[string]interface{} func convertForPut(m map[string]string) (map[string]interface{}, error) { cfg := map[string]interface{}{} - /* - pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} - for _, pwdKey := range pwdKeys { - if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 { - c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey()) - if err != nil { - return nil, err - } - } - } - */ for k, v := range m { cfg[k] = v } @@ -319,14 +338,9 @@ func convertForPut(m map[string]string) (map[string]interface{}, error) { func convertForGet(cfg map[string]interface{}) (map[string]*value, error) { result := map[string]*value{} - dels := []string{ - comcfg.AdminInitialPassword, - comcfg.EmailPassword, - comcfg.LDAPSearchPwd, - comcfg.MySQLPassword} - for _, del := range dels { - if _, ok := cfg[del]; ok { - delete(cfg, del) + for _, k := range passwordKeys { + if _, ok := cfg[k]; ok { + delete(cfg, k) } }