From b62a95825047f24b7dd8d92d3b0892dd82efa243 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 30 Dec 2016 18:04:01 +0800 Subject: [PATCH] configure harbor --- .gitignore | 3 + make/common/templates/adminserver/env | 37 +++ make/common/templates/jobservice/env | 16 +- make/common/templates/ui/app.conf | 11 +- make/common/templates/ui/env | 29 +- make/dev/adminserver/Dockerfile | 12 + make/dev/docker-compose.yml | 19 ++ make/docker-compose.tpl | 18 ++ make/photon/adminserver/Dockerfile | 8 + src/adminserver/api/base.go | 29 ++ src/adminserver/api/cfg.go | 161 +++++++++++ src/adminserver/main.go | 60 ++++ src/adminserver/router.go | 30 ++ src/adminserver/systemcfg/driver.go | 30 ++ src/adminserver/systemcfg/driver_json.go | 104 +++++++ src/adminserver/systemcfg/driver_json_test.go | 50 ++++ src/adminserver/systemcfg/systemcfg.go | 176 ++++++++++++ src/common/api/base.go | 10 +- src/common/config/config.go | 227 ++++++--------- src/common/dao/base.go | 32 ++- src/common/dao/config.go | 32 +++ src/common/dao/config_test.go | 236 ++++++++++++++++ src/common/dao/dao_test.go | 84 +++--- src/common/dao/mysql.go | 33 +-- src/common/dao/user_test.go | 13 + src/common/models/config.go | 95 +++++++ src/common/utils/{ => email}/mail.go | 29 +- src/common/utils/log/logger.go | 4 +- .../utils/registry/auth/tokenauthorizer.go | 17 +- src/common/utils/utils.go | 38 +++ src/jobservice/api/replication.go | 14 +- src/jobservice/config/config.go | 182 +++++++----- src/jobservice/job/statemachine.go | 32 ++- src/jobservice/job/workerpool.go | 17 +- src/jobservice/main.go | 29 +- src/jobservice/utils/logger.go | 24 +- src/ui/api/config.go | 245 ++++++++++++++++ src/ui/api/project.go | 8 +- src/ui/api/repository.go | 14 +- src/ui/api/target.go | 14 +- src/ui/api/user.go | 18 +- src/ui/api/utils.go | 50 ++-- src/ui/auth/authenticator.go | 5 +- src/ui/auth/ldap/ldap.go | 24 +- src/ui/config/config.go | 266 ++++++++++++------ src/ui/controllers/base.go | 15 +- src/ui/controllers/password.go | 18 +- src/ui/controllers/project.go | 9 +- src/ui/controllers/repository.go | 11 +- src/ui/main.go | 23 +- src/ui/router.go | 1 + src/ui/service/token/authutils.go | 19 +- 52 files changed, 2119 insertions(+), 562 deletions(-) create mode 100644 make/common/templates/adminserver/env create mode 100644 make/dev/adminserver/Dockerfile create mode 100644 make/photon/adminserver/Dockerfile create mode 100644 src/adminserver/api/base.go create mode 100644 src/adminserver/api/cfg.go create mode 100644 src/adminserver/main.go create mode 100644 src/adminserver/router.go create mode 100644 src/adminserver/systemcfg/driver.go create mode 100644 src/adminserver/systemcfg/driver_json.go create mode 100644 src/adminserver/systemcfg/driver_json_test.go create mode 100644 src/adminserver/systemcfg/systemcfg.go create mode 100644 src/common/dao/config.go create mode 100644 src/common/dao/config_test.go create mode 100644 src/common/models/config.go rename src/common/utils/{ => email}/mail.go (89%) create mode 100644 src/ui/api/config.go diff --git a/.gitignore b/.gitignore index 7e451d1d5..572a7f08d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,11 @@ harbor make/common/config/* +make/dev/adminserver/harbor_adminserver make/dev/ui/harbor_ui make/dev/jobservice/harbor_jobservice +src/adminserver/adminserver src/ui/ui src/jobservice/jobservice +src/common/dao/dao.test *.pyc jobservice/test diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env new file mode 100644 index 000000000..7c88ba28a --- /dev/null +++ b/make/common/templates/adminserver/env @@ -0,0 +1,37 @@ +LOG_LEVEL=debug +EXT_ENDPOINT=$ui_url +AUTH_MODE=$auth_mode +SELF_REGISTRATION=$self_registration +LDAP_URL=$ldap_url +LDAP_SEARCH_DN=$ldap_searchdn +LDAP_SEARCH_PWD=$ldap_search_pwd +LDAP_BASE_DN=$ldap_basedn +LDAP_FILTER=$ldap_filter +LDAP_UID=$ldap_uid +LDAP_SCOPE=$ldap_scope +DATABASE_TYPE=mysql +MYSQL_HOST=mysql +MYSQL_PORT=3306 +MYSQL_USR=root +MYSQL_PWD=$db_password +MYSQL_DATABASE=registry +REGISTRY_URL=http://registry:5000 +TOKEN_SERVICE_URL=http://ui/service/token +EMAIL_HOST=$email_host +EMAIL_PORT=$email_port +EMAIL_USR=$email_usr +EMAIL_PWD=$email_pwd +EMAIL_TLS=$email_tls +EMAIL_FROM=$email_from +EMAIL_IDENTITY=$email_identity +HARBOR_ADMIN_PASSWORD=$harbor_admin_password +PROJECT_CREATION_RESTRICTION=$project_creation_restriction +VERIFY_REMOTE_CERT=$verify_remote_cert +MAX_JOB_WORKERS=$max_job_workers +LOG_DIR=/var/log/jobs +UI_SECRET=$ui_secret +SECRET_KEY=$secret_key +TOKEN_EXPIRATION=$token_expiration +CFG_EXPIRATION=$cfg_expiration +USE_COMPRESSED_JS=$use_compressed_js +GODEBUG=netdns=cgo \ No newline at end of file diff --git a/make/common/templates/jobservice/env b/make/common/templates/jobservice/env index c6e9fd736..d75972ac3 100644 --- a/make/common/templates/jobservice/env +++ b/make/common/templates/jobservice/env @@ -1,15 +1,5 @@ -MYSQL_HOST=mysql -MYSQL_PORT=3306 -MYSQL_USR=root -MYSQL_PWD=$db_password -UI_SECRET=$ui_secret -SECRET_KEY=$secret_key -CONFIG_PATH=/etc/jobservice/app.conf -REGISTRY_URL=http://registry:5000 -VERIFY_REMOTE_CERT=$verify_remote_cert -MAX_JOB_WORKERS=$max_job_workers LOG_LEVEL=debug -LOG_DIR=/var/log/jobs +UI_SECRET=$ui_secret +CONFIG_PATH=/etc/jobservice/app.conf +MAX_JOB_WORKERS=$max_job_workers GODEBUG=netdns=cgo -EXT_ENDPOINT=$ui_url -TOKEN_ENDPOINT=http://ui diff --git a/make/common/templates/ui/app.conf b/make/common/templates/ui/app.conf index 3cda6d877..b4090f33c 100644 --- a/make/common/templates/ui/app.conf +++ b/make/common/templates/ui/app.conf @@ -6,13 +6,4 @@ types = en-US|zh-CN names = en-US|zh-CN [dev] -httpport = 80 - -[mail] -identity = $email_identity -host = $email_server -port = $email_server_port -username = $email_username -password = $email_password -from = $email_from -ssl = $email_ssl +httpport = 80 \ No newline at end of file diff --git a/make/common/templates/ui/env b/make/common/templates/ui/env index e4feedc2e..fc0d133ab 100644 --- a/make/common/templates/ui/env +++ b/make/common/templates/ui/env @@ -1,29 +1,4 @@ -MYSQL_HOST=mysql -MYSQL_PORT=3306 -MYSQL_USR=root -MYSQL_PWD=$db_password -REGISTRY_URL=http://registry:5000 -JOB_SERVICE_URL=http://jobservice -UI_URL=http://ui -CONFIG_PATH=/etc/ui/app.conf -EXT_REG_URL=$hostname -HARBOR_ADMIN_PASSWORD=$harbor_admin_password -AUTH_MODE=$auth_mode -LDAP_URL=$ldap_url -LDAP_SEARCH_DN=$ldap_searchdn -LDAP_SEARCH_PWD=$ldap_search_pwd -LDAP_BASE_DN=$ldap_basedn -LDAP_FILTER=$ldap_filter -LDAP_UID=$ldap_uid -LDAP_SCOPE=$ldap_scope -UI_SECRET=$ui_secret -SECRET_KEY=$secret_key -SELF_REGISTRATION=$self_registration -USE_COMPRESSED_JS=$use_compressed_js LOG_LEVEL=debug +CONFIG_PATH=/etc/ui/app.conf +UI_SECRET=$ui_secret GODEBUG=netdns=cgo -EXT_ENDPOINT=$ui_url -TOKEN_ENDPOINT=http://ui -VERIFY_REMOTE_CERT=$verify_remote_cert -TOKEN_EXPIRATION=$token_expiration -PROJECT_CREATION_RESTRICTION=$project_creation_restriction diff --git a/make/dev/adminserver/Dockerfile b/make/dev/adminserver/Dockerfile new file mode 100644 index 000000000..d05f87da2 --- /dev/null +++ b/make/dev/adminserver/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.7.3 + +MAINTAINER yinw@vmware.com + +COPY . /go/src/github.com/vmware/harbor + +WORKDIR /go/src/github.com/vmware/harbor/src/adminserver + +RUN go build -v -a -o /go/bin/harbor_adminserver \ + && chmod u+x /go/bin/harbor_adminserver +WORKDIR /go/bin/ +ENTRYPOINT ["/go/bin/harbor_adminserver"] diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index 33c7f2be3..c94a2d875 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -40,6 +40,22 @@ services: options: syslog-address: "tcp://127.0.0.1:1514" tag: "mysql" + adminserver: + build: + context: ../../ + dockerfile: make/dev/adminserver/Dockerfile + env_file: + - ../common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "adminserver" ui: build: context: ../../ @@ -52,6 +68,8 @@ services: - ../common/config/ui/private_key.pem:/etc/ui/private_key.pem depends_on: - log + - adminserver + - registry logging: driver: "syslog" options: @@ -69,6 +87,7 @@ services: - ../common/config/jobservice/app.conf:/etc/jobservice/app.conf depends_on: - ui + - adminserver logging: driver: "syslog" options: diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index ac6c33d71..b66eb2698 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -41,6 +41,21 @@ services: options: syslog-address: "tcp://127.0.0.1:1514" tag: "mysql" + adminserver: + image: vmware/harbor-adminserver + container_name: harbor-adminserver + env_file: + - ./common/config/adminserver/env + restart: always + volumes: + - /data/config/:/etc/harbor/ + depends_on: + - log + logging: + driver: "syslog" + options: + syslog-address: "tcp://127.0.0.1:1514" + tag: "adminserver" ui: image: vmware/harbor-ui container_name: harbor-ui @@ -53,6 +68,8 @@ services: - /data:/harbor_storage depends_on: - log + - adminserver + - registry logging: driver: "syslog" options: @@ -69,6 +86,7 @@ services: - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf depends_on: - ui + - adminserver logging: driver: "syslog" options: diff --git a/make/photon/adminserver/Dockerfile b/make/photon/adminserver/Dockerfile new file mode 100644 index 000000000..cb145275b --- /dev/null +++ b/make/photon/adminserver/Dockerfile @@ -0,0 +1,8 @@ +FROM library/photon:1.0 + +RUN mkdir /harbor/ +COPY ./make/dev/adminserver/harbor_adminserver /harbor/ + +RUN chmod u+x /harbor/harbor_adminserver +WORKDIR /harbor/ +ENTRYPOINT ["/harbor/harbor_adminserver"] diff --git a/src/adminserver/api/base.go b/src/adminserver/api/base.go new file mode 100644 index 000000000..47938c8b7 --- /dev/null +++ b/src/adminserver/api/base.go @@ -0,0 +1,29 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "net/http" +) + +func handleInternalServerError(w http.ResponseWriter) { + http.Error(w, http.StatusText(http.StatusInternalServerError), + http.StatusInternalServerError) +} + +func handleBadRequestError(w http.ResponseWriter, error string) { + http.Error(w, error, http.StatusBadRequest) +} diff --git a/src/adminserver/api/cfg.go b/src/adminserver/api/cfg.go new file mode 100644 index 000000000..09499fec6 --- /dev/null +++ b/src/adminserver/api/cfg.go @@ -0,0 +1,161 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "io/ioutil" + "net/http" + "strconv" + + cfg "github.com/vmware/harbor/src/adminserver/systemcfg" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +// ListCfgs lists configurations +func ListCfgs(w http.ResponseWriter, r *http.Request) { + cfg, err := cfg.GetSystemCfg() + if err != nil { + log.Errorf("failed to get system configurations: %v", err) + handleInternalServerError(w) + return + } + + b, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + log.Errorf("failed to marshal configurations: %v", err) + handleInternalServerError(w) + return + } + if _, err = w.Write(b); err != nil { + log.Errorf("failed to write response: %v", err) + } +} + +// UpdateCfgs updates configurations +func UpdateCfgs(w http.ResponseWriter, r *http.Request) { + b, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Errorf("failed to read request body: %v", err) + handleInternalServerError(w) + return + } + + m := &map[string]string{} + if err = json.Unmarshal(b, m); err != nil { + handleBadRequestError(w, err.Error()) + return + } + + log.Info(m) + + system, err := cfg.GetSystemCfg() + if err != nil { + handleInternalServerError(w) + return + } + + if err := populate(system, *m); err != nil { + log.Errorf("failed to populate system configurations: %v", err) + handleInternalServerError(w) + return + } + + log.Info(system.Authentication.SelfRegistration) + + if err = cfg.UpdateSystemCfg(system); err != nil { + log.Errorf("failed to update system configurations: %v", err) + handleInternalServerError(w) + return + } +} + +// populate attrs of cfg according to m +func populate(cfg *models.SystemCfg, m map[string]string) error { + if mode, ok := m[comcfg.AUTH_MODE]; ok { + cfg.Authentication.Mode = mode + } + if value, ok := m[comcfg.SELF_REGISTRATION]; ok { + cfg.Authentication.SelfRegistration = value == "true" + } + if url, ok := m[comcfg.LDAP_URL]; ok { + cfg.Authentication.LDAP.URL = url + } + if dn, ok := m[comcfg.LDAP_SEARCH_DN]; ok { + cfg.Authentication.LDAP.SearchDN = dn + } + if pwd, ok := m[comcfg.LDAP_SEARCH_PWD]; ok { + cfg.Authentication.LDAP.SearchPwd = pwd + } + if dn, ok := m[comcfg.LDAP_BASE_DN]; ok { + cfg.Authentication.LDAP.BaseDN = dn + } + if uid, ok := m[comcfg.LDAP_UID]; ok { + cfg.Authentication.LDAP.UID = uid + } + if filter, ok := m[comcfg.LDAP_FILTER]; ok { + cfg.Authentication.LDAP.Filter = filter + } + if scope, ok := m[comcfg.LDAP_SCOPE]; ok { + i, err := strconv.Atoi(scope) + if err != nil { + return err + } + cfg.Authentication.LDAP.Scope = i + } + + if value, ok := m[comcfg.EMAIL_SERVER]; ok { + cfg.Email.Host = value + } + if value, ok := m[comcfg.EMAIL_SERVER_PORT]; ok { + cfg.Email.Port = value + } + if value, ok := m[comcfg.EMAIL_USERNAME]; ok { + cfg.Email.Username = value + } + if value, ok := m[comcfg.EMAIL_PWD]; ok { + cfg.Email.Host = value + } + if value, ok := m[comcfg.EMAIL_SSL]; ok { + cfg.Email.Password = value + } + if value, ok := m[comcfg.EMAIL_FROM]; ok { + cfg.Email.From = value + } + if value, ok := m[comcfg.EMAIL_IDENTITY]; ok { + cfg.Email.Identity = value + } + + if value, ok := m[comcfg.PROJECT_CREATION_RESTRICTION]; ok { + cfg.ProjectCreationRestriction = value + } + + if value, ok := m[comcfg.VERIFY_REMOTE_CERT]; ok { + cfg.VerifyRemoteCert = value == "true" + } + + if value, ok := m[comcfg.MAX_JOB_WORKERS]; ok { + if i, err := strconv.Atoi(value); err != nil { + return err + } else { + cfg.MaxJobWorkers = i + } + } + + return nil +} diff --git a/src/adminserver/main.go b/src/adminserver/main.go new file mode 100644 index 000000000..d83d95d1a --- /dev/null +++ b/src/adminserver/main.go @@ -0,0 +1,60 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "net/http" + "os" + + cfg "github.com/vmware/harbor/src/adminserver/systemcfg" + "github.com/vmware/harbor/src/common/utils/log" +) + +// Server for admin component +type Server struct { + Port string + Handler http.Handler +} + +// Serve the API +func (s *Server) Serve() error { + server := &http.Server{ + Addr: ":" + s.Port, + Handler: s.Handler, + } + + return server.ListenAndServe() +} + +func main() { + log.Info("initializing system configurations...") + if err := cfg.Init(); err != nil { + log.Fatalf("failed to initialize the system: %v", err) + } + log.Info("system initialization completed") + + port := os.Getenv("PORT") + if len(port) == 0 { + port = "80" + } + server := &Server{ + Port: port, + Handler: newHandler(), + } + if err := server.Serve(); err != nil { + log.Fatal(err) + } +} diff --git a/src/adminserver/router.go b/src/adminserver/router.go new file mode 100644 index 000000000..49083f253 --- /dev/null +++ b/src/adminserver/router.go @@ -0,0 +1,30 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "net/http" + + "github.com/gorilla/mux" + "github.com/vmware/harbor/src/adminserver/api" +) + +func newHandler() http.Handler { + r := mux.NewRouter() + r.HandleFunc("/api/configurations", api.ListCfgs).Methods("GET") + r.HandleFunc("/api/configurations", api.UpdateCfgs).Methods("PUT") + return r +} diff --git a/src/adminserver/systemcfg/driver.go b/src/adminserver/systemcfg/driver.go new file mode 100644 index 000000000..bfb110f24 --- /dev/null +++ b/src/adminserver/systemcfg/driver.go @@ -0,0 +1,30 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// Driver defines methods that a configuration store driver must implement +type Driver interface { + // Name returns a human-readable name of the driver + Name() string + // Read reads the configurations from store + Read() (*models.SystemCfg, error) + // Write writes the configurations to store + Write(*models.SystemCfg) error +} diff --git a/src/adminserver/systemcfg/driver_json.go b/src/adminserver/systemcfg/driver_json.go new file mode 100644 index 000000000..62e2c2ee7 --- /dev/null +++ b/src/adminserver/systemcfg/driver_json.go @@ -0,0 +1,104 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "sync" + + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +const ( + // the default path of configuration file + defaultPath = "/etc/harbor/config.json" +) + +type cfgStore struct { + path string // the path of cfg file + sync.RWMutex +} + +// NewCfgStore returns an instance of cfgStore that stores the configurations +// in a json file. The file will be created if it does not exist. +func NewCfgStore(path ...string) (Driver, error) { + p := defaultPath + if len(path) != 0 { + p = path[0] + } + if _, err := os.Stat(p); os.IsNotExist(err) { + log.Infof("the configuration file %s does not exist, creating it...", p) + if err = os.MkdirAll(filepath.Dir(p), 0600); err != nil { + return nil, err + } + if err = ioutil.WriteFile(p, []byte{}, 0600); err != nil { + return nil, err + } + } + + return &cfgStore{ + path: p, + }, nil +} + +// Name ... +func (c *cfgStore) Name() string { + return "JSON" +} + +// Read ... +func (c *cfgStore) Read() (*models.SystemCfg, error) { + c.RLock() + defer c.RUnlock() + + b, err := ioutil.ReadFile(c.path) + if err != nil { + return nil, err + } + + // empty file + if len(b) == 0 { + return nil, nil + } + + config := &models.SystemCfg{} + if err = json.Unmarshal(b, config); err != nil { + return nil, err + } + + return config, nil +} + +// Write ... +func (c *cfgStore) Write(config *models.SystemCfg) error { + b, err := json.MarshalIndent(config, "", " ") + if err != nil { + return err + } + + c.Lock() + defer c.Unlock() + + if err = ioutil.WriteFile(c.path, b, 0600); err != nil { + return err + } + + return nil +} diff --git a/src/adminserver/systemcfg/driver_json_test.go b/src/adminserver/systemcfg/driver_json_test.go new file mode 100644 index 000000000..83a672cec --- /dev/null +++ b/src/adminserver/systemcfg/driver_json_test.go @@ -0,0 +1,50 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "os" + "testing" +) + +func TestReadWrite(t *testing.T) { + path := "/tmp/config.json" + store, err := NewCfgStore(path) + if err != nil { + t.Fatalf("failed to create json cfg store: %v", err) + } + defer func() { + if err := os.Remove(path); err != nil { + t.Fatalf("failed to remove the json file %s: %v", path, err) + } + }() + + config := &cfg.SystemCfg{ + Authentication: &cfg.Authentication{ + LDAP: &cfg.LDAP{}, + }, + Database: &cfg.Database{ + MySQL: &cfg.MySQL{}, + }, + } + if err := store.Write(config); err != nil { + t.Fatalf("failed to write configurations to json file: %v", err) + } + + if _, err = store.Read(); err != nil { + t.Fatalf("failed to read configurations from json file: %v", err) + } +} diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go new file mode 100644 index 000000000..dda9cf144 --- /dev/null +++ b/src/adminserver/systemcfg/systemcfg.go @@ -0,0 +1,176 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package systemcfg + +import ( + "fmt" + "os" + "strconv" + + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/common/utils/log" +) + +var store Driver + +// Init system configurations. Read from config store first, if null read from env +func Init() (err error) { + s := getCfgStore() + switch s { + case "json": + store, err = NewCfgStore() + if err != nil { + return + } + default: + return fmt.Errorf("unsupported configuration store driver %s", s) + } + + log.Infof("configuration store driver: %s", store.Name()) + cfg, err := store.Read() + if err != nil { + return err + } + + if cfg == nil { + log.Info("configurations read from store driver are null, initializing system from environment variables...") + cfg, err = initFromEnv() + if err != nil { + return err + } + } else { + //read the following attrs from env every time boots up, + //and sync them into cfg store + cfg.DomainName = os.Getenv("EXT_ENDPOINT") + cfg.Database.MySQL.Password = os.Getenv("MYSQL_PWD") + cfg.JobLogDir = os.Getenv("LOG_DIR") + cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" + exp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + if err != nil { + return err + } + cfg.TokenExpiration = exp + cfg.SecretKey = os.Getenv("SECRET_KEY") + + cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + if err != nil { + return err + } + cfg.CfgExpiration = cfgExp + } + + if err = store.Write(cfg); err != nil { + return err + } + + return nil +} + +func getCfgStore() string { + return "json" +} + +func initFromEnv() (*models.SystemCfg, error) { + cfg := &models.SystemCfg{} + cfg.DomainName = os.Getenv("EXT_ENDPOINT") + cfg.Authentication = &models.Authentication{ + Mode: os.Getenv("AUTH_MODE"), + SelfRegistration: os.Getenv("SELF_REGISTRATION") == "true", + LDAP: &models.LDAP{ + URL: os.Getenv("LDAP_URL"), + SearchDN: os.Getenv("LDAP_SEARCH_DN"), + SearchPwd: os.Getenv("LDAP_SEARCH_PWD"), + BaseDN: os.Getenv("LDAP_BASE_DN"), + Filter: os.Getenv("LDAP_FILTER"), + UID: os.Getenv("LDAP_UID"), + }, + } + scope, err := strconv.Atoi(os.Getenv("LDAP_SCOPE")) + if err != nil { + return nil, err + } + cfg.Authentication.LDAP.Scope = scope + cfg.Database = &models.Database{ + Type: os.Getenv("DATABASE_TYPE"), + MySQL: &models.MySQL{ + Host: os.Getenv("MYSQL_HOST"), + Username: os.Getenv("MYSQL_USR"), + Password: os.Getenv("MYSQL_PWD"), + Database: os.Getenv("MYSQL_DATABASE"), + }, + SQLite: &models.SQLite{ + File: os.Getenv("SQLITE_FILE"), + }, + } + port, err := strconv.Atoi(os.Getenv("MYSQL_PORT")) + if err != nil { + return nil, err + } + cfg.Database.MySQL.Port = port + + cfg.TokenService = &models.TokenService{ + URL: os.Getenv("TOKEN_SERVICE_URL"), + } + cfg.Registry = &models.Registry{ + URL: os.Getenv("REGISTRY_URL"), + } + cfg.Email = &models.Email{ + Host: os.Getenv("EMAIL_HOST"), + Port: os.Getenv("EMAIL_PORT"), + Username: os.Getenv("EMAIL_USR"), + Password: os.Getenv("EMAIL_PWD"), + TLS: os.Getenv("EMAIL_TLS") == "true", + From: os.Getenv("EMAIL_FROM"), + Identity: os.Getenv("EMAIL_IDENTITY"), + } + cfg.VerifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") == "true" + cfg.ProjectCreationRestriction = os.Getenv("PROJECT_CREATION_RESTRICTION") + + workers, err := strconv.Atoi(os.Getenv("MAX_JOB_WORKERS")) + if err != nil { + return nil, err + } + cfg.MaxJobWorkers = workers + cfg.JobLogDir = os.Getenv("LOG_DIR") + cfg.InitialAdminPwd = os.Getenv("HARBOR_ADMIN_PASSWORD") + cfg.CompressJS = os.Getenv("USE_COMPRESSED_JS") == "on" + + tokenExp, err := strconv.Atoi(os.Getenv("TOKEN_EXPIRATION")) + if err != nil { + return nil, err + } + cfg.TokenExpiration = tokenExp + + cfg.SecretKey = os.Getenv("SECRET_KEY") + + cfgExp, err := strconv.Atoi(os.Getenv("CFG_EXPIRATION")) + if err != nil { + return nil, err + } + cfg.CfgExpiration = cfgExp + + return cfg, nil +} + +// GetSystemCfg returns the system configurations +func GetSystemCfg() (*models.SystemCfg, error) { + return store.Read() +} + +// UpdateSystemCfg updates the system configurations +func UpdateSystemCfg(cfg *models.SystemCfg) error { + return store.Write(cfg) +} diff --git a/src/common/api/base.go b/src/common/api/base.go index 53d916849..227327339 100644 --- a/src/common/api/base.go +++ b/src/common/api/base.go @@ -22,11 +22,11 @@ import ( "strconv" "github.com/astaxie/beego/validation" - "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/auth" + "github.com/vmware/harbor/src/ui/config" "github.com/astaxie/beego" ) @@ -212,6 +212,10 @@ func (b *BaseAPI) GetPaginationParams() (page, pageSize int64) { } // GetIsInsecure ... -func GetIsInsecure() bool { - return !config.VerifyRemoteCert() +func GetIsInsecure() (bool, error) { + verify, err := config.VerifyRemoteCert() + if err != nil { + return false, err + } + return !verify, nil } diff --git a/src/common/config/config.go b/src/common/config/config.go index e9c54c27c..756bff8d1 100644 --- a/src/common/config/config.go +++ b/src/common/config/config.go @@ -17,162 +17,119 @@ package config import ( + "bytes" "fmt" - "os" + "io/ioutil" + "net/http" "strings" + + "github.com/astaxie/beego/cache" + "github.com/vmware/harbor/src/common/utils" ) -// ConfLoader is the interface to load configurations -type ConfLoader interface { - // Load will load configuration from different source into a string map, the values in the map will be parsed in to configurations. - Load() (map[string]string, error) +const ( + //auth mode + DB_AUTH = "db_auth" + LDAP_AUTH = "ldap_auth" + + //project_creation_restriction + PRO_CRT_RESTR_EVERYONE = "everyone" + PRO_CRT_RESTR_ADM_ONLY = "adminonly" + + LDAP_SCOPE_BASE = "1" + LDAP_SCOPE_ONELEVEL = "2" + LDAP_SCOPE_SUBTREE = "3" + + AUTH_MODE = "auth_mode" + SELF_REGISTRATION = "self_registration" + LDAP_URL = "ldap_url" + LDAP_SEARCH_DN = "ldap_search_dn" + LDAP_SEARCH_PWD = "ldap_search_pwd" + LDAP_BASE_DN = "ldap_base_dn" + LDAP_UID = "ldap_uid" + LDAP_FILTER = "ldap_filter" + LDAP_SCOPE = "ldap_scope" + EMAIL_SERVER = "email_server" + EMAIL_SERVER_PORT = "email_server_port" + EMAIL_USERNAME = "email_server_username" + EMAIL_PWD = "email_server_pwd" + EMAIL_FROM = "email_from" + EMAIL_SSL = "email_ssl" + EMAIL_IDENTITY = "email_identity" + PROJECT_CREATION_RESTRICTION = "project_creation_restriction" + VERIFY_REMOTE_CERT = "verify_remote_cert" + MAX_JOB_WORKERS = "max_job_workers" + CFG_EXPIRATION = "cfg_expiration" +) + +type Manager struct { + Key string + Cache cache.Cache + Loader *Loader } -// EnvConfigLoader loads the config from env vars. -type EnvConfigLoader struct { - Keys []string -} - -// Load ... -func (ec *EnvConfigLoader) Load() (map[string]string, error) { - m := make(map[string]string) - for _, k := range ec.Keys { - m[k] = os.Getenv(k) +func NewManager(key, url string) *Manager { + return &Manager{ + Key: key, + Cache: cache.NewMemoryCache(), + Loader: NewLoader(url), } - return m, nil } -// ConfParser ... -type ConfParser interface { - - //Parse parse the input raw map into a config map - Parse(raw map[string]string, config map[string]interface{}) error -} - -// Config wraps a map for the processed configuration values, -// and loader parser to read configuration from external source and process the values. -type Config struct { - Config map[string]interface{} - Loader ConfLoader - Parser ConfParser -} - -// Load reload the configurations -func (conf *Config) Load() error { - rawMap, err := conf.Loader.Load() - if err != nil { - return err +func (m *Manager) GetFromCache() interface{} { + value := m.Cache.Get(m.Key) + if value != nil { + return value } - err = conf.Parser.Parse(rawMap, conf.Config) - return err -} - -// MySQLSetting wraps the settings of a MySQL DB -type MySQLSetting struct { - Database string - User string - Password string - Host string - Port string -} - -// SQLiteSetting wraps the settings of a SQLite DB -type SQLiteSetting struct { - FilePath string -} - -type commonParser struct{} - -// Parse parses the db settings, veryfy_remote_cert, ext_endpoint, token_endpoint -func (cp *commonParser) Parse(raw map[string]string, config map[string]interface{}) error { - db := strings.ToLower(raw["DATABASE"]) - if db == "mysql" || db == "" { - db = "mysql" - mySQLDB := raw["MYSQL_DATABASE"] - if len(mySQLDB) == 0 { - mySQLDB = "registry" - } - setting := MySQLSetting{ - mySQLDB, - raw["MYSQL_USR"], - raw["MYSQL_PWD"], - raw["MYSQL_HOST"], - raw["MYSQL_PORT"], - } - config["mysql"] = setting - } else if db == "sqlite" { - f := raw["SQLITE_FILE"] - if len(f) == 0 { - f = "registry.db" - } - setting := SQLiteSetting{ - f, - } - config["sqlite"] = setting - } else { - return fmt.Errorf("Invalid DB: %s", db) - } - config["database"] = db - - //By default it's true - config["verify_remote_cert"] = raw["VERIFY_REMOTE_CERT"] != "off" - - config["ext_endpoint"] = raw["EXT_ENDPOINT"] - config["token_endpoint"] = raw["TOKEN_ENDPOINT"] - config["log_level"] = raw["LOG_LEVEL"] return nil } -var commonConfig *Config +type Loader struct { + url string + client *http.Client +} -func init() { - commonKeys := []string{"DATABASE", "MYSQL_DATABASE", "MYSQL_USR", "MYSQL_PWD", "MYSQL_HOST", "MYSQL_PORT", "SQLITE_FILE", "VERIFY_REMOTE_CERT", "EXT_ENDPOINT", "TOKEN_ENDPOINT", "LOG_LEVEL"} - commonConfig = &Config{ - Config: make(map[string]interface{}), - Loader: &EnvConfigLoader{Keys: commonKeys}, - Parser: &commonParser{}, - } - if err := commonConfig.Load(); err != nil { - panic(err) +func NewLoader(url string) *Loader { + return &Loader{ + url: url, + client: &http.Client{}, } } -// Reload will reload the configuration. -func Reload() error { - return commonConfig.Load() +func (l *Loader) Init() error { + addr := l.url + if strings.Contains(addr, "://") { + addr = strings.Split(addr, "://")[1] + } + return utils.TestTCPConn(addr, 60, 2) } -// Database returns the DB type in configuration. -func Database() string { - return commonConfig.Config["database"].(string) +func (l *Loader) Load() ([]byte, error) { + resp, err := l.client.Get(l.url + "/api/configurations") + if err != nil { + return nil, err + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return b, nil } -// MySQL returns the mysql setting in configuration. -func MySQL() MySQLSetting { - return commonConfig.Config["mysql"].(MySQLSetting) -} +func (l *Loader) Upload(b []byte) error { + req, err := http.NewRequest("PUT", l.url+"/api/configurations", bytes.NewReader(b)) + if err != nil { + return err + } + resp, err := l.client.Do(req) + if err != nil { + return err + } -// SQLite returns the SQLite setting -func SQLite() SQLiteSetting { - return commonConfig.Config["sqlite"].(SQLiteSetting) -} + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected http status code: %d", resp.StatusCode) + } -// VerifyRemoteCert returns bool value. -func VerifyRemoteCert() bool { - return commonConfig.Config["verify_remote_cert"].(bool) -} - -// ExtEndpoint ... -func ExtEndpoint() string { - return commonConfig.Config["ext_endpoint"].(string) -} - -// TokenEndpoint returns the endpoint string of token service, which can be accessed by internal service of Harbor. -func TokenEndpoint() string { - return commonConfig.Config["token_endpoint"].(string) -} - -// LogLevel returns the log level in string format. -func LogLevel() string { - return commonConfig.Config["log_level"].(string) + return nil } diff --git a/src/common/dao/base.go b/src/common/dao/base.go index 9028ce587..e05f77561 100644 --- a/src/common/dao/base.go +++ b/src/common/dao/base.go @@ -17,11 +17,12 @@ package dao import ( "fmt" + "strconv" "strings" "sync" "github.com/astaxie/beego/orm" - "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" ) @@ -39,27 +40,32 @@ type Database interface { } // InitDatabase initializes the database -func InitDatabase() { - database, err := getDatabase() +func InitDatabase(database *models.Database) error { + db, err := getDatabase(database) if err != nil { - panic(err) + return err } - log.Infof("initializing database: %s", database.String()) - if err := database.Register(); err != nil { - panic(err) + log.Infof("initializing database: %s", db.String()) + if err := db.Register(); err != nil { + return err } + log.Info("initialize database completed") + return nil } -func getDatabase() (db Database, err error) { - switch config.Database() { +func getDatabase(database *models.Database) (db Database, err error) { + switch database.Type { case "", "mysql": - db = NewMySQL(config.MySQL().Host, config.MySQL().Port, config.MySQL().User, - config.MySQL().Password, config.MySQL().Database) + db = NewMySQL(database.MySQL.Host, + strconv.Itoa(database.MySQL.Port), + database.MySQL.Username, + database.MySQL.Password, + database.MySQL.Database) case "sqlite": - db = NewSQLite(config.SQLite().FilePath) + db = NewSQLite(database.SQLite.File) default: - err = fmt.Errorf("invalid database: %s", config.Database()) + err = fmt.Errorf("invalid database: %s", database.Type) } return } diff --git a/src/common/dao/config.go b/src/common/dao/config.go new file mode 100644 index 000000000..eb700c8af --- /dev/null +++ b/src/common/dao/config.go @@ -0,0 +1,32 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dao + +import ( + "github.com/vmware/harbor/src/common/models" +) + +// AuthModeCanBeModified determines whether auth mode can be +// modified or not. Auth mode can modified when there is only admin +// user in database. +func AuthModeCanBeModified() (bool, error) { + c, err := GetOrmer().QueryTable(&models.User{}).Count() + if err != nil { + return false, err + } + + return c == 1, nil +} diff --git a/src/common/dao/config_test.go b/src/common/dao/config_test.go new file mode 100644 index 000000000..ce624e313 --- /dev/null +++ b/src/common/dao/config_test.go @@ -0,0 +1,236 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package dao + +/* +import ( + "testing" + + "github.com/vmware/harbor/src/common/models" +) + +func deleteConfigByKey(key string) error { + if _, err := GetOrmer().Raw("delete from properties where k = ?", key). + Exec(); err != nil { + return err + } + return nil +} + +func TestGetConfigByKey(t *testing.T) { + cfg := &models.Config{ + Key: "key", + Value: "value", + } + + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + config, err := GetConfigByKey(cfg.Key) + if err != nil { + t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) + } + + if config == nil { + t.Fatal("configuration is nil") + } + + if config.Value != cfg.Value { + t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) + } +} + +func TestListConfigs(t *testing.T) { + configs, err := ListConfigs() + if err != nil { + t.Fatalf("failed to list configurations: %v", err) + } + size := len(configs) + + cfg := &models.Config{ + Key: "key", + Value: "value", + } + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + configs, err = ListConfigs() + if err != nil { + t.Fatalf("failed to list configurations: %v", err) + } + + if size+1 != len(configs) { + t.Fatalf("unexpected length of configurations: %d != %d", len(configs), size+1) + } +} + +func TestInsertConfig(t *testing.T) { + cfg := &models.Config{ + Key: "key1", + Value: "value1", + } + + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + config, err := GetConfigByKey(cfg.Key) + if err != nil { + t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) + } + if config == nil { + t.Fatal("configuration is nil") + } + + if config.Value != cfg.Value { + t.Fatalf("unexpected value: %s != %s", config.Value, cfg.Value) + } +} + +func TestUpdateConfig(t *testing.T) { + cfg := &models.Config{ + Key: "key", + Value: "value", + } + + if err := InsertConfig(cfg); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg.Key) + + newCfg := &models.Config{ + Key: "key", + Value: "new_value", + } + if err := UpdateConfig(newCfg); err != nil { + t.Fatalf("failed to update configuration: %v", err) + } + + config, err := GetConfigByKey(cfg.Key) + if err != nil { + t.Fatalf("failed to get configuration by key %s: %v", cfg.Key, err) + } + + if config == nil { + t.Fatal("configuration is nil") + } + + if config.Value != newCfg.Value { + t.Fatalf("unexpected value: %s != %s", config.Value, newCfg.Value) + } +} + +func TestInsertOrUpdateConfigs(t *testing.T) { + cfg1 := &models.Config{ + Key: "key1", + Value: "value1", + } + + if err := InsertConfig(cfg1); err != nil { + t.Fatalf("failed to insert configuration into table: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg1.Key) + + cfg2 := &models.Config{ + Key: "key2", + Value: "value2", + } + + if err := InsertOrUpdateConfigs([]*models.Config{cfg1, cfg2}); err != nil { + t.Fatalf("failed to insert or update configurations: %v", err) + } + defer func(key string) { + if err := deleteConfigByKey(key); err != nil { + t.Fatalf("failed to delete configuration %s: %v", key, err) + } + }(cfg2.Key) +} + +func TestAuthModeCanBeModified(t *testing.T) { + c, err := GetOrmer().QueryTable(&models.User{}).Count() + if err != nil { + t.Fatalf("failed to count users: %v", err) + } + + if c == 1 { + flag, err := AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if !flag { + t.Errorf("unexpected result: %t != %t", flag, true) + } + + user := models.User{ + Username: "user_for_config_test", + Email: "user_for_config_test@vmware.com", + Password: "P@ssword", + Realname: "user_for_config_test", + } + id, err := Register(user) + if err != nil { + t.Fatalf("failed to register user: %v", err) + } + defer func(id int64) { + if err := deleteUser(id); err != nil { + t.Fatalf("failed to delete user %d: %v", id, err) + } + }(id) + + flag, err = AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if flag { + t.Errorf("unexpected result: %t != %t", flag, false) + } + + } else { + flag, err := AuthModeCanBeModified() + if err != nil { + t.Fatalf("failed to determine whether auth mode can be modified: %v", err) + } + if flag { + t.Errorf("unexpected result: %t != %t", flag, false) + } + } +} +*/ diff --git a/src/common/dao/dao_test.go b/src/common/dao/dao_test.go index 861bf3ee7..dd3594c79 100644 --- a/src/common/dao/dao_test.go +++ b/src/common/dao/dao_test.go @@ -17,10 +17,12 @@ package dao import ( "os" + "strconv" "testing" "time" "github.com/astaxie/beego/orm" + //"github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" @@ -42,7 +44,7 @@ func execUpdate(o orm.Ormer, sql string, params ...interface{}) error { func clearUp(username string) { var err error - o := orm.NewOrm() + o := GetOrmer() o.Begin() err = execUpdate(o, `delete @@ -156,53 +158,63 @@ func TestMain(m *testing.M) { } func testForMySQL(m *testing.M) int { - db := os.Getenv("DATABASE") - defer os.Setenv("DATABASE", db) - - os.Setenv("DATABASE", "mysql") - - dbHost := os.Getenv("DB_HOST") + dbHost := os.Getenv("MYSQL_HOST") if len(dbHost) == 0 { - log.Fatalf("environment variable DB_HOST is not set") + log.Fatalf("environment variable MYSQL_HOST is not set") } - dbUser := os.Getenv("DB_USR") + dbUser := os.Getenv("MYSQL_USR") if len(dbUser) == 0 { - log.Fatalf("environment variable DB_USR is not set") + log.Fatalf("environment variable MYSQL_USR is not set") } - dbPort := os.Getenv("DB_PORT") - if len(dbPort) == 0 { - log.Fatalf("environment variable DB_PORT is not set") + dbPortStr := os.Getenv("MYSQL_PORT") + if len(dbPortStr) == 0 { + log.Fatalf("environment variable MYSQL_PORT is not set") + } + dbPort, err := strconv.Atoi(dbPortStr) + if err != nil { + log.Fatalf("invalid MYSQL_PORT: %v", err) } - dbPassword := os.Getenv("DB_PWD") - log.Infof("DB_HOST: %s, DB_USR: %s, DB_PORT: %s, DB_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) + dbPassword := os.Getenv("MYSQL_PWD") + dbDatabase := os.Getenv("MYSQL_DATABASE") + if len(dbDatabase) == 0 { + log.Fatalf("environment variable MYSQL_DATABASE is not set") + } - os.Setenv("MYSQL_HOST", dbHost) - os.Setenv("MYSQL_PORT", dbPort) - os.Setenv("MYSQL_USR", dbUser) - os.Setenv("MYSQL_PWD", dbPassword) + database := &models.Database{ + Type: "mysql", + MySQL: &models.MySQL{ + Host: dbHost, + Port: dbPort, + Username: dbUser, + Password: dbPassword, + Database: dbDatabase, + }, + } - return testForAll(m) + log.Infof("MYSQL_HOST: %s, MYSQL_USR: %s, MYSQL_PORT: %s, MYSQL_PWD: %s\n", dbHost, dbUser, dbPort, dbPassword) + + return testForAll(m, database) } func testForSQLite(m *testing.M) int { - db := os.Getenv("DATABASE") - defer os.Setenv("DATABASE", db) - - os.Setenv("DATABASE", "sqlite") - file := os.Getenv("SQLITE_FILE") if len(file) == 0 { - os.Setenv("SQLITE_FILE", "/registry.db") - defer os.Setenv("SQLITE_FILE", "") + log.Fatalf("environment variable SQLITE_FILE is not set") } - return testForAll(m) + database := &models.Database{ + Type: "sqlite", + SQLite: &models.SQLite{ + File: file, + }, + } + + return testForAll(m, database) } -func testForAll(m *testing.M) int { - os.Setenv("AUTH_MODE", "db_auth") - initDatabaseForTest() +func testForAll(m *testing.M, database *models.Database) int { + initDatabaseForTest(database) clearUp(username) return m.Run() @@ -210,8 +222,8 @@ func testForAll(m *testing.M) int { var defaultRegistered = false -func initDatabaseForTest() { - database, err := getDatabase() +func initDatabaseForTest(db *models.Database) { + database, err := getDatabase(db) if err != nil { panic(err) } @@ -226,6 +238,12 @@ func initDatabaseForTest() { if err := database.Register(alias); err != nil { panic(err) } + + if alias != "default" { + if err = globalOrm.Using(alias); err != nil { + log.Fatalf("failed to create new orm: %v", err) + } + } } func TestRegister(t *testing.T) { diff --git a/src/common/dao/mysql.go b/src/common/dao/mysql.go index a050b054e..8a9c2c304 100644 --- a/src/common/dao/mysql.go +++ b/src/common/dao/mysql.go @@ -16,15 +16,11 @@ package dao import ( - "errors" "fmt" - "net" - - "time" "github.com/astaxie/beego/orm" _ "github.com/go-sql-driver/mysql" //register mysql driver - "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/common/utils" ) type mysql struct { @@ -48,7 +44,8 @@ func NewMySQL(host, port, usr, pwd, database string) Database { // Register registers MySQL as the underlying database used func (m *mysql) Register(alias ...string) error { - if err := m.testConn(m.host, m.port); err != nil { + + if err := utils.TestTCPConn(m.host+":"+m.port, 60, 2); err != nil { return err } @@ -65,30 +62,6 @@ func (m *mysql) Register(alias ...string) error { return orm.RegisterDataBase(an, "mysql", conn) } -func (m *mysql) testConn(host, port string) error { - ch := make(chan int, 1) - go func() { - var err error - var c net.Conn - for { - c, err = net.DialTimeout("tcp", host+":"+port, 20*time.Second) - if err == nil { - c.Close() - ch <- 1 - } else { - log.Errorf("failed to connect to db, retry after 2 seconds :%v", err) - time.Sleep(2 * time.Second) - } - } - }() - select { - case <-ch: - return nil - case <-time.After(60 * time.Second): - return errors.New("failed to connect to database after 60 seconds") - } -} - // Name returns the name of MySQL func (m *mysql) Name() string { return "MySQL" diff --git a/src/common/dao/user_test.go b/src/common/dao/user_test.go index 7d421d848..c227d835e 100644 --- a/src/common/dao/user_test.go +++ b/src/common/dao/user_test.go @@ -38,6 +38,11 @@ func TestDeleteUser(t *testing.T) { if err != nil { t.Fatalf("failed to register user: %v", err) } + defer func(id int64) { + if err := deleteUser(id); err != nil { + t.Fatalf("failed to delete user %d: %v", id, err) + } + }(id) err = DeleteUser(int(id)) if err != nil { @@ -67,3 +72,11 @@ func TestDeleteUser(t *testing.T) { expected) } } + +func deleteUser(id int64) error { + if _, err := GetOrmer().QueryTable(&models.User{}). + Filter("UserID", id).Delete(); err != nil { + return err + } + return nil +} diff --git a/src/common/models/config.go b/src/common/models/config.go new file mode 100644 index 000000000..4039ea7ef --- /dev/null +++ b/src/common/models/config.go @@ -0,0 +1,95 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package models + +// Authentication ... +type Authentication struct { + Mode string `json:"mode"` + SelfRegistration bool `json:"self_registration"` + LDAP *LDAP `json:"ldap,omitempty"` +} + +// LDAP ... +type LDAP struct { + URL string `json:"url"` + SearchDN string `json:"search_dn"` + SearchPwd string `json:"search_pwd"` + BaseDN string `json:"base_dn"` + Filter string `json:"filter"` + UID string `json:"uid"` + Scope int `json:"scope"` +} + +// Database ... +type Database struct { + Type string `json:"type"` + MySQL *MySQL `json:"mysql,omitempty"` + SQLite *SQLite `json:"sqlite,omitempty"` +} + +// MySQL ... +type MySQL struct { + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password"` + Database string `json:"database"` +} + +// SQLite ... +type SQLite struct { + File string `json:"file"` +} + +// Email ... +type Email struct { + Host string `json:"host"` + Port string `json:"port"` + Username string `json:"username"` + Password string `json:"password"` + TLS bool `json:"tls"` + Identity string `json:"identity"` + From string `json:"from"` +} + +// Registry ... +type Registry struct { + URL string `json:"url"` +} + +// TokenService ... +type TokenService struct { + URL string `json:"url"` +} + +// SystemCfg holds all configurations of system +type SystemCfg struct { + DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port + Authentication *Authentication `json:"authentication"` + Database *Database `json:"database"` + TokenService *TokenService `json:"token_service"` + Registry *Registry `json:"registry"` + Email *Email `json:"email"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + ProjectCreationRestriction string `json:"project_creation_restriction"` + MaxJobWorkers int `json:"max_job_workers"` + JobLogDir string `json:"job_log_dir"` + InitialAdminPwd string `json:"initial_admin_pwd"` + CompressJS bool `json:"compress_js"` //TODO remove + TokenExpiration int `json:"token_expiration"` // in minute + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration"` +} diff --git a/src/common/utils/mail.go b/src/common/utils/email/mail.go similarity index 89% rename from src/common/utils/mail.go rename to src/common/utils/email/mail.go index c4a364cb8..d82f1d9bf 100644 --- a/src/common/utils/mail.go +++ b/src/common/utils/email/mail.go @@ -13,17 +13,19 @@ limitations under the License. */ -package utils +package email import ( "bytes" "crypto/tls" - "strings" + //"strings" "net/smtp" "text/template" - "github.com/astaxie/beego" + //"github.com/astaxie/beego" + "github.com/vmware/harbor/src/common/models" + "github.com/vmware/harbor/src/ui/config" ) // Mail holds information about content of Email @@ -34,24 +36,15 @@ type Mail struct { Message string } -// MailConfig holds information about Email configurations -type MailConfig struct { - Identity string - Host string - Port string - Username string - Password string - TLS bool -} - -var mc MailConfig +var mc models.Email // SendMail sends Email according to the configurations func (m Mail) SendMail() error { - - if mc.Host == "" { - loadConfig() + mc, err := config.Email() + if err != nil { + return err } + mailTemplate, err := template.ParseFiles("views/mail.tpl") if err != nil { return err @@ -123,6 +116,7 @@ func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error { return client.Quit() } +/* func loadConfig() { config, err := beego.AppConfig.GetSection("mail") if err != nil { @@ -142,3 +136,4 @@ func loadConfig() { TLS: useTLS, } } +*/ diff --git a/src/common/utils/log/logger.go b/src/common/utils/log/logger.go index dceb70a97..db5d3b4a2 100644 --- a/src/common/utils/log/logger.go +++ b/src/common/utils/log/logger.go @@ -22,8 +22,6 @@ import ( "runtime" "sync" "time" - - "github.com/vmware/harbor/src/common/config" ) var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) @@ -31,7 +29,7 @@ var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) func init() { logger.callDepth = 4 - lvl := config.LogLevel() + lvl := os.Getenv("LOG_LEVEL") if len(lvl) == 0 { logger.SetLevel(InfoLevel) return diff --git a/src/common/utils/registry/auth/tokenauthorizer.go b/src/common/utils/registry/auth/tokenauthorizer.go index 14b230e5a..917e43f38 100644 --- a/src/common/utils/registry/auth/tokenauthorizer.go +++ b/src/common/utils/registry/auth/tokenauthorizer.go @@ -25,7 +25,7 @@ import ( "sync" "time" - "github.com/vmware/harbor/src/common/config" + //"github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/common/utils/registry" registry_error "github.com/vmware/harbor/src/common/utils/registry/error" @@ -234,12 +234,15 @@ func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes [] // 2. the realm field returned by registry is an IP which can not reachable // inside Harbor func tokenURL(realm string) string { - extEndpoint := config.ExtEndpoint() - tokenEndpoint := config.TokenEndpoint() - if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && - strings.Contains(realm, extEndpoint) { - realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" - } + //TODO + /* + extEndpoint := config.ExtEndpoint() + tokenEndpoint := config.TokenEndpoint() + if len(extEndpoint) != 0 && len(tokenEndpoint) != 0 && + strings.Contains(realm, extEndpoint) { + realm = strings.TrimRight(tokenEndpoint, "/") + "/service/token" + } + */ return realm } diff --git a/src/common/utils/utils.go b/src/common/utils/utils.go index 4e930514e..a3ee7a5c7 100644 --- a/src/common/utils/utils.go +++ b/src/common/utils/utils.go @@ -16,10 +16,14 @@ package utils import ( + "fmt" "math/rand" + "net" "net/url" "strings" "time" + + "github.com/vmware/harbor/src/common/utils/log" ) // FormatEndpoint formats endpoint @@ -70,3 +74,37 @@ func GenerateRandomString() string { } return string(result) } + +// timeout in second +func TestTCPConn(addr string, timeout, interval int) error { + success := make(chan int) + cancel := make(chan int) + + go func() { + for { + select { + case <-cancel: + break + default: + conn, err := net.DialTimeout("tcp", addr, time.Duration(timeout)*time.Second) + if err != nil { + log.Errorf("failed to connect to tcp://%s, retry after %d seconds :%v", + addr, interval, err) + time.Sleep(time.Duration(interval) * time.Second) + continue + } + conn.Close() + success <- 1 + break + } + } + }() + + select { + case <-success: + return nil + case <-time.After(time.Duration(timeout) * time.Second): + cancel <- 1 + return fmt.Errorf("failed to connect to tcp:%s after %d seconds", addr, timeout) + } +} diff --git a/src/jobservice/api/replication.go b/src/jobservice/api/replication.go index 51807ed39..25aef81e4 100644 --- a/src/jobservice/api/replication.go +++ b/src/jobservice/api/replication.go @@ -25,12 +25,12 @@ import ( "github.com/vmware/harbor/src/common/api" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/job" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/jobservice/utils" "github.com/vmware/harbor/src/common/models" u "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/job" + "github.com/vmware/harbor/src/jobservice/utils" ) // ReplicationJob handles /api/replicationJobs /api/replicationJobs/:id/log @@ -171,7 +171,13 @@ func (rj *ReplicationJob) GetLog() { rj.RenderError(http.StatusBadRequest, "Invalid job id") return } - logFile := utils.GetJobLogPath(jid) + logFile, err := utils.GetJobLogPath(jid) + if err != nil { + log.Errorf("failed to get log path of job %s: %v", idStr, err) + rj.RenderError(http.StatusInternalServerError, + http.StatusText(http.StatusInternalServerError)) + return + } rj.Ctx.Output.Download(logFile) } diff --git a/src/jobservice/config/config.go b/src/jobservice/config/config.go index 2e1b2c31d..18999e855 100644 --- a/src/jobservice/config/config.go +++ b/src/jobservice/config/config.go @@ -16,121 +16,149 @@ package config import ( - "fmt" + "encoding/json" "os" - "strconv" + "time" - "github.com/astaxie/beego" - "github.com/vmware/harbor/src/common/utils/log" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" + //"github.com/vmware/harbor/src/common/utils/log" ) -const defaultMaxWorkers int = 10 +var mg *comcfg.Manager -var maxJobWorkers int -var localUIURL string -var localRegURL string -var logDir string -var uiSecret string -var secretKey string -var verifyRemoteCert string +// Configuration holds configurations of Jobservice +type Configuration struct { + Database *models.Database `json:"database"` + Registry *models.Registry `json:"registry"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + MaxJobWorkers int `json:"max_job_workers"` + JobLogDir string `json:"job_log_dir"` + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration"` +} -func init() { - maxWorkersEnv := os.Getenv("MAX_JOB_WORKERS") - maxWorkers64, err := strconv.ParseInt(maxWorkersEnv, 10, 32) - maxJobWorkers = int(maxWorkers64) +func Init() error { + adminServerURL := os.Getenv("ADMIN_SERVER_URL") + if len(adminServerURL) == 0 { + adminServerURL = "http://admin_server" + } + mg = comcfg.NewManager("cfg", adminServerURL) + + if err := mg.Loader.Init(); err != nil { + return err + } + + if err := load(); err != nil { + return err + } + + path, err := LogDir() if err != nil { - log.Warningf("Failed to parse max works setting, error: %v, the default value: %d will be used", err, defaultMaxWorkers) - maxJobWorkers = defaultMaxWorkers + return err + } + if err := os.MkdirAll(path, 0600); err != nil { + return err } - localRegURL = os.Getenv("REGISTRY_URL") - if len(localRegURL) == 0 { - localRegURL = "http://registry:5000" + return nil +} + +// get returns configurations of jobservice from cache, +// if cache is null, it loads first +func get() (*Configuration, error) { + cfg := mg.GetFromCache() + if cfg != nil { + return cfg.(*Configuration), nil } - localUIURL = os.Getenv("UI_URL") - if len(localUIURL) == 0 { - localUIURL = "http://ui" + if err := load(); err != nil { + return nil, err } - logDir = os.Getenv("LOG_DIR") - if len(logDir) == 0 { - logDir = "/var/log" - } + return mg.GetFromCache().(*Configuration), nil +} - f, err := os.Open(logDir) - defer f.Close() +// load loads configurations of jobservice and puts them into cache +func load() error { + raw, err := mg.Loader.Load() if err != nil { - panic(err) + return err } - finfo, err := f.Stat() + + cfg := &Configuration{} + if err = json.Unmarshal(raw, cfg); err != nil { + return err + } + + if err = mg.Cache.Put(mg.Key, cfg, + time.Duration(cfg.CfgExpiration)*time.Second); err != nil { + return err + } + + return nil +} + +// VerifyRemoteCert returns bool value. +func VerifyRemoteCert() (bool, error) { + cfg, err := get() if err != nil { - panic(err) - } - if !finfo.IsDir() { - panic(fmt.Sprintf("%s is not a direcotry", logDir)) + return true, err } + return cfg.VerifyRemoteCert, nil +} - uiSecret = os.Getenv("UI_SECRET") - if len(uiSecret) == 0 { - panic("UI Secret is not set") +// Database ... +func Database() (*models.Database, error) { + cfg, err := get() + if err != nil { + return nil, err } - - verifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT") - if len(verifyRemoteCert) == 0 { - verifyRemoteCert = "on" - } - - configPath := os.Getenv("CONFIG_PATH") - if len(configPath) != 0 { - log.Infof("Config path: %s", configPath) - beego.LoadAppConfig("ini", configPath) - } - - secretKey = os.Getenv("SECRET_KEY") - if len(secretKey) != 16 { - panic("The length of secretkey has to be 16 characters!") - } - - log.Debugf("config: maxJobWorkers: %d", maxJobWorkers) - log.Debugf("config: localUIURL: %s", localUIURL) - log.Debugf("config: localRegURL: %s", localRegURL) - log.Debugf("config: verifyRemoteCert: %s", verifyRemoteCert) - log.Debugf("config: logDir: %s", logDir) - log.Debugf("config: uiSecret: ******") + return cfg.Database, nil } // MaxJobWorkers ... -func MaxJobWorkers() int { - return maxJobWorkers +func MaxJobWorkers() (int, error) { + cfg, err := get() + if err != nil { + return 0, err + } + return cfg.MaxJobWorkers, nil } // LocalUIURL returns the local ui url, job service will use this URL to call API hosted on ui process func LocalUIURL() string { - return localUIURL + return "http://ui" } // LocalRegURL returns the local registry url, job service will use this URL to pull image from the registry -func LocalRegURL() string { - return localRegURL +func LocalRegURL() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Registry.URL, nil } // LogDir returns the absolute path to which the log file will be written -func LogDir() string { - return logDir +func LogDir() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.JobLogDir, nil } // UISecret will return the value of secret cookie for jobsevice to call UI API. func UISecret() string { - return uiSecret + return os.Getenv("UI_SECRET") } // SecretKey will return the secret key for encryption/decryption password in target. -func SecretKey() string { - return secretKey -} - -// VerifyRemoteCert return the flag to tell jobservice whether or not verify the cert of remote registry -func VerifyRemoteCert() bool { - return verifyRemoteCert != "off" +func SecretKey() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.SecretKey, nil } diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index fd12d5443..b6c9cd47a 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -20,12 +20,12 @@ import ( "sync" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/jobservice/replication" - "github.com/vmware/harbor/src/jobservice/utils" "github.com/vmware/harbor/src/common/models" uti "github.com/vmware/harbor/src/common/utils" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/replication" + "github.com/vmware/harbor/src/jobservice/utils" ) // RepJobParm wraps the parm of a job @@ -184,14 +184,17 @@ func (sm *SM) Init() { } // Reset resets the state machine so it will start handling another job. -func (sm *SM) Reset(jid int64) error { +func (sm *SM) Reset(jid int64) (err error) { //To ensure the new jobID is visible to the thread to stop the SM sm.lock.Lock() sm.JobID = jid sm.desiredState = "" sm.lock.Unlock() - sm.Logger = utils.NewLogger(sm.JobID) + sm.Logger, err = utils.NewLogger(sm.JobID) + if err != nil { + return + } //init parms job, err := dao.GetRepJob(sm.JobID) if err != nil { @@ -207,13 +210,22 @@ func (sm *SM) Reset(jid int64) error { if policy == nil { return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID) } + + regURL, err := config.LocalRegURL() + if err != nil { + return err + } + verify, err := config.VerifyRemoteCert() + if err != nil { + return err + } sm.Parms = &RepJobParm{ - LocalRegURL: config.LocalRegURL(), + LocalRegURL: regURL, Repository: job.Repository, Tags: job.TagList, Enabled: policy.Enabled, Operation: job.Operation, - Insecure: !config.VerifyRemoteCert(), + Insecure: !verify, } if policy.Enabled == 0 { //worker will cancel this job @@ -231,7 +243,11 @@ func (sm *SM) Reset(jid int64) error { pwd := target.Password if len(pwd) != 0 { - pwd, err = uti.ReversibleDecrypt(pwd, config.SecretKey()) + key, err := config.SecretKey() + if err != nil { + return err + } + pwd, err = uti.ReversibleDecrypt(pwd, key) if err != nil { return fmt.Errorf("failed to decrypt password: %v", err) } diff --git a/src/jobservice/job/workerpool.go b/src/jobservice/job/workerpool.go index a1034441a..06301cd69 100644 --- a/src/jobservice/job/workerpool.go +++ b/src/jobservice/job/workerpool.go @@ -17,9 +17,9 @@ package job import ( "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/config" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" ) type workerPool struct { @@ -111,17 +111,22 @@ func NewWorker(id int) *Worker { } // InitWorkerPool create workers according to configuration. -func InitWorkerPool() { - WorkerPool = &workerPool{ - workerChan: make(chan *Worker, config.MaxJobWorkers()), - workerList: make([]*Worker, 0, config.MaxJobWorkers()), +func InitWorkerPool() error { + n, err := config.MaxJobWorkers() + if err != nil { + return err } - for i := 0; i < config.MaxJobWorkers(); i++ { + WorkerPool = &workerPool{ + workerChan: make(chan *Worker, n), + workerList: make([]*Worker, 0, n), + } + for i := 0; i < n; i++ { worker := NewWorker(i) WorkerPool.workerList = append(WorkerPool.workerList, worker) worker.Start() log.Debugf("worker %d started", worker.ID) } + return nil } // Dispatch will listen to the jobQueue of job service and try to pick a free worker from the worker pool and assign the job to it. diff --git a/src/jobservice/main.go b/src/jobservice/main.go index 5467d96c6..33516761a 100644 --- a/src/jobservice/main.go +++ b/src/jobservice/main.go @@ -18,13 +18,28 @@ package main import ( "github.com/astaxie/beego" "github.com/vmware/harbor/src/common/dao" - "github.com/vmware/harbor/src/jobservice/job" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" + "github.com/vmware/harbor/src/jobservice/job" ) func main() { - dao.InitDatabase() + log.Info("initializing configurations...") + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + log.Info("configurations initialization completed") + + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configurations: %v", err) + } + + if err := dao.InitDatabase(database); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + initRouters() job.InitWorkerPool() go job.Dispatch() @@ -48,3 +63,13 @@ func resumeJobs() { log.Warningf("Failed to jobs to resume, error: %v", err) } } + +/* +func init() { + configPath := os.Getenv("CONFIG_PATH") + if len(configPath) != 0 { + log.Infof("Config path: %s", configPath) + beego.LoadAppConfig("ini", configPath) + } +} +*/ diff --git a/src/jobservice/utils/logger.go b/src/jobservice/utils/logger.go index 6a358285c..7b3b164f4 100644 --- a/src/jobservice/utils/logger.go +++ b/src/jobservice/utils/logger.go @@ -18,16 +18,20 @@ package utils import ( "fmt" - "github.com/vmware/harbor/src/jobservice/config" - "github.com/vmware/harbor/src/common/utils/log" "os" "path/filepath" "strconv" + + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/jobservice/config" ) // NewLogger create a logger for a speicified job -func NewLogger(jobID int64) *log.Logger { - logFile := GetJobLogPath(jobID) +func NewLogger(jobID int64) (*log.Logger, error) { + logFile, err := GetJobLogPath(jobID) + if err != nil { + return nil, err + } d := filepath.Dir(logFile) if _, err := os.Stat(d); os.IsNotExist(err) { err := os.MkdirAll(d, 0660) @@ -40,11 +44,11 @@ func NewLogger(jobID int64) *log.Logger { log.Errorf("Failed to open log file %s, the log of job %d will be printed to standard output, the error: %v", logFile, jobID, err) f = os.Stdout } - return log.New(f, log.NewTextFormatter(), log.InfoLevel) + return log.New(f, log.NewTextFormatter(), log.InfoLevel), nil } // GetJobLogPath returns the absolute path in which the job log file is located. -func GetJobLogPath(jobID int64) string { +func GetJobLogPath(jobID int64) (string, error) { f := fmt.Sprintf("job_%d.log", jobID) k := jobID / 1000 p := "" @@ -61,6 +65,10 @@ func GetJobLogPath(jobID int64) string { p = filepath.Join(d, p) } - p = filepath.Join(config.LogDir(), p, f) - return p + base, err := config.LogDir() + if err != nil { + return "", err + } + p = filepath.Join(base, p, f) + return p, nil } diff --git a/src/ui/api/config.go b/src/ui/api/config.go new file mode 100644 index 000000000..70e41459f --- /dev/null +++ b/src/ui/api/config.go @@ -0,0 +1,245 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + "strconv" + //"strings" + + "github.com/vmware/harbor/src/common/api" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/dao" + //"github.com/vmware/harbor/src/common/models" + //"github.com/vmware/harbor/src/common/utils" + "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" +) + +type ConfigAPI struct { + api.BaseAPI +} + +// Prepare validates the user +func (c *ConfigAPI) Prepare() { + userID := c.ValidateUser() + isSysAdmin, err := dao.IsAdminRole(userID) + if err != nil { + log.Errorf("failed to check the role of user: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if !isSysAdmin { + c.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden)) + } +} + +// Get returns configurations +func (c *ConfigAPI) Get() { + cfg, err := config.GetSystemCfg() + if err != nil { + log.Errorf("failed to get configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + //TODO filter attr in sys config + + c.Data["json"] = cfg + c.ServeJSON() +} + +// Put updates configurations +func (c *ConfigAPI) Put() { + m := map[string]string{} + c.DecodeJSONReq(&m) + if err := validateCfg(m); err != nil { + c.CustomAbort(http.StatusBadRequest, err.Error()) + } + + if value, ok := m[comcfg.AUTH_MODE]; ok { + mode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if mode != value { + flag, err := authModeCanBeModified() + if err != nil { + log.Errorf("failed to determine whether auth mode can be modified: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if !flag { + c.CustomAbort(http.StatusBadRequest, + fmt.Sprintf("%s can not be modified as new users have been inserted into database", + comcfg.AUTH_MODE)) + } + } + } + + log.Info(m) + + if err := config.Upload(m); err != nil { + log.Errorf("failed to upload configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if err := config.Load(); err != nil { + log.Errorf("failed to load configurations: %v", err) + c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } +} + +// TODO ldap timeout, scope value +func validateCfg(c map[string]string) error { + if value, ok := c[comcfg.AUTH_MODE]; ok { + if value != comcfg.DB_AUTH && value != comcfg.LDAP_AUTH { + return fmt.Errorf("invalid %s, shoud be %s or %s", comcfg.AUTH_MODE, comcfg.DB_AUTH, comcfg.LDAP_AUTH) + } + + if value == comcfg.LDAP_AUTH { + if _, ok := c[comcfg.LDAP_URL]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_URL) + } + if _, ok := c[comcfg.LDAP_BASE_DN]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_BASE_DN) + } + if _, ok := c[comcfg.LDAP_UID]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_UID) + } + if _, ok := c[comcfg.LDAP_SCOPE]; !ok { + return fmt.Errorf("%s is missing", comcfg.LDAP_SCOPE) + } + } + } + + if ldapURL, ok := c[comcfg.LDAP_URL]; ok && len(ldapURL) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAP_URL) + } + if baseDN, ok := c[comcfg.LDAP_BASE_DN]; ok && len(baseDN) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAP_BASE_DN) + } + if uID, ok := c[comcfg.LDAP_UID]; ok && len(uID) == 0 { + return fmt.Errorf("%s is empty", comcfg.LDAP_UID) + } + if scope, ok := c[comcfg.LDAP_SCOPE]; ok && + scope != comcfg.LDAP_SCOPE_BASE && + scope != comcfg.LDAP_SCOPE_ONELEVEL && + scope != comcfg.LDAP_SCOPE_SUBTREE { + return fmt.Errorf("invalid %s, should be %s, %s or %s", + comcfg.LDAP_SCOPE, + comcfg.LDAP_SCOPE_BASE, + comcfg.LDAP_SCOPE_ONELEVEL, + comcfg.LDAP_SCOPE_SUBTREE) + } + + if self, ok := c[comcfg.SELF_REGISTRATION]; ok && + self != "true" && self != "false" { + return fmt.Errorf("%s should be %s or %s", + comcfg.SELF_REGISTRATION, "true", "false") + } + + if port, ok := c[comcfg.EMAIL_SERVER_PORT]; ok { + if p, err := strconv.Atoi(port); err != nil || p < 0 || p > 65535 { + return fmt.Errorf("invalid %s", comcfg.EMAIL_SERVER_PORT) + } + } + + if ssl, ok := c[comcfg.EMAIL_SSL]; ok && ssl != "true" && ssl != "false" { + return fmt.Errorf("%s should be true or false", comcfg.EMAIL_SSL) + } + + if crt, ok := c[comcfg.PROJECT_CREATION_RESTRICTION]; ok && + crt != comcfg.PRO_CRT_RESTR_EVERYONE && + crt != comcfg.PRO_CRT_RESTR_ADM_ONLY { + return fmt.Errorf("invalid %s, should be %s or %s", + comcfg.PROJECT_CREATION_RESTRICTION, + comcfg.PRO_CRT_RESTR_ADM_ONLY, + comcfg.PRO_CRT_RESTR_EVERYONE) + } + + if verify, ok := c[comcfg.VERIFY_REMOTE_CERT]; ok && verify != "true" && verify != "false" { + return fmt.Errorf("invalid %s, should be true or false", comcfg.VERIFY_REMOTE_CERT) + } + + if worker, ok := c[comcfg.MAX_JOB_WORKERS]; ok { + if w, err := strconv.Atoi(worker); err != nil || w <= 0 { + return fmt.Errorf("invalid %s", comcfg.MAX_JOB_WORKERS) + } + } + + return nil +} + +/* +func convert() ([]*models.Config, error) { + cfgs := []*models.Config{} + var err error + pwdKeys := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + for _, pwdKey := range pwdKeys { + if pwd, ok := c[pwdKey]; ok && len(pwd) != 0 { + c[pwdKey], err = utils.ReversibleEncrypt(pwd, ui_cfg.SecretKey()) + if err != nil { + return nil, err + } + } + } + + for _, key := range configKeys { + if value, ok := c[key]; ok { + cfgs = append(cfgs, &models.Config{ + Key: key, + Value: value, + }) + } + } + + return cfgs, nil +} +*/ +/* +//[]*models.Config >> cfgForGet +func convert(cfg *config.Configuration) (map[string]interface{}, error) { + result := map[string]interface{}{} + + for _, config := range configs { + cfg[config.Key] = &value{ + Value: config.Value, + Editable: true, + } + } + + dels := []string{config.LDAP_SEARCH_PWD, config.EMAIL_PWD} + for _, del := range dels { + if _, ok := cfg[del]; ok { + delete(cfg, del) + } + } + + flag, err := authModeCanBeModified() + if err != nil { + return nil, err + } + cfg[config.AUTH_MODE].Editable = flag + + return cfgForGet(cfg), nil +} +*/ +func authModeCanBeModified() (bool, error) { + return dao.AuthModeCanBeModified() +} diff --git a/src/ui/api/project.go b/src/ui/api/project.go index 7ba50fa3b..58ec9b96f 100644 --- a/src/ui/api/project.go +++ b/src/ui/api/project.go @@ -77,7 +77,13 @@ func (p *ProjectAPI) Post() { if err != nil { log.Errorf("Failed to check admin role: %v", err) } - if !isSysAdmin && config.OnlyAdminCreateProject() { + + onlyAdmin, err := config.OnlyAdminCreateProject() + if err != nil { + log.Errorf("failed to determine whether only admin can create projects: %v", err) + p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + if !isSysAdmin && onlyAdmin { log.Errorf("Only sys admin can create project") p.RenderError(http.StatusForbidden, "Only system admin can create project") return diff --git a/src/ui/api/repository.go b/src/ui/api/repository.go index 4db0d1eb1..9bc7036c0 100644 --- a/src/ui/api/repository.go +++ b/src/ui/api/repository.go @@ -361,11 +361,19 @@ func (ra *RepositoryAPI) GetManifests() { } func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return nil, err + } + + insecure, err := api.GetIsInsecure() + if err != nil { + return nil, err + } username, password, ok := ra.Ctx.Request.BasicAuth() if ok { - return newRepositoryClient(endpoint, api.GetIsInsecure(), username, password, + return newRepositoryClient(endpoint, insecure, username, password, repoName, "repository", repoName, "pull", "push", "*") } @@ -374,7 +382,7 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo return nil, err } - return cache.NewRepositoryClient(endpoint, api.GetIsInsecure(), username, repoName, + return cache.NewRepositoryClient(endpoint, insecure, username, repoName, "repository", repoName, "pull", "push", "*") } diff --git a/src/ui/api/target.go b/src/ui/api/target.go index abe59a09c..c1f88d2ba 100644 --- a/src/ui/api/target.go +++ b/src/ui/api/target.go @@ -41,7 +41,12 @@ type TargetAPI struct { // Prepare validates the user func (t *TargetAPI) Prepare() { - t.secretKey = config.SecretKey() + var err error + t.secretKey, err = config.SecretKey() + if err != nil { + log.Errorf("failed to get secret key: %v", err) + t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } userID := t.ValidateUser() isSysAdmin, err := dao.IsAdminRole(userID) @@ -97,7 +102,12 @@ func (t *TargetAPI) Ping() { password = t.GetString("password") } - registry, err := newRegistryClient(endpoint, api.GetIsInsecure(), username, password, + insecure, err := api.GetIsInsecure() + if err != nil { + log.Errorf("failed to check whether insecure or not: %v", err) + t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + registry, err := newRegistryClient(endpoint, insecure, username, password, "", "", "") if err != nil { // timeout, dns resolve error, connection refused, etc. diff --git a/src/ui/api/user.go b/src/ui/api/user.go index 7c1212b45..d5ca84efb 100644 --- a/src/ui/api/user.go +++ b/src/ui/api/user.go @@ -46,10 +46,21 @@ type passwordReq struct { // Prepare validates the URL and parms func (ua *UserAPI) Prepare() { + mode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } - ua.AuthMode = config.AuthMode() + ua.AuthMode = mode - ua.SelfRegistration = config.SelfRegistration() + self, err := config.SelfRegistration() + if err != nil { + log.Errorf("failed to get self registration: %v", err) + ua.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + ua.SelfRegistration = self if ua.Ctx.Input.IsPost() { sessionUserID := ua.GetSession("userId") @@ -82,7 +93,6 @@ func (ua *UserAPI) Prepare() { } } - var err error ua.IsAdmin, err = dao.IsAdminRole(ua.currentUserID) if err != nil { log.Errorf("Error occurred in IsAdminRole:%v", err) @@ -234,7 +244,7 @@ func (ua *UserAPI) Delete() { return } - if config.AuthMode() == "ldap_auth" { + if ua.AuthMode == "ldap_auth" { ua.CustomAbort(http.StatusForbidden, "user can not be deleted in LDAP authentication mode") } diff --git a/src/ui/api/utils.go b/src/ui/api/utils.go index 185db1f18..73a11c869 100644 --- a/src/ui/api/utils.go +++ b/src/ui/api/utils.go @@ -20,11 +20,9 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net" "net/http" "sort" "strings" - "time" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" @@ -242,7 +240,7 @@ func addAuthentication(req *http.Request) { // SyncRegistry syncs the repositories of registry with database. func SyncRegistry() error { - log.Debugf("Start syncing repositories from registry to DB... ") + log.Infof("Start syncing repositories from registry to DB... ") reposInRegistry, err := catalog() if err != nil { @@ -304,7 +302,7 @@ func SyncRegistry() error { } } - log.Debugf("Sync repositories from registry to DB is done.") + log.Infof("Sync repositories from registry to DB is done.") return nil } @@ -350,7 +348,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string } // TODO remove the workaround when the bug of registry is fixed - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return needsAdd, needsDel, err + } client, err := cache.NewRepositoryClient(endpoint, true, "admin", repoInR, "repository", repoInR) if err != nil { @@ -372,7 +373,10 @@ func diffRepos(reposInRegistry []string, reposInDB []string) ([]string, []string j++ } else { // TODO remove the workaround when the bug of registry is fixed - endpoint := config.InternalRegistryURL() + endpoint, err := config.RegistryURL() + if err != nil { + return needsAdd, needsDel, err + } client, err := cache.NewRepositoryClient(endpoint, true, "admin", repoInR, "repository", repoInR) if err != nil { @@ -422,32 +426,18 @@ func projectExists(repository string) (bool, error) { } func initRegistryClient() (r *registry.Registry, err error) { - endpoint := config.InternalRegistryURL() - - addr := endpoint - if strings.Contains(endpoint, "/") { - addr = endpoint[strings.LastIndex(endpoint, "/")+1:] + endpoint, err := config.RegistryURL() + if err != nil { + return nil, err } - ch := make(chan int, 1) - go func() { - var err error - var c net.Conn - for { - c, err = net.DialTimeout("tcp", addr, 20*time.Second) - if err == nil { - c.Close() - ch <- 1 - } else { - log.Errorf("failed to connect to registry client, retry after 2 seconds :%v", err) - time.Sleep(2 * time.Second) - } - } - }() - select { - case <-ch: - case <-time.After(60 * time.Second): - panic("Failed to connect to registry client after 60 seconds") + addr := endpoint + if strings.Contains(endpoint, "://") { + addr = strings.Split(endpoint, "://")[1] + } + + if err := utils.TestTCPConn(addr, 60, 2); err != nil { + return nil, err } registryClient, err := cache.NewRegistryClient(endpoint, true, "admin", diff --git a/src/ui/auth/authenticator.go b/src/ui/auth/authenticator.go index 23abdc8f2..d255b60a9 100644 --- a/src/ui/auth/authenticator.go +++ b/src/ui/auth/authenticator.go @@ -50,7 +50,10 @@ func Register(name string, authenticator Authenticator) { // Login authenticates user credentials based on setting. func Login(m models.AuthModel) (*models.User, error) { - var authMode = config.AuthMode() + authMode, err := config.AuthMode() + if err != nil { + return nil, err + } if authMode == "" || m.Principal == "admin" { authMode = "db_auth" } diff --git a/src/ui/auth/ldap/ldap.go b/src/ui/auth/ldap/ldap.go index 46c563cbe..df27ba837 100644 --- a/src/ui/auth/ldap/ldap.go +++ b/src/ui/auth/ldap/ldap.go @@ -46,7 +46,13 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { return nil, fmt.Errorf("the principal contains meta char: %q", c) } } - ldapURL := config.LDAP().URL + + settings, err := config.LDAP() + if err != nil { + return nil, err + } + + ldapURL := settings.URL if ldapURL == "" { return nil, errors.New("can not get any available LDAP_URL") } @@ -57,16 +63,16 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } ldap.SetOption(openldap.LDAP_OPT_PROTOCOL_VERSION, openldap.LDAP_VERSION3) - ldapBaseDn := config.LDAP().BaseDn + ldapBaseDn := settings.BaseDN if ldapBaseDn == "" { return nil, errors.New("can not get any available LDAP_BASE_DN") } log.Debug("baseDn:", ldapBaseDn) - ldapSearchDn := config.LDAP().SearchDn + ldapSearchDn := settings.SearchDN if ldapSearchDn != "" { log.Debug("Search DN: ", ldapSearchDn) - ldapSearchPwd := config.LDAP().SearchPwd + ldapSearchPwd := settings.SearchPwd err = ldap.Bind(ldapSearchDn, ldapSearchPwd) if err != nil { log.Debug("Bind search dn error", err) @@ -74,8 +80,8 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } } - attrName := config.LDAP().UID - filter := config.LDAP().Filter + attrName := settings.UID + filter := settings.Filter if filter != "" { filter = "(&" + filter + "(" + attrName + "=" + m.Principal + "))" } else { @@ -83,11 +89,11 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } log.Debug("one or more filter", filter) - ldapScope := config.LDAP().Scope + ldapScope := settings.Scope var scope int - if ldapScope == "1" { + if ldapScope == 1 { scope = openldap.LDAP_SCOPE_BASE - } else if ldapScope == "2" { + } else if ldapScope == 2 { scope = openldap.LDAP_SCOPE_ONELEVEL } else { scope = openldap.LDAP_SCOPE_SUBTREE diff --git a/src/ui/config/config.go b/src/ui/config/config.go index 783779c4d..aa726e810 100644 --- a/src/ui/config/config.go +++ b/src/ui/config/config.go @@ -13,143 +13,225 @@ limitations under the License. */ -// Package config provides methods to get configurations required by code in src/ui package config import ( - "strconv" - "strings" + "encoding/json" + "os" + "time" - commonConfig "github.com/vmware/harbor/src/common/config" - "github.com/vmware/harbor/src/common/utils/log" + comcfg "github.com/vmware/harbor/src/common/config" + "github.com/vmware/harbor/src/common/models" ) -// LDAPSetting wraps the setting of an LDAP server -type LDAPSetting struct { - URL string - BaseDn string - SearchDn string - SearchPwd string - UID string - Filter string - Scope string +var mg *comcfg.Manager + +type Configuration struct { + DomainName string `json:"domain_name"` // Harbor external URL: protocal://host:port + Authentication *models.Authentication `json:"authentication"` + Database *models.Database `json:"database"` + TokenService *models.TokenService `json:"token_service"` + Registry *models.Registry `json:"registry"` + Email *models.Email `json:"email"` + VerifyRemoteCert bool `json:"verify_remote_cert"` + ProjectCreationRestriction string `json:"project_creation_restriction"` + InitialAdminPwd string `json:"initial_admin_pwd"` + //TODO remove + CompressJS bool `json:"compress_js"` + TokenExpiration int `json:"token_expiration"` + SecretKey string `json:"secret_key"` + CfgExpiration int `json:"cfg_expiration` } -type uiParser struct{} +func Init() error { + adminServerURL := os.Getenv("ADMIN_SERVER_URL") + if len(adminServerURL) == 0 { + adminServerURL = "http://admin_server" + } + mg = comcfg.NewManager("cfg", adminServerURL) -// Parse parses the auth settings url settings and other configuration consumed by code under src/ui -func (up *uiParser) Parse(raw map[string]string, config map[string]interface{}) error { - mode := raw["AUTH_MODE"] - if mode == "ldap_auth" { - setting := LDAPSetting{ - URL: raw["LDAP_URL"], - BaseDn: raw["LDAP_BASE_DN"], - SearchDn: raw["LDAP_SEARCH_DN"], - SearchPwd: raw["LDAP_SEARCH_PWD"], - UID: raw["LDAP_UID"], - Filter: raw["LDAP_FILTER"], - Scope: raw["LDAP_SCOPE"], - } - config["ldap"] = setting + if err := mg.Loader.Init(); err != nil { + return err } - config["auth_mode"] = mode - var tokenExpiration = 30 //minutes - if len(raw["TOKEN_EXPIRATION"]) > 0 { - i, err := strconv.Atoi(raw["TOKEN_EXPIRATION"]) - if err != nil { - log.Warningf("failed to parse token expiration: %v, using default value %d", err, tokenExpiration) - } else if i <= 0 { - log.Warningf("invalid token expiration, using default value: %d minutes", tokenExpiration) - } else { - tokenExpiration = i - } + + if err := Load(); err != nil { + return err } - config["token_exp"] = tokenExpiration - config["admin_password"] = raw["HARBOR_ADMIN_PASSWORD"] - config["ext_reg_url"] = raw["EXT_REG_URL"] - config["ui_secret"] = raw["UI_SECRET"] - config["secret_key"] = raw["SECRET_KEY"] - config["self_registration"] = raw["SELF_REGISTRATION"] != "off" - config["admin_create_project"] = strings.ToLower(raw["PROJECT_CREATION_RESTRICTION"]) == "adminonly" - registryURL := raw["REGISTRY_URL"] - registryURL = strings.TrimRight(registryURL, "/") - config["internal_registry_url"] = registryURL - jobserviceURL := raw["JOB_SERVICE_URL"] - jobserviceURL = strings.TrimRight(jobserviceURL, "/") - config["internal_jobservice_url"] = jobserviceURL + return nil } -var uiConfig *commonConfig.Config +// Get returns configurations of UI, if cache is null, it loads first +func get() (*Configuration, error) { + cfg := mg.GetFromCache() + if cfg != nil { + return cfg.(*Configuration), nil + } -func init() { - uiKeys := []string{"AUTH_MODE", "LDAP_URL", "LDAP_BASE_DN", "LDAP_SEARCH_DN", "LDAP_SEARCH_PWD", "LDAP_UID", "LDAP_FILTER", "LDAP_SCOPE", "TOKEN_EXPIRATION", "HARBOR_ADMIN_PASSWORD", "EXT_REG_URL", "UI_SECRET", "SECRET_KEY", "SELF_REGISTRATION", "PROJECT_CREATION_RESTRICTION", "REGISTRY_URL", "JOB_SERVICE_URL"} - uiConfig = &commonConfig.Config{ - Config: make(map[string]interface{}), - Loader: &commonConfig.EnvConfigLoader{Keys: uiKeys}, - Parser: &uiParser{}, - } - if err := uiConfig.Load(); err != nil { - panic(err) + if err := Load(); err != nil { + return nil, err } + + return mg.GetFromCache().(*Configuration), nil } -// Reload ... -func Reload() error { - return uiConfig.Load() +// Load loads configurations of UI and puts them into cache +func Load() error { + raw, err := mg.Loader.Load() + if err != nil { + return err + } + + cfg := &Configuration{} + if err = json.Unmarshal(raw, cfg); err != nil { + return err + } + + if err = mg.Cache.Put(mg.Key, cfg, + time.Duration(cfg.CfgExpiration)*time.Second); err != nil { + return err + } + + return nil +} + +// Upload uploads all system configutations to admin server +func Upload(cfg map[string]string) error { + b, err := json.Marshal(cfg) + if err != nil { + return err + } + return mg.Loader.Upload(b) +} + +// GetSystemCfg returns the system configurations +func GetSystemCfg() (*models.SystemCfg, error) { + raw, err := mg.Loader.Load() + if err != nil { + return nil, err + } + + cfg := &models.SystemCfg{} + if err = json.Unmarshal(raw, cfg); err != nil { + return nil, err + } + return cfg, nil } // AuthMode ... -func AuthMode() string { - return uiConfig.Config["auth_mode"].(string) +func AuthMode() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Authentication.Mode, nil } // LDAP returns the setting of ldap server -func LDAP() LDAPSetting { - return uiConfig.Config["ldap"].(LDAPSetting) +func LDAP() (*models.LDAP, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Authentication.LDAP, nil } // TokenExpiration returns the token expiration time (in minute) -func TokenExpiration() int { - return uiConfig.Config["token_exp"].(int) +func TokenExpiration() (int, error) { + cfg, err := get() + if err != nil { + return 0, err + } + return cfg.TokenExpiration, nil } -// ExtRegistryURL returns the registry URL to exposed to external client -func ExtRegistryURL() string { - return uiConfig.Config["ext_reg_url"].(string) -} - -// UISecret returns the value of UI secret cookie, used for communication between UI and JobService -func UISecret() string { - return uiConfig.Config["ui_secret"].(string) +// DomainName returns the external URL of Harbor: protocal://host:port +func DomainName() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.DomainName, nil } // SecretKey returns the secret key to encrypt the password of target -func SecretKey() string { - return uiConfig.Config["secret_key"].(string) +func SecretKey() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.SecretKey, nil } // SelfRegistration returns the enablement of self registration -func SelfRegistration() bool { - return uiConfig.Config["self_registration"].(bool) +func SelfRegistration() (bool, error) { + cfg, err := get() + if err != nil { + return false, err + } + return cfg.Authentication.SelfRegistration, nil } -// InternalRegistryURL returns registry URL for internal communication between Harbor containers -func InternalRegistryURL() string { - return uiConfig.Config["internal_registry_url"].(string) +// RegistryURL ... +func RegistryURL() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.Registry.URL, nil } // InternalJobServiceURL returns jobservice URL for internal communication between Harbor containers func InternalJobServiceURL() string { - return uiConfig.Config["internal_jobservice_url"].(string) + return "http://jobservice" } // InitialAdminPassword returns the initial password for administrator -func InitialAdminPassword() string { - return uiConfig.Config["admin_password"].(string) +func InitialAdminPassword() (string, error) { + cfg, err := get() + if err != nil { + return "", err + } + return cfg.InitialAdminPwd, nil } +// TODO // OnlyAdminCreateProject returns the flag to restrict that only sys admin can create project -func OnlyAdminCreateProject() bool { - return uiConfig.Config["admin_create_project"].(bool) +func OnlyAdminCreateProject() (bool, error) { + cfg, err := get() + if err != nil { + return true, err + } + return cfg.ProjectCreationRestriction == comcfg.PRO_CRT_RESTR_ADM_ONLY, nil +} + +// VerifyRemoteCert returns bool value. +func VerifyRemoteCert() (bool, error) { + cfg, err := get() + if err != nil { + return true, err + } + return cfg.VerifyRemoteCert, nil +} + +func Email() (*models.Email, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Email, nil +} + +func Database() (*models.Database, error) { + cfg, err := get() + if err != nil { + return nil, err + } + return cfg.Database, nil +} + +// TODO +// UISecret returns the value of UI secret cookie, used for communication between UI and JobService +func UISecret() string { + return os.Getenv("UI_SECRET") } diff --git a/src/ui/controllers/base.go b/src/ui/controllers/base.go index 165136a1f..691880af2 100644 --- a/src/ui/controllers/base.go +++ b/src/ui/controllers/base.go @@ -103,7 +103,12 @@ func (b *BaseController) Prepare() { b.Data["CurLang"] = curLang.Name b.Data["RestLangs"] = restLangs - authMode := config.AuthMode() + authMode, err := config.AuthMode() + if err != nil { + log.Errorf("failed to get auth mode: %v", err) + b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + if authMode == "" { authMode = "db_auth" } @@ -120,9 +125,13 @@ func (b *BaseController) Prepare() { b.UseCompressedJS = false } - b.SelfRegistration = config.SelfRegistration() + b.SelfRegistration, err = config.SelfRegistration() + if err != nil { + log.Errorf("failed to get self registration: %v", err) + b.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } - b.Data["SelfRegistration"] = config.SelfRegistration() + b.Data["SelfRegistration"] = b.SelfRegistration sessionUserID := b.GetSession("userId") if sessionUserID != nil { diff --git a/src/ui/controllers/password.go b/src/ui/controllers/password.go index c3dce0801..786b010a2 100644 --- a/src/ui/controllers/password.go +++ b/src/ui/controllers/password.go @@ -6,12 +6,12 @@ import ( "regexp" "text/template" - "github.com/astaxie/beego" - "github.com/vmware/harbor/src/common/config" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" + email_util "github.com/vmware/harbor/src/common/utils/email" "github.com/vmware/harbor/src/common/utils/log" + "github.com/vmware/harbor/src/ui/config" ) type messageDetail struct { @@ -49,7 +49,11 @@ func (cc *CommonController) SendEmail() { message := new(bytes.Buffer) - harborURL := config.ExtEndpoint() + harborURL, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } if harborURL == "" { harborURL = "localhost" } @@ -65,14 +69,14 @@ func (cc *CommonController) SendEmail() { cc.CustomAbort(http.StatusInternalServerError, "internal_error") } - config, err := beego.AppConfig.GetSection("mail") + emailSettings, err := config.Email() if err != nil { - log.Errorf("Can not load app.conf: %v", err) + log.Errorf("failed to get email configurations: %v", err) cc.CustomAbort(http.StatusInternalServerError, "internal_error") } - mail := utils.Mail{ - From: config["from"], + mail := email_util.Mail{ + From: emailSettings.From, To: []string{email}, Subject: cc.Tr("reset_email_subject"), Message: message.String()} diff --git a/src/ui/controllers/project.go b/src/ui/controllers/project.go index 71c7242b0..d2e1ece59 100644 --- a/src/ui/controllers/project.go +++ b/src/ui/controllers/project.go @@ -1,6 +1,8 @@ package controllers import ( + "net/http" + "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" @@ -23,6 +25,11 @@ func (pc *ProjectController) Get() { isSysAdmin = false } } - pc.Data["CanCreate"] = !config.OnlyAdminCreateProject() || isSysAdmin + onlyAdmin, err := config.OnlyAdminCreateProject() + if err != nil { + log.Errorf("failed to determine whether only admin can create projects: %v", err) + pc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + pc.Data["CanCreate"] = !onlyAdmin || isSysAdmin pc.Forward("page_title_project", "project.htm") } diff --git a/src/ui/controllers/repository.go b/src/ui/controllers/repository.go index b85ad8782..0fddff097 100644 --- a/src/ui/controllers/repository.go +++ b/src/ui/controllers/repository.go @@ -1,6 +1,10 @@ package controllers import ( + "net/http" + "strings" + + "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/config" ) @@ -11,6 +15,11 @@ type RepositoryController struct { // Get renders repository page func (rc *RepositoryController) Get() { - rc.Data["HarborRegUrl"] = config.ExtRegistryURL() + url, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + rc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + rc.Data["HarborRegUrl"] = strings.Split(url, "://")[1] rc.Forward("page_title_repository", "repository.htm") } diff --git a/src/ui/main.go b/src/ui/main.go index 6c4427279..14770c41b 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -64,7 +64,6 @@ func updateInitPassword(userID int, password string) error { } func main() { - beego.BConfig.WebConfig.Session.SessionOn = true //TODO redisURL := os.Getenv("_REDIS_URL") @@ -72,12 +71,28 @@ func main() { beego.BConfig.WebConfig.Session.SessionProvider = "redis" beego.BConfig.WebConfig.Session.SessionProviderConfig = redisURL } - // beego.AddTemplateExt("htm") - dao.InitDatabase() + log.Info("initializing configurations...") + if err := config.Init(); err != nil { + log.Fatalf("failed to initialize configurations: %v", err) + } + log.Info("configurations initialization completed") - if err := updateInitPassword(adminUserID, config.InitialAdminPassword()); err != nil { + database, err := config.Database() + if err != nil { + log.Fatalf("failed to get database configuration: %v", err) + } + + if err := dao.InitDatabase(database); err != nil { + log.Fatalf("failed to initialize database: %v", err) + } + + password, err := config.InitialAdminPassword() + if err != nil { + log.Fatalf("failed to get admin's initia password: %v", err) + } + if err := updateInitPassword(adminUserID, password); err != nil { log.Error(err) } initRouters() diff --git a/src/ui/router.go b/src/ui/router.go index 537748b03..0bf121c2e 100644 --- a/src/ui/router.go +++ b/src/ui/router.go @@ -84,6 +84,7 @@ func initRouters() { beego.Router("/api/users/:id/sysadmin", &api.UserAPI{}, "put:ToggleUserAdminRole") beego.Router("/api/repositories/top", &api.RepositoryAPI{}, "get:GetTopRepos") beego.Router("/api/logs", &api.LogAPI{}) + beego.Router("/api/configurations", &api.ConfigAPI{}) beego.Router("/api/systeminfo/volumes", &api.SystemInfoAPI{}, "get:GetVolumeInfo") beego.Router("/api/systeminfo/getcert", &api.SystemInfoAPI{}, "get:GetCert") diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index 1e687365b..3bda33ca4 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -37,13 +37,6 @@ const ( privateKey = "/etc/ui/private_key.pem" ) -var expiration int //minutes - -func init() { - expiration = config.TokenExpiration() - log.Infof("token expiration: %d minutes", expiration) -} - // GetResourceActions ... func GetResourceActions(scopes []string) []*token.ResourceActions { log.Debugf("scopes: %+v", scopes) @@ -91,7 +84,12 @@ func FilterAccess(username string, a *token.ResourceActions) { repoLength := len(repoSplit) if repoLength > 1 { //Only check the permission when the requested image has a namespace, i.e. project var projectName string - registryURL := config.ExtRegistryURL() + registryURL, err := config.DomainName() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + return + } + registryURL = strings.Split(registryURL, "://")[1] if repoSplit[0] == registryURL { projectName = repoSplit[1] log.Infof("Detected Registry URL in Project Name. Assuming this is a notary request and setting Project Name as %s\n", projectName) @@ -153,6 +151,11 @@ func MakeToken(username, service string, access []*token.ResourceActions) (token if err != nil { return "", 0, nil, err } + expiration, err := config.TokenExpiration() + if err != nil { + return "", 0, nil, err + } + tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk) if err != nil { return "", 0, nil, err