diff --git a/.travis.yml b/.travis.yml index cd7588c36..d26364069 100644 --- a/.travis.yml +++ b/.travis.yml @@ -74,8 +74,8 @@ before_script: - sudo chmod 777 /tmp/registry.db script: - - sudo mkdir -p /harbor_storage/ca_download - - sudo mv ./tests/ca.crt /harbor_storage/ca_download + - sudo mkdir -p /etc/ui/ca/ + - sudo mv ./tests/ca.crt /etc/ui/ca/ - sudo mkdir -p /harbor - sudo mv ./VERSION /harbor/VERSION - sudo service mysql stop diff --git a/make/common/templates/adminserver/env b/make/common/templates/adminserver/env index 9b268f189..190acac5a 100644 --- a/make/common/templates/adminserver/env +++ b/make/common/templates/adminserver/env @@ -33,7 +33,6 @@ UI_SECRET=$ui_secret JOBSERVICE_SECRET=$jobservice_secret TOKEN_EXPIRATION=$token_expiration CFG_EXPIRATION=5 -USE_COMPRESSED_JS=$use_compressed_js GODEBUG=netdns=cgo ADMIRAL_URL=$admiral_url WITH_NOTARY=$with_notary diff --git a/make/common/templates/nginx/nginx.http.conf b/make/common/templates/nginx/nginx.http.conf index 980d9ece2..b776f7436 100644 --- a/make/common/templates/nginx/nginx.http.conf +++ b/make/common/templates/nginx/nginx.http.conf @@ -21,6 +21,12 @@ http { server ui:80; } + log_format timed_combined '$$remote_addr - ' + '"$$request" $$status $$body_bytes_sent ' + '"$$http_referer" "$$http_user_agent" ' + '$$request_time $$upstream_response_time $$pipe'; + + access_log /dev/stdout timed_combined; server { listen 80; diff --git a/make/common/templates/nginx/nginx.https.conf b/make/common/templates/nginx/nginx.https.conf index 141961ecf..131aec466 100644 --- a/make/common/templates/nginx/nginx.https.conf +++ b/make/common/templates/nginx/nginx.https.conf @@ -21,6 +21,13 @@ http { server ui:80; } + log_format timed_combined '$$remote_addr - ' + '"$$request" $$status $$body_bytes_sent ' + '"$$http_referer" "$$http_user_agent" ' + '$$request_time $$upstream_response_time $$pipe'; + + access_log /dev/stdout timed_combined; + include /etc/nginx/conf.d/*.server.conf; server { diff --git a/make/dev/docker-compose.yml b/make/dev/docker-compose.yml index 3079e5c1b..c25a4d7c8 100644 --- a/make/dev/docker-compose.yml +++ b/make/dev/docker-compose.yml @@ -69,6 +69,7 @@ services: - ../common/config/ui/app.conf:/etc/ui/app.conf - ../common/config/ui/private_key.pem:/etc/ui/private_key.pem - /data/secretkey:/etc/ui/key + - /data/ca_download/:/etc/ui/ca/ depends_on: - log - adminserver diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl index 302c15dd1..2cd98d09b 100644 --- a/make/docker-compose.tpl +++ b/make/docker-compose.tpl @@ -76,6 +76,7 @@ services: - ./common/config/ui/app.conf:/etc/ui/app.conf - ./common/config/ui/private_key.pem:/etc/ui/private_key.pem - /data/secretkey:/etc/ui/key + - /data/ca_download/:/etc/ui/ca/ networks: - harbor depends_on: diff --git a/make/harbor.cfg b/make/harbor.cfg index 8c9575585..b476812e4 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -11,10 +11,6 @@ ui_url_protocol = http #The password for the root user of mysql db, change this before any production use. db_password = root123 -#Determine whether the UI should use compressed js files. -#For production, set it to on. For development, set it to off. -use_compressed_js = on - #Maximum number of job workers in job service max_job_workers = 3 diff --git a/make/prepare b/make/prepare index c298393d6..57da6cbbb 100755 --- a/make/prepare +++ b/make/prepare @@ -138,7 +138,6 @@ ldap_scope = rcp.get("configuration", "ldap_scope") ldap_timeout = rcp.get("configuration", "ldap_timeout") db_password = rcp.get("configuration", "db_password") self_registration = rcp.get("configuration", "self_registration") -use_compressed_js = rcp.get("configuration", "use_compressed_js") if protocol == "https": cert_path = rcp.get("configuration", "ssl_cert") cert_key_path = rcp.get("configuration", "ssl_cert_key") @@ -223,8 +222,7 @@ render(os.path.join(templates_dir, "adminserver", "env"), jobservice_secret=jobservice_secret, token_expiration=token_expiration, admiral_url=admiral_url, - with_notary=args.notary_mode, - use_compressed_js=use_compressed_js + with_notary=args.notary_mode ) render(os.path.join(templates_dir, "ui", "env"), diff --git a/src/adminserver/systemcfg/systemcfg.go b/src/adminserver/systemcfg/systemcfg.go index 577221a81..1cef91005 100644 --- a/src/adminserver/systemcfg/systemcfg.go +++ b/src/adminserver/systemcfg/systemcfg.go @@ -98,10 +98,6 @@ var ( env: "TOKEN_EXPIRATION", parse: parseStringToInt, }, - common.UseCompressedJS: &parser{ - env: "USE_COMPRESSED_JS", - parse: parseStringToBool, - }, common.CfgExpiration: &parser{ env: "CFG_EXPIRATION", parse: parseStringToInt, @@ -132,11 +128,6 @@ var ( env: "MAX_JOB_WORKERS", parse: parseStringToInt, }, - // TODO remove this config? - common.UseCompressedJS: &parser{ - env: "USE_COMPRESSED_JS", - parse: parseStringToBool, - }, common.CfgExpiration: &parser{ env: "CFG_EXPIRATION", parse: parseStringToInt, diff --git a/src/common/const.go b/src/common/const.go index 6c18fc58d..204b4f398 100644 --- a/src/common/const.go +++ b/src/common/const.go @@ -58,7 +58,6 @@ const ( TokenExpiration = "token_expiration" CfgExpiration = "cfg_expiration" JobLogDir = "job_log_dir" - UseCompressedJS = "use_compressed_js" AdminInitialPassword = "admin_initial_password" AdmiralEndpoint = "admiral_url" WithNotary = "with_notary" diff --git a/src/common/models/config.go b/src/common/models/config.go index 16f518d2b..491f1582f 100644 --- a/src/common/models/config.go +++ b/src/common/models/config.go @@ -92,7 +92,6 @@ type SystemCfg struct { MaxJobWorkers int `json:"max_job_workers"` JobLogDir string `json:"job_log_dir"` InitialAdminPwd string `json:"initial_admin_pwd,omitempty"` - CompressJS bool `json:"compress_js"` //TODO remove TokenExpiration int `json:"token_expiration"` // in minute SecretKey string `json:"secret_key,omitempty"` CfgExpiration int `json:"cfg_expiration"` diff --git a/src/common/utils/email/mail.go b/src/common/utils/email/mail.go index 8df4eec96..41ada91d4 100644 --- a/src/common/utils/email/mail.go +++ b/src/common/utils/email/mail.go @@ -16,87 +16,34 @@ package email import ( - "bytes" tlspkg "crypto/tls" + "fmt" "net" - "strconv" + "net/smtp" + "strings" "time" - "net/smtp" - "text/template" - - "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils/log" - "github.com/vmware/harbor/src/ui/config" ) -// Mail holds information about content of Email -type Mail struct { - From string - To []string - Subject string - Message string -} +// Send ... +func Send(addr, identity, username, password string, + timeout int, tls, insecure bool, from string, + to []string, subject, message string) error { -var mc *models.Email - -// SendMail sends Email according to the configurations -func (m Mail) SendMail() error { - var err error - mc, err = config.Email() - if err != nil { - return err - } - - mailTemplate, err := template.ParseFiles("views/mail.tpl") - if err != nil { - return err - } - mailContent := new(bytes.Buffer) - err = mailTemplate.Execute(mailContent, m) - if err != nil { - return err - } - content := mailContent.Bytes() - - auth := smtp.PlainAuth(mc.Identity, mc.Username, mc.Password, mc.Host) - if mc.SSL { - err = sendMailWithTLS(m, auth, content) - } else { - err = sendMail(m, auth, content) - } - - return err -} - -func sendMail(m Mail, auth smtp.Auth, content []byte) error { - return smtp.SendMail(mc.Host+":"+strconv.Itoa(mc.Port), auth, m.From, m.To, content) -} - -func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error { - conn, err := tlspkg.Dial("tcp", mc.Host+":"+strconv.Itoa(mc.Port), nil) - if err != nil { - return err - } - - client, err := smtp.NewClient(conn, mc.Host) + client, err := newClient(addr, identity, username, + password, timeout, tls, insecure) if err != nil { return err } defer client.Close() - if ok, _ := client.Extension("AUTH"); ok { - if err = client.Auth(auth); err != nil { - return err - } - } - - if err = client.Mail(m.From); err != nil { + if err = client.Mail(from); err != nil { return err } - for _, to := range m.To { - if err = client.Rcpt(to); err != nil { + for _, t := range to { + if err = client.Rcpt(t); err != nil { return err } } @@ -106,7 +53,11 @@ func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error { return err } - _, err = w.Write(content) + template := "From: %s\r\nTo: %s\r\nSubject: %s\r\nMIME-version: 1.0;\r\nContent-Type: text/html; charset=\"UTF-8\"\r\n\n%s\r\n" + data := fmt.Sprintf(template, from, + strings.Join(to, ","), subject, message) + + _, err = w.Write([]byte(data)) if err != nil { return err } @@ -126,18 +77,29 @@ func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error { // Ping doesn't verify the server's certificate and hostname when // needed if the parameter insecure is ture func Ping(addr, identity, username, password string, - timeout int, tls, insecure bool) (err error) { + timeout int, tls, insecure bool) error { + client, err := newClient(addr, identity, username, password, + timeout, tls, insecure) + if err != nil { + return err + } + defer client.Close() + return nil +} + +// caller needs to close the client +func newClient(addr, identity, username, password string, + timeout int, tls, insecure bool) (*smtp.Client, error) { log.Debugf("establishing TCP connection with %s ...", addr) conn, err := net.DialTimeout("tcp", addr, time.Duration(timeout)*time.Second) if err != nil { - return + return nil, err } - defer conn.Close() host, _, err := net.SplitHostPort(addr) if err != nil { - return + return nil, err } if tls { @@ -147,9 +109,8 @@ func Ping(addr, identity, username, password string, InsecureSkipVerify: insecure, }) if err = tlsConn.Handshake(); err != nil { - return + return nil, err } - defer tlsConn.Close() conn = tlsConn } @@ -157,9 +118,8 @@ func Ping(addr, identity, username, password string, log.Debugf("creating SMTP client for %s ...", host) client, err := smtp.NewClient(conn, host) if err != nil { - return + return nil, err } - defer client.Close() //try to swith to SSL/TLS if !tls { @@ -169,7 +129,7 @@ func Ping(addr, identity, username, password string, ServerName: host, InsecureSkipVerify: insecure, }); err != nil { - return + return nil, err } } else { log.Debugf("the email server %s does not support STARTTLS", addr) @@ -181,14 +141,14 @@ func Ping(addr, identity, username, password string, // only support plain auth if err = client.Auth(smtp.PlainAuth(identity, username, password, host)); err != nil { - return + return nil, err } } else { log.Debugf("the email server %s does not support AUTH, skip", addr) } - log.Debug("ping email server successfully") + log.Debug("create smtp client successfully") - return + return client, nil } diff --git a/src/common/utils/email/mail_test.go b/src/common/utils/email/mail_test.go index 3ece8d748..7f5e7d183 100644 --- a/src/common/utils/email/mail_test.go +++ b/src/common/utils/email/mail_test.go @@ -18,20 +18,81 @@ package email import ( "strings" "testing" + + "github.com/stretchr/testify/assert" ) +func TestSend(t *testing.T) { + addr := "smtp.gmail.com:465" + identity := "" + username := "harbortestonly@gmail.com" + password := "harborharbor" + timeout := 60 + tls := true + insecure := false + from := "from" + to := []string{username} + subject := "subject" + message := "message" + + // tls connection + tls = true + err := Send(addr, identity, username, password, + timeout, tls, insecure, from, to, + subject, message) + assert.Nil(t, err) + + /*not work on travis + // non-tls connection + addr = "smtp.gmail.com:25" + tls = false + err = Send(addr, identity, username, password, + timeout, tls, insecure, from, to, + subject, message) + assert.Nil(t, err) + */ + + //invalid username/password + username = "invalid_username" + err = Send(addr, identity, username, password, + timeout, tls, insecure, from, to, + subject, message) + if err == nil { + t.Errorf("there should be an auth error") + } else { + if !strings.Contains(err.Error(), "535") { + t.Errorf("unexpected error: %v", err) + } + } +} + func TestPing(t *testing.T) { addr := "smtp.gmail.com:465" identity := "" - username := "wrong_username" - password := "wrong_password" - timeout := 60 + username := "harbortestonly@gmail.com" + password := "harborharbor" + timeout := 0 tls := true insecure := false - // test secure connection + // tls connection err := Ping(addr, identity, username, password, timeout, tls, insecure) + assert.Nil(t, err) + + /*not work on travis + // non-tls connection + addr = "smtp.gmail.com:25" + tls = false + err = Ping(addr, identity, username, password, + timeout, tls, insecure) + assert.Nil(t, err) + */ + + //invalid username/password + username = "invalid_username" + err = Ping(addr, identity, username, password, + timeout, tls, insecure) if err == nil { t.Errorf("there should be an auth error") } else { diff --git a/src/common/utils/ldap/ldap_test.go b/src/common/utils/ldap/ldap_test.go index 60cf7c34a..a5767a005 100644 --- a/src/common/utils/ldap/ldap_test.go +++ b/src/common/utils/ldap/ldap_test.go @@ -63,7 +63,6 @@ var adminServerLdapTestConfig = map[string]interface{}{ // config.TokenExpiration: 30, common.CfgExpiration: 5, // config.JobLogDir: "/var/log/jobs", - // config.UseCompressedJS: true, common.AdminInitialPassword: "password", } diff --git a/src/common/utils/test/adminserver.go b/src/common/utils/test/adminserver.go index 0a0741d1d..af2b96e70 100644 --- a/src/common/utils/test/adminserver.go +++ b/src/common/utils/test/adminserver.go @@ -57,7 +57,6 @@ var adminServerDefaultConfig = map[string]interface{}{ common.MaxJobWorkers: 3, common.TokenExpiration: 30, common.CfgExpiration: 5, - common.UseCompressedJS: true, common.AdminInitialPassword: "password", common.AdmiralEndpoint: "http://www.vmware.com", common.WithNotary: false, diff --git a/src/ui/api/config.go b/src/ui/api/config.go index 22ccea112..be9a8cb09 100644 --- a/src/ui/api/config.go +++ b/src/ui/api/config.go @@ -63,7 +63,6 @@ var ( common.TokenExpiration, common.CfgExpiration, common.JobLogDir, - common.UseCompressedJS, common.AdminInitialPassword, } @@ -81,7 +80,6 @@ var ( common.EmailSSL, common.SelfRegistration, common.VerifyRemoteCert, - common.UseCompressedJS, } passwordKeys = []string{ diff --git a/src/ui/api/systeminfo.go b/src/ui/api/systeminfo.go index 88dff92bc..0d8f7cb46 100644 --- a/src/ui/api/systeminfo.go +++ b/src/ui/api/systeminfo.go @@ -20,7 +20,7 @@ type SystemInfoAPI struct { isAdmin bool } -const defaultRootCert = "/harbor_storage/ca_download/ca.crt" +const defaultRootCert = "/etc/ui/ca/ca.crt" const harborVersionFile = "/harbor/VERSION" //SystemInfo models for system info. diff --git a/src/ui/auth/ldap/ldap_test.go b/src/ui/auth/ldap/ldap_test.go index 9b001ae55..e14227819 100644 --- a/src/ui/auth/ldap/ldap_test.go +++ b/src/ui/auth/ldap/ldap_test.go @@ -48,7 +48,6 @@ var adminServerLdapTestConfig = map[string]interface{}{ // config.TokenExpiration: 30, common.CfgExpiration: 5, // config.JobLogDir: "/var/log/jobs", - // config.UseCompressedJS: true, common.AdminInitialPassword: "password", } diff --git a/src/ui/controllers/base.go b/src/ui/controllers/base.go index 4e8ddc9e9..cfde32b41 100644 --- a/src/ui/controllers/base.go +++ b/src/ui/controllers/base.go @@ -3,19 +3,21 @@ package controllers import ( "bytes" "html/template" + "net" "net/http" "os" "regexp" + "strconv" "github.com/astaxie/beego" "github.com/beego/i18n" - "github.com/vmware/harbor/src/common" "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" "github.com/vmware/harbor/src/common/utils" - "github.com/vmware/harbor/src/common/utils/email" + email_util "github.com/vmware/harbor/src/common/utils/email" "github.com/vmware/harbor/src/common/utils/log" "github.com/vmware/harbor/src/ui/auth" + "github.com/vmware/harbor/src/ui/config" ) // CommonController handles request from UI that doesn't expect a page, such as /SwitchLanguage /logout ... @@ -87,72 +89,81 @@ func (cc *CommonController) UserExists() { // SendEmail verifies the Email address and contact SMTP server to send reset password Email. func (cc *CommonController) SendEmail() { - emailStr := cc.GetString("email") - - pass, _ := regexp.MatchString(`^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$`, emailStr) - - if !pass { - cc.CustomAbort(http.StatusBadRequest, "email_content_illegal") - } else { - - queryUser := models.User{Email: emailStr} - exist, err := dao.UserExists(queryUser, "email") - if err != nil { - log.Errorf("Error occurred in UserExists: %v", err) - cc.CustomAbort(http.StatusInternalServerError, "Internal error.") - } - if !exist { - cc.CustomAbort(http.StatusNotFound, "email_does_not_exist") - } - - messageTemplate, err := template.ParseFiles("views/reset-password-mail.tpl") - if err != nil { - log.Errorf("Parse email template file failed: %v", err) - cc.CustomAbort(http.StatusInternalServerError, err.Error()) - } - - message := new(bytes.Buffer) - - harborURL := common.ExtEndpoint - if harborURL == "" { - harborURL = "localhost" - } - uuid := utils.GenerateRandomString() - err = messageTemplate.Execute(message, messageDetail{ - Hint: cc.Tr("reset_email_hint"), - URL: harborURL, - UUID: uuid, - }) - - if err != nil { - log.Errorf("Message template error: %v", err) - cc.CustomAbort(http.StatusInternalServerError, "internal_error") - } - - config, err := beego.AppConfig.GetSection("mail") - if err != nil { - log.Errorf("Can not load app.conf: %v", err) - cc.CustomAbort(http.StatusInternalServerError, "internal_error") - } - - mail := email.Mail{ - From: config["from"], - To: []string{emailStr}, - Subject: cc.Tr("reset_email_subject"), - Message: message.String()} - - err = mail.SendMail() - - if err != nil { - log.Errorf("Send email failed: %v", err) - cc.CustomAbort(http.StatusInternalServerError, "send_email_failed") - } - - user := models.User{ResetUUID: uuid, Email: emailStr} - dao.UpdateUserResetUUID(user) + email := cc.GetString("email") + valid, err := regexp.MatchString(`^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$`, email) + if err != nil { + log.Errorf("failed to match regexp: %v", err) + cc.CustomAbort(http.StatusInternalServerError, "Internal error.") } + if !valid { + cc.CustomAbort(http.StatusBadRequest, "invalid email") + } + + queryUser := models.User{Email: email} + exist, err := dao.UserExists(queryUser, "email") + if err != nil { + log.Errorf("Error occurred in UserExists: %v", err) + cc.CustomAbort(http.StatusInternalServerError, "Internal error.") + } + if !exist { + log.Debugf("email %s not found", email) + cc.CustomAbort(http.StatusNotFound, "email_does_not_exist") + } + + uuid := utils.GenerateRandomString() + user := models.User{ResetUUID: uuid, Email: email} + if err = dao.UpdateUserResetUUID(user); err != nil { + log.Errorf("failed to update user reset UUID: %v", err) + cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + messageTemplate, err := template.ParseFiles("views/reset-password-mail.tpl") + if err != nil { + log.Errorf("Parse email template file failed: %v", err) + cc.CustomAbort(http.StatusInternalServerError, err.Error()) + } + + message := new(bytes.Buffer) + + harborURL, err := config.ExtEndpoint() + if err != nil { + log.Errorf("failed to get domain name: %v", err) + cc.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + err = messageTemplate.Execute(message, messageDetail{ + Hint: cc.Tr("reset_email_hint"), + URL: harborURL, + UUID: uuid, + }) + + if err != nil { + log.Errorf("Message template error: %v", err) + cc.CustomAbort(http.StatusInternalServerError, "internal_error") + } + + settings, err := config.Email() + if err != nil { + log.Errorf("failed to get email configurations: %v", err) + cc.CustomAbort(http.StatusInternalServerError, "internal_error") + } + + addr := net.JoinHostPort(settings.Host, strconv.Itoa(settings.Port)) + err = email_util.Send(addr, + settings.Identity, + settings.Username, + settings.Password, + 60, settings.SSL, + false, settings.From, + []string{email}, + cc.Tr("reset_email_subject"), + message.String()) + if err != nil { + log.Errorf("Send email failed: %v", err) + cc.CustomAbort(http.StatusInternalServerError, "send_email_failed") + } } // ResetPassword handles request from the reset page and reset password diff --git a/src/ui_ng/src/app/account/account-settings/account-settings-modal.component.html b/src/ui_ng/src/app/account/account-settings/account-settings-modal.component.html index 1bc7df843..9809cffd8 100644 --- a/src/ui_ng/src/app/account/account-settings/account-settings-modal.component.html +++ b/src/ui_ng/src/app/account/account-settings/account-settings-modal.component.html @@ -13,7 +13,8 @@