Merge branch 'master' into job-service

.github/ISSUE_TEMPLATE

@@ -0,0 +1,5 @@
+If you are reporting a problem, please make sure the following information are provided:
+1)Version of docker engine and docker-compose
+2)Config files of harbor, you can get them by packaging "Deploy/config" directory
+3)Log files, you can get them by package the /var/log/harbor/ 
+

Deploy/config/nginx/nginx.conf

@@ -33,7 +33,10 @@ http {
       proxy_set_header Host $host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      
+      # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
       proxy_set_header X-Forwarded-Proto $scheme;
+      
       proxy_buffering off;
       proxy_request_buffering off;
     }
@@ -47,7 +50,10 @@ http {
       proxy_set_header Host $host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      
+      # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
       proxy_set_header X-Forwarded-Proto $scheme;
+      
       proxy_buffering off;
       proxy_request_buffering off;
 
@@ -58,7 +64,10 @@ http {
       proxy_set_header Host $host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      
+      # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
       proxy_set_header X-Forwarded-Proto $scheme;
+      
       proxy_buffering off;
       proxy_request_buffering off;
     }

Deploy/config/nginx/nginx.https.conf

@@ -47,7 +47,10 @@ http {
       proxy_set_header Host $http_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      
+      # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
       proxy_set_header X-Forwarded-Proto $scheme;
+
       proxy_buffering off;
       proxy_request_buffering off;
     }
@@ -61,7 +64,10 @@ http {
       proxy_set_header Host $http_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      
+      # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
       proxy_set_header X-Forwarded-Proto $scheme;
+
       proxy_buffering off;
       proxy_request_buffering off;
 
@@ -72,7 +78,10 @@ http {
       proxy_set_header Host $http_host;
       proxy_set_header X-Real-IP $remote_addr;
       proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      
+      # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
       proxy_set_header X-Forwarded-Proto $scheme;
+
       proxy_buffering off;
       proxy_request_buffering off;
     }

Deploy/harbor.cfg

@@ -9,14 +9,15 @@ hostname = reg.mydomain.com
 ui_url_protocol = http
 
 #Email account settings for sending out password resetting emails.
-email_server = smtp.mydomain.com 
+email_server = smtp.mydomain.com
 email_server_port = 25
 email_username = sample_admin@mydomain.com
 email_password = abc
 email_from = admin <sample_admin@mydomain.com>
+email_ssl = false
 
 ##The password of Harbor admin, change this before any production use.
-harbor_admin_password= Harbor12345
+harbor_admin_password = Harbor12345
 
 ##By default the auth mode is db_auth, i.e. the credentials are stored in a local database.
 #Set it to ldap_auth if you want to verify a user's credentials against an LDAP server.
@@ -33,4 +34,16 @@ db_password = root123
 
 #Turn on or off the self-registration feature
 self_registration = on
+
+#Turn on or off the customize your certicate
+customize_crt = on
+
+#fill in your certicate message
+crt_country = CN
+crt_state = State
+crt_location = CN
+crt_organization = organization
+crt_organizationalunit = organizational unit
+crt_commonname = example.com
+crt_email = example@example.com
 #####

Deploy/prepare

@@ -29,12 +29,21 @@ email_server_port = rcp.get("configuration", "email_server_port")
 email_username = rcp.get("configuration", "email_username")
 email_password = rcp.get("configuration", "email_password")
 email_from = rcp.get("configuration", "email_from")
+email_ssl = rcp.get("configuration", "email_ssl")
 harbor_admin_password = rcp.get("configuration", "harbor_admin_password")
 auth_mode = rcp.get("configuration", "auth_mode")
 ldap_url = rcp.get("configuration", "ldap_url")
 ldap_basedn = rcp.get("configuration", "ldap_basedn")
 db_password = rcp.get("configuration", "db_password")
 self_registration = rcp.get("configuration", "self_registration")
+customize_crt = rcp.get("configuration", "customize_crt")
+crt_country = rcp.get("configuration", "crt_country")
+crt_state = rcp.get("configuration", "crt_state")
+crt_location = rcp.get("configuration", "crt_location")
+crt_organization = rcp.get("configuration", "crt_organization")
+crt_organizationalunit = rcp.get("configuration", "crt_organizationalunit")
+crt_commonname = rcp.get("configuration", "crt_commonname")
+crt_email = rcp.get("configuration", "crt_email")
 ########
 
 base_dir = os.path.dirname(__file__)
@@ -62,10 +71,12 @@ registry_conf = os.path.join(config_dir, "registry", "config.yml")
 db_conf_env = os.path.join(config_dir, "db", "env")
 
 conf_files = [ ui_conf, ui_conf_env, registry_conf, db_conf_env ]
-for f in conf_files:
+def rmdir(cf):
-    if os.path.exists(f):
+    for f in cf:
-        print("Clearing the configuration file: %s" % f)
+        if os.path.exists(f):
-        os.remove(f)
+            print("Clearing the configuration file: %s" % f)
+            os.remove(f)
+rmdir(conf_files)
 
 render(os.path.join(templates_dir, "ui", "env"),
         ui_conf_env,
@@ -73,7 +84,7 @@ render(os.path.join(templates_dir, "ui", "env"),
         db_password=db_password,
         ui_url=ui_url,
         auth_mode=auth_mode,
-        admin_pwd=harbor_admin_password,
+        harbor_admin_password=harbor_admin_password,
         ldap_url=ldap_url,
         ldap_basedn=ldap_basedn,
 	self_registration=self_registration)
@@ -82,9 +93,10 @@ render(os.path.join(templates_dir, "ui", "app.conf"),
         ui_conf,
         email_server=email_server,
         email_server_port=email_server_port,
-        email_user_name=email_username,
+        email_username=email_username,
-        email_user_password=email_password,
+        email_password=email_password,
         email_from=email_from,
+        email_ssl=email_ssl,
         ui_url=ui_url)
 
 render(os.path.join(templates_dir, "registry", "config.yml"),
@@ -95,4 +107,58 @@ render(os.path.join(templates_dir, "db", "env"),
         db_conf_env,
         db_password=db_password)
 
+def validate_crt_subj(dirty_subj):
+    subj_list = [item for item in dirty_subj.strip().split("/") \
+        if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0]
+    return "/" + "/".join(subj_list)
+
+FNULL = open(os.devnull, 'w')
+
+from functools import wraps
+def stat_decorator(func):
+    #@wraps(func)
+    def check_wrapper(*args, **kwargs):
+        stat = func(*args, **kwargs)
+        message = "Generated configuration file: %s" % kwargs['path'] \
+                if stat == 0 else "Fail to generate %s" % kwargs['path']
+        print(message)
+        if stat != 0:
+            sys.exit(1)
+    return check_wrapper
+
+@stat_decorator
+def check_private_key_stat(*args, **kwargs):
+    return subprocess.call(["openssl", "genrsa", "-out", kwargs['path'], "4096"],\
+        stdout=FNULL, stderr=subprocess.STDOUT)
+
+@stat_decorator
+def check_certificate_stat(*args, **kwargs):
+    dirty_subj = "/C={0}/ST={1}/L={2}/O={3}/OU={4}/CN={5}/emailAddress={6}"\
+        .format(crt_country, crt_state, crt_location, crt_organization,\
+        crt_organizationalunit, crt_commonname, crt_email)
+    subj = validate_crt_subj(dirty_subj)
+    return subprocess.call(["openssl", "req", "-new", "-x509", "-key",\
+        private_key_pem, "-out", root_crt, "-days", "3650", "-subj", subj], \
+        stdout=FNULL, stderr=subprocess.STDOUT)
+
+def openssl_is_installed(stat):
+    if stat == 0:
+        return True
+    else:
+        print("Cannot find openssl installed in this computer\nUse default SSL certificate file")
+        return False
+
+if customize_crt == 'on':
+    import subprocess
+    shell_stat = subprocess.check_call(["which", "openssl"], stdout=FNULL, stderr=subprocess.STDOUT)
+    if openssl_is_installed(shell_stat):
+        private_key_pem = os.path.join(config_dir, "ui", "private_key.pem")
+        root_crt = os.path.join(config_dir, "registry", "root.crt")
+        crt_conf_files = [ private_key_pem, root_crt ]
+        rmdir(crt_conf_files)
+
+        check_private_key_stat(path=private_key_pem)
+        check_certificate_stat(path=root_crt)
+
+FNULL.close()
 print("The configuration files are ready, please use docker-compose to start the service.")

Deploy/templates/ui/app.conf

@@ -11,6 +11,7 @@ httpport = 80
 [mail]
 host = $email_server
 port = $email_server_port
-username = $email_user_name
+username = $email_username
-password = $email_user_password
+password = $email_password
 from = $email_from
+ssl = $email_ssl

Deploy/templates/ui/env

@@ -5,8 +5,8 @@ MYSQL_PWD=$db_password
 REGISTRY_URL=http://registry:5000
 CONFIG_PATH=/etc/ui/app.conf
 HARBOR_REG_URL=$hostname
-HARBOR_ADMIN_PASSWORD=$admin_pwd
+HARBOR_ADMIN_PASSWORD=$harbor_admin_password
-HARBOR_URL=$ui_url
+HARBOR_URL=$hostname
 AUTH_MODE=$auth_mode
 LDAP_URL=$ldap_url
 LDAP_BASE_DN=$ldap_basedn

README.md

@@ -57,8 +57,8 @@ To simplify the installation process, a pre-built installation package of Harbor
 
 For information on how to use Harbor, please see [User Guide](docs/user_guide.md) .
 
-### Deploy harbor on Kubernetes
+### Deploy Harbor on Kubernetes
-Detailed instruction about deploying harbor on Kubernetes is described [here](https://github.com/vmware/harbor/blob/master/kubernetes_deployment.md).
+Detailed instruction about deploying Harbor on Kubernetes is described [here](docs/kubernetes_deployment.md).
 
 ### Contribution
 We welcome contributions from the community. If you wish to contribute code and you have not signed our contributor license agreement (CLA), our bot will update the issue when you open a pull request. For any questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq).

api/user.go

@@ -36,6 +36,11 @@ type UserAPI struct {
 	AuthMode         string
 }
 
+type passwordReq struct {
+	OldPassword string `json:"old_password"`
+	NewPassword string `json:"new_password"`
+}
+
 // Prepare validates the URL and parms
 func (ua *UserAPI) Prepare() {
 
@@ -177,3 +182,46 @@ func (ua *UserAPI) Delete() {
 		return
 	}
 }
+
+// ChangePassword handles PUT to /api/users/{}/password
+func (ua *UserAPI) ChangePassword() {
+
+	if !(ua.AuthMode == "db_auth") {
+		ua.CustomAbort(http.StatusForbidden, "")
+	}
+
+	if !ua.IsAdmin {
+		if ua.userID != ua.currentUserID {
+			log.Error("Guests can only change their own account.")
+			ua.CustomAbort(http.StatusForbidden, "Guests can only change their own account.")
+		}
+	}
+
+	var req passwordReq
+	ua.DecodeJSONReq(&req)
+	if req.OldPassword == "" {
+		log.Error("Old password is blank")
+		ua.CustomAbort(http.StatusBadRequest, "Old password is blank")
+	}
+
+	queryUser := models.User{UserID: ua.userID, Password: req.OldPassword}
+	user, err := dao.CheckUserPassword(queryUser)
+	if err != nil {
+		log.Errorf("Error occurred in CheckUserPassword: %v", err)
+		ua.CustomAbort(http.StatusInternalServerError, "Internal error.")
+	}
+	if user == nil {
+		log.Warning("Password input is not correct")
+		ua.CustomAbort(http.StatusForbidden, "old_password_is_not_correct")
+	}
+
+	if req.NewPassword == "" {
+		ua.CustomAbort(http.StatusBadRequest, "please_input_new_password")
+	}
+	updateUser := models.User{UserID: ua.userID, Password: req.NewPassword, Salt: user.Salt}
+	err = dao.ChangeUserPassword(updateUser, req.OldPassword)
+	if err != nil {
+		log.Errorf("Error occurred in ChangeUserPassword: %v", err)
+		ua.CustomAbort(http.StatusInternalServerError, "Internal error.")
+	}
+}

controllers/password.go

@@ -46,47 +46,6 @@ func (cpc *ChangePasswordController) Get() {
 	cpc.ForwardTo("page_title_change_password", "change-password")
 }
 
-// UpdatePassword handles UI request to update user's password, it only works when the auth mode is db_auth.
-func (cc *CommonController) UpdatePassword() {
-
-	sessionUserID := cc.GetSession("userId")
-
-	if sessionUserID == nil {
-		log.Warning("User does not login.")
-		cc.CustomAbort(http.StatusUnauthorized, "please_login_first")
-	}
-
-	oldPassword := cc.GetString("old_password")
-	if oldPassword == "" {
-		log.Error("Old password is blank")
-		cc.CustomAbort(http.StatusBadRequest, "Old password is blank")
-	}
-
-	queryUser := models.User{UserID: sessionUserID.(int), Password: oldPassword}
-	user, err := dao.CheckUserPassword(queryUser)
-	if err != nil {
-		log.Errorf("Error occurred in CheckUserPassword: %v", err)
-		cc.CustomAbort(http.StatusInternalServerError, "Internal error.")
-	}
-
-	if user == nil {
-		log.Warning("Password input is not correct")
-		cc.CustomAbort(http.StatusForbidden, "old_password_is_not_correct")
-	}
-
-	password := cc.GetString("password")
-	if password != "" {
-		updateUser := models.User{UserID: sessionUserID.(int), Password: password, Salt: user.Salt}
-		err = dao.ChangeUserPassword(updateUser, oldPassword)
-		if err != nil {
-			log.Errorf("Error occurred in ChangeUserPassword: %v", err)
-			cc.CustomAbort(http.StatusInternalServerError, "Internal error.")
-		}
-	} else {
-		cc.CustomAbort(http.StatusBadRequest, "please_input_new_password")
-	}
-}
-
 // ForgotPasswordController handles request to /forgotPassword
 type ForgotPasswordController struct {
 	BaseController

dao/base.go

@@ -77,7 +77,7 @@ func InitDB() {
 		var err error
 		var c net.Conn
 		for {
-			c, err = net.Dial("tcp", addr+":"+port)
+			c, err = net.DialTimeout("tcp", addr+":"+port, 20*time.Second)
 			if err == nil {
 				c.Close()
 				ch <- 1

docs/configure_https.md

@@ -124,3 +124,4 @@ After setting up HTTPS for Harbor, you can verify it by the follow steps:
     cp yourdomain.com.crt /etc/pki/ca-trust/source/anchors/reg.yourdomain.com.crt
     update-ca-trust
     ```
+    

docs/installation_guide.md

@@ -29,12 +29,13 @@ At minimum, you need to change the **hostname** attribute in **harbor.cfg**. The
 
 **hostname**: The hostname for a user to access the user interface and the registry service. It should be the IP address or the fully qualified domain name (FQDN) of your target machine, for example 192.168.1.10 or reg.yourdomain.com . Do NOT use localhost or 127.0.0.1 for the hostname because the registry service needs to be accessed by external clients.  
 **ui_url_protocol**: The protocol for accessing the user interface and the token/notification service, by default it is http. To set up the https protocol, refer to [Configuring Harbor with HTTPS Access](configure_https.md).  
-**Email settings**: the following 5 attributes are used to send an email to reset a user's password,  they are not mandatory unless the password reset function is needed in Harbor.  
+**Email settings**: the following 6 attributes are used to send an email to reset a user's password,  they are not mandatory unless the password reset function is needed in Harbor. By default SSL connection is not enabled, if your smtp server(such as exmail.qq.com) requires SSL connection and doesn't support STARTTLS, then you should enable it by set **email_ssl = true**.
 * email_server = smtp.mydomain.com 
 * email_server_port = 25
 * email_username = sample_admin@mydomain.com
 * email_password = abc
 * email_from = admin <sample_admin@mydomain.com>  
+* email_ssl = false
 
 **harbor_admin_password**: The password for the administrator of Harbor, by default the password is Harbor12345, the user name is admin.  
 **auth_mode**: The authentication mode of Harbor. By default it is *db_auth*, i.e. the credentials are stored in a database. Please set it to *ldap_auth* if you want to verify user's credentials against an LDAP server.  
@@ -203,4 +204,10 @@ $ rm -r /data/registry
 [Docker Compose command-line reference](https://docs.docker.com/compose/reference/) describes the usage information for the docker-compose subcommands.
 
 ### Persistent data and log files
-By default, the data of database and image files in the registry are persisted in the directory **/data/** of the target machine. When Harbor's containers are removed and recreated, the data  remain unchanged. Harbor leverages rsyslog to collect the logs of each container, by default the log files are stored in the directory **/var/log/harbor/** on Harbor's host.
+By default, the data of database and image files in the registry are persisted in the directory **/data/** of the target machine. When Harbor's containers are removed and recreated, the data  remain unchanged. Harbor leverages rsyslog to collect the logs of each container, by default the log files are stored in the directory **/var/log/harbor/** on Harbor's host.  
+
+##Troubleshooting
+1.When setting up Harbor behind another nginx proxy or elastic load balancing, remove the below line if the proxy already has similar settings. Be sure to remove the line under these 3 sections: "location /", "location /v2/" and "location /service/".
+```
+proxy_set_header X-Forwarded-Proto $scheme;
+```

static/resources/js/change-password.js

@@ -56,16 +56,18 @@ jQuery(function(){
 		validateOptions.Validate(function(){
 			var oldPassword = $("#OldPassword").val();
 			var password = $("#Password").val();
-			$.ajax({
+			new AjaxUtil({
-				"url": "/updatePassword",
+				url: "/api/users/current/password",
-				"type": "post",
+				type: "put",
-				"data": {"old_password": oldPassword, "password" : password},
+				data: {"old_password": oldPassword, "new_password" : password},
-				"beforeSend": function(e){
+				beforeSend: function(e){
 				   unbindEnterKey();
 				   $("h1").append(spinner.el);
 				   $("#btnSubmit").prop("disabled", true);	
 				},
-				"success": function(data, status, xhr){
+				complete: function(xhr, status){
+					spinner.stop();
+					$("#btnSubmit").prop("disabled", false);	
 					if(xhr && xhr.status == 200){
 						$("#dlgModal")
 							.dialogModal({
@@ -77,22 +79,20 @@ jQuery(function(){
 							});
 					}
 				},
-				"error": function(jqXhr, status, error){
+				error: function(jqXhr, status, error){
-					$("#dlgModal")
+					if(jqXhr && jqXhr.responseText.length){
-						.dialogModal({
+						$("#dlgModal")
-							"title": i18n.getMessage("title_change_password"), 
+							.dialogModal({
-							"content": i18n.getMessage(jqXhr.responseText),
+								"title": i18n.getMessage("title_change_password"), 
-							"callback": function(){ 
+								"content": i18n.getMessage(jqXhr.responseText), 
-								bindEnterKey();
+								"callback": function(){ 
-								return;
+									bindEnterKey();
-							}
+									return;
-						});
+								}
-				},
+							});
-				"complete": function(){
+					}
-					spinner.stop();
-					$("#btnSubmit").prop("disabled", false);	
 				}
-			});
+			}).exec();
 		});
 	});
 });

ui/router.go

@@ -36,7 +36,6 @@ func initRouters() {
 	beego.Router("/userExists", &controllers.CommonController{}, "post:UserExists")
 	beego.Router("/reset", &controllers.CommonController{}, "post:ResetPassword")
 	beego.Router("/sendEmail", &controllers.CommonController{}, "get:SendEmail")
-	beego.Router("/updatePassword", &controllers.CommonController{}, "post:UpdatePassword")
 
 	beego.Router("/", &controllers.IndexController{})
 	beego.Router("/signIn", &controllers.SignInController{})
@@ -58,6 +57,7 @@ func initRouters() {
 	beego.Router("/api/projects/:id/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog")
 	beego.Router("/api/users", &api.UserAPI{})
 	beego.Router("/api/users/?:id", &api.UserAPI{})
+	beego.Router("/api/users/:id/password", &api.UserAPI{}, "put:ChangePassword")
 	beego.Router("/api/repositories", &api.RepositoryAPI{})
 	beego.Router("/api/repositories/tags", &api.RepositoryAPI{}, "get:GetTags")
 	beego.Router("/api/repositories/manifests", &api.RepositoryAPI{}, "get:GetManifests")

utils/mail.go

@@ -17,6 +17,8 @@ package utils
 
 import (
 	"bytes"
+	"crypto/tls"
+	"strings"
 
 	"net/smtp"
 	"text/template"
@@ -39,6 +41,7 @@ type MailConfig struct {
 	Port     string
 	Username string
 	Password string
+	TLS      bool
 }
 
 var mc MailConfig
@@ -58,10 +61,66 @@ func (m Mail) SendMail() error {
 	if err != nil {
 		return err
 	}
-	return smtp.
+	content := mailContent.Bytes()
-		SendMail(mc.Host+":"+mc.Port,
+
-		smtp.PlainAuth(mc.Identity, mc.Username, mc.Password, mc.Host),
+	auth := smtp.PlainAuth(mc.Identity, mc.Username, mc.Password, mc.Host)
-		m.From, m.To, mailContent.Bytes())
+	if mc.TLS {
+		err = sendMailWithTLS(m, auth, content)
+	} else {
+		err = sendMail(m, auth, content)
+	}
+
+	return err
+}
+
+func sendMail(m Mail, auth smtp.Auth, content []byte) error {
+	return smtp.SendMail(mc.Host+":"+mc.Port, auth, m.From, m.To, content)
+}
+
+func sendMailWithTLS(m Mail, auth smtp.Auth, content []byte) error {
+	conn, err := tls.Dial("tcp", mc.Host+":"+mc.Port, nil)
+	if err != nil {
+		return err
+	}
+
+	client, err := smtp.NewClient(conn, mc.Host)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	if ok, _ := client.Extension("AUTH"); ok {
+		if err = client.Auth(auth); err != nil {
+			return err
+		}
+	}
+
+	if err = client.Mail(m.From); err != nil {
+		return err
+	}
+
+	for _, to := range m.To {
+		if err = client.Rcpt(to); err != nil {
+			return err
+		}
+	}
+
+	w, err := client.Data()
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(content)
+	if err != nil {
+		return err
+	}
+
+	err = w.Close()
+	if err != nil {
+		return err
+	}
+
+	return client.Quit()
 }
 
 func loadConfig() {
@@ -69,11 +128,17 @@ func loadConfig() {
 	if err != nil {
 		panic(err)
 	}
+
+	var useTLS = false
+	if config["ssl"] != "" && strings.ToLower(config["ssl"]) == "true" {
+		useTLS = true
+	}
 	mc = MailConfig{
 		Identity: "Mail Config",
 		Host:     config["host"],
 		Port:     config["port"],
 		Username: config["username"],
 		Password: config["password"],
+		TLS:      useTLS,
 	}
 }