Merge remote-tracking branch 'upstream/job-service' into job-service

Deploy/log/Dockerfile

@@ -2,15 +2,9 @@ FROM library/ubuntu:14.04
 
 # run logrotate hourly, disable imklog model, provides TCP/UDP syslog reception
 RUN mv /etc/cron.daily/logrotate /etc/cron.hourly/ \
-	&& sed 's/$ModLoad imklog/#$ModLoad imklog/' -i /etc/rsyslog.conf \
+	&& rm /etc/rsyslog.d/* \
-	&& sed 's/$KLogPermitNonKernelFacility on/#$KLogPermitNonKernelFacility on/' -i /etc/rsyslog.conf \
+        && rm /etc/rsyslog.conf
-	&& sed 's/#$ModLoad imudp/$ModLoad imudp/' -i /etc/rsyslog.conf \
+ADD rsyslog.conf /etc/rsyslog.conf
-	&& sed 's/#$UDPServerRun 514/$UDPServerRun 514/' -i /etc/rsyslog.conf \
-	&& sed 's/#$ModLoad imtcp/$ModLoad imtcp/' -i /etc/rsyslog.conf \
-	&& sed 's/#$InputTCPServerRun 514/$InputTCPServerRun 514/' -i /etc/rsyslog.conf \
-	&& sed 's/$PrivDropToUser syslog/#$PrivDropToUser syslog/' -i /etc/rsyslog.conf \
-	&& sed 's/$PrivDropToGroup syslog/#$PrivDropToGroup syslog/' -i /etc/rsyslog.conf \
-	&& rm /etc/rsyslog.d/*
 
 # logrotate configuration file for docker
 ADD logrotate_docker.conf /etc/logrotate.d/
@@ -23,4 +17,3 @@ VOLUME /var/log/docker/
 EXPOSE 514
 
 CMD cron && rsyslogd -n
-

Deploy/log/rsyslog.conf

@@ -0,0 +1,60 @@
+#  /etc/rsyslog.conf	Configuration file for rsyslog.
+#
+#			For more information see
+#			/usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
+#
+#  Default logging rules can be found in /etc/rsyslog.d/50-default.conf
+
+
+#################
+#### MODULES ####
+#################
+
+$ModLoad imuxsock # provides support for local system logging
+#$ModLoad imklog   # provides kernel logging support
+#$ModLoad immark  # provides --MARK-- message capability
+
+# provides UDP syslog reception
+$ModLoad imudp
+$UDPServerRun 514
+
+# provides TCP syslog reception
+$ModLoad imtcp
+$InputTCPServerRun 514
+             
+# Enable non-kernel facility klog messages
+#$KLogPermitNonKernelFacility on
+
+###########################
+#### GLOBAL DIRECTIVES ####
+###########################
+
+#
+# Use traditional timestamp format.
+# To enable high precision timestamps, comment out the following line.
+#
+$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+# Filter duplicated messages
+$RepeatedMsgReduction on
+
+#
+# Set the default permissions for all log files.
+#
+$FileOwner syslog
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+#$PrivDropToUser syslog
+#$PrivDropToGroup syslog
+
+#
+# Where to place spool and state files
+#
+$WorkDirectory /var/spool/rsyslog
+
+#
+# Include all config files in /etc/rsyslog.d/
+#
+$IncludeConfig /etc/rsyslog.d/*.conf

api/member.go

@@ -114,23 +114,10 @@ func (pma *ProjectMemberAPI) Get() {
 
 // Post ...
 func (pma *ProjectMemberAPI) Post() {
-	pid := pma.project.ProjectID
+	currentUserID := pma.currentUserID
-
+	projectID := pma.project.ProjectID
-	//userQuery := models.User{UserID: pma.currentUserID, RoleID: models.PROJECTADMIN}
+	if !hasProjectAdminRole(currentUserID, projectID) {
-	rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid)
+		log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, projectID)
-	if err != nil {
-		log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
-		pma.CustomAbort(http.StatusInternalServerError, "Internal error.")
-	}
-	hasProjectAdminRole := false
-	for _, role := range rolelist {
-		if role.RoleID == models.PROJECTADMIN {
-			hasProjectAdminRole = true
-			break
-		}
-	}
-	if !hasProjectAdminRole {
-		log.Warningf("Current user, id: %d does not have project admin role for project, id:", pma.currentUserID, pid)
 		pma.RenderError(http.StatusForbidden, "")
 		return
 	}
@@ -144,21 +131,21 @@ func (pma *ProjectMemberAPI) Post() {
 		pma.RenderError(http.StatusNotFound, "User does not exist")
 		return
 	}
-	rolelist, err = dao.GetUserProjectRoles(userID, pid)
+	rolelist, err := dao.GetUserProjectRoles(userID, projectID)
 	if err != nil {
 		log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
 		pma.CustomAbort(http.StatusInternalServerError, "Internal error.")
 	}
 	if len(rolelist) > 0 {
-		log.Warningf("user is already added to project, user id: %d, project id: %d", userID, pid)
+		log.Warningf("user is already added to project, user id: %d, project id: %d", userID, projectID)
 		pma.RenderError(http.StatusConflict, "user is ready in project")
 		return
 	}
 
 	for _, rid := range req.Roles {
-		err = dao.AddProjectMember(pid, userID, int(rid))
+		err = dao.AddProjectMember(projectID, userID, int(rid))
 		if err != nil {
-			log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", pid, userID, rid)
+			log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", projectID, userID, rid)
 			pma.RenderError(http.StatusInternalServerError, "Failed to update data in database")
 			return
 		}
@@ -167,27 +154,16 @@ func (pma *ProjectMemberAPI) Post() {
 
 // Put ...
 func (pma *ProjectMemberAPI) Put() {
+	currentUserID := pma.currentUserID
 	pid := pma.project.ProjectID
-	mid := pma.memberID
+	if !hasProjectAdminRole(currentUserID, pid) {
-
+		log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid)
-	rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid)
-	if err != nil {
-		log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
-		pma.CustomAbort(http.StatusInternalServerError, "Internal error.")
-	}
-	hasProjectAdminRole := false
-	for _, role := range rolelist {
-		if role.RoleID == models.PROJECTADMIN {
-			hasProjectAdminRole = true
-			break
-		}
-	}
-
-	if !hasProjectAdminRole {
-		log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid)
 		pma.RenderError(http.StatusForbidden, "")
 		return
 	}
+
+	mid := pma.memberID
+
 	var req memberReq
 	pma.DecodeJSONReq(&req)
 	roleList, err := dao.GetUserProjectRoles(mid, pid)
@@ -217,51 +193,20 @@ func (pma *ProjectMemberAPI) Put() {
 
 // Delete ...
 func (pma *ProjectMemberAPI) Delete() {
+	currentUserID := pma.currentUserID
 	pid := pma.project.ProjectID
-	mid := pma.memberID
+	if !hasProjectAdminRole(currentUserID, pid) {
-
+		log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid)
-	rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid)
-	hasProjectAdminRole := false
-	for _, role := range rolelist {
-		if role.RoleID == models.PROJECTADMIN {
-			hasProjectAdminRole = true
-			break
-		}
-	}
-
-	if !hasProjectAdminRole {
-		log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid)
 		pma.RenderError(http.StatusForbidden, "")
 		return
 	}
-	err = dao.DeleteProjectMember(pid, mid)
+
+	mid := pma.memberID
+
+	err := dao.DeleteProjectMember(pid, mid)
 	if err != nil {
 		log.Errorf("Failed to delete project roles for user, user id: %d, project id: %d, error: %v", mid, pid, err)
 		pma.RenderError(http.StatusInternalServerError, "Failed to update data in DB")
 		return
 	}
 }
-
-//sysadmin has all privileges to all projects
-func listRoles(userID int, projectID int64) ([]models.Role, error) {
-	roles := make([]models.Role, 1)
-	isSysAdmin, err := dao.IsAdminRole(userID)
-	if err != nil {
-		return roles, err
-	}
-	if isSysAdmin {
-		role, err := dao.GetRoleByID(models.PROJECTADMIN)
-		if err != nil {
-			return roles, err
-		}
-		roles = append(roles, *role)
-		return roles, nil
-	}
-
-	rs, err := dao.GetUserProjectRoles(userID, projectID)
-	if err != nil {
-		return roles, err
-	}
-	roles = append(roles, rs...)
-	return roles, nil
-}

api/project.go

@@ -43,7 +43,6 @@ const projectNameMaxLen int = 30
 
 // Prepare validates the URL and the user
 func (p *ProjectAPI) Prepare() {
-	p.userID = p.ValidateUser()
 	idStr := p.Ctx.Input.Param(":id")
 	if len(idStr) > 0 {
 		var err error
@@ -65,6 +64,8 @@ func (p *ProjectAPI) Prepare() {
 
 // Post ...
 func (p *ProjectAPI) Post() {
+	p.userID = p.ValidateUser()
+
 	var req projectReq
 	var public int
 	p.DecodeJSONReq(&req)
@@ -99,20 +100,52 @@ func (p *ProjectAPI) Post() {
 // Head ...
 func (p *ProjectAPI) Head() {
 	projectName := p.GetString("project_name")
-	result, err := dao.ProjectExists(projectName)
+	if len(projectName) == 0 {
+		p.CustomAbort(http.StatusBadRequest, "project_name is needed")
+	}
+
+	project, err := dao.GetProjectByName(projectName)
 	if err != nil {
-		log.Errorf("Error while communicating with DB, error: %v", err)
+		log.Errorf("error occurred in GetProjectByName: %v", err)
-		p.RenderError(http.StatusInternalServerError, "Error while communicating with DB")
+		p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
+	}
+
+	// only public project can be Headed by user without login
+	if project != nil && project.Public == 1 {
 		return
 	}
-	if !result {
+
-		p.RenderError(http.StatusNotFound, "")
+	userID := p.ValidateUser()
-		return
+	if project == nil {
+		p.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
+	}
+
+	if !checkProjectPermission(userID, project.ProjectID) {
+		p.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden))
 	}
 }
 
 // Get ...
 func (p *ProjectAPI) Get() {
+	project, err := dao.GetProjectByID(p.projectID)
+	if err != nil {
+		log.Errorf("failed to get project %d: %v", p.projectID, err)
+		p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
+	}
+
+	if project.Public == 0 {
+		userID := p.ValidateUser()
+		if !checkProjectPermission(userID, p.projectID) {
+			p.CustomAbort(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized))
+		}
+	}
+
+	p.Data["json"] = project
+	p.ServeJSON()
+}
+
+// List ...
+func (p *ProjectAPI) List() {
 	var projectList []models.Project
 	projectName := p.GetString("project_name")
 	if len(projectName) > 0 {
@@ -132,6 +165,8 @@ func (p *ProjectAPI) Get() {
 	if public == 1 {
 		projectList, err = dao.GetPublicProjects(projectName)
 	} else {
+		//if the request is not for public projects, user must login or provide credential
+		p.userID = p.ValidateUser()
 		isAdmin, err = dao.IsAdminRole(p.userID)
 		if err != nil {
 			log.Errorf("Error occured in check admin, error: %v", err)
@@ -164,6 +199,8 @@ func (p *ProjectAPI) Get() {
 
 // Put ...
 func (p *ProjectAPI) Put() {
+	p.userID = p.ValidateUser()
+
 	var req projectReq
 	var public int
 
@@ -192,6 +229,7 @@ func (p *ProjectAPI) Put() {
 
 // FilterAccessLog handles GET to /api/projects/{}/logs
 func (p *ProjectAPI) FilterAccessLog() {
+	p.userID = p.ValidateUser()
 
 	var filter models.AccessLog
 	p.DecodeJSONReq(&filter)

api/replication_job.go

@@ -1,9 +1,12 @@
 package api
 
 import (
+	"io/ioutil"
+	"net/http"
+	"strconv"
+
 	"github.com/vmware/harbor/dao"
 	"github.com/vmware/harbor/utils/log"
-	"net/http"
 )
 
 type RepJobAPI struct {
@@ -19,6 +22,7 @@ func (ja *RepJobAPI) Prepare() {
 	if !isAdmin {
 		ja.CustomAbort(http.StatusForbidden, "")
 	}
+
 }
 
 func (ja *RepJobAPI) Get() {
@@ -38,4 +42,38 @@ func (ja *RepJobAPI) Get() {
 	ja.ServeJSON()
 }
 
+// GetLog ...
+func (ja *RepJobAPI) GetLog() {
+	id := ja.Ctx.Input.Param(":id")
+	if len(id) == 0 {
+		ja.CustomAbort(http.StatusBadRequest, "id is nil")
+	}
+
+	resp, err := http.Get(buildJobLogURL(id))
+	if err != nil {
+		log.Errorf("failed to get log for job %s: %v", id, err)
+		ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
+	}
+
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Errorf("failed to read response body for job %s: %v", id, err)
+		ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Disposition"), "attachment; filename=replication_job.log")
+		ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), resp.Header.Get(http.CanonicalHeaderKey("Content-Type")))
+		ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(b)))
+		if _, err = ja.Ctx.ResponseWriter.Write(b); err != nil {
+			log.Errorf("failed to write log to response; %v", err)
+			ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
+		}
+		return
+	}
+
+	ja.CustomAbort(resp.StatusCode, string(b))
+}
+
 //TODO:add Post handler to call job service API to submit jobs by policy

api/replication_policy.go

@@ -2,16 +2,19 @@ package api
 
 import (
 	"fmt"
+
+	"net/http"
+	"strconv"
+
 	"github.com/vmware/harbor/dao"
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/utils/log"
-	"net/http"
-	"strconv"
 )
 
 type RepPolicyAPI struct {
 	BaseAPI
 	policyID int64
+	policy   *models.RepPolicy
 }
 
 func (pa *RepPolicyAPI) Prepare() {
@@ -39,6 +42,7 @@ func (pa *RepPolicyAPI) Prepare() {
 		if p == nil {
 			pa.CustomAbort(http.StatusNotFound, fmt.Sprintf("policy does not exist, id: %v", pa.policyID))
 		}
+		pa.policy = p
 	}
 }
 
@@ -70,6 +74,17 @@ func (pa *RepPolicyAPI) Post() {
 		pa.RenderError(http.StatusInternalServerError, "Internal Error")
 		return
 	}
+
+	if policy.Enabled == 1 {
+		go func() {
+			if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil {
+				log.Errorf("failed to trigger replication of %d: %v", pid, err)
+			} else {
+				log.Infof("replication of %d triggered", pid)
+			}
+		}()
+	}
+
 	pa.Redirect(http.StatusCreated, strconv.FormatInt(pid, 10))
 }
 
@@ -79,19 +94,29 @@ type enablementReq struct {
 
 func (pa *RepPolicyAPI) UpdateEnablement() {
 	e := enablementReq{}
-	var err error
 	pa.DecodeJSONReq(&e)
-	if e.Enabled == 1 {
+	if e.Enabled != 0 && e.Enabled != 1 {
-		err = dao.EnableRepPolicy(pa.policyID)
-	} else if e.Enabled == 0 {
-		err = dao.DisableRepPolicy(pa.policyID)
-	} else {
 		pa.RenderError(http.StatusBadRequest, "invalid enabled value")
 		return
 	}
-	if err != nil {
+
+	if pa.policy.Enabled == e.Enabled {
+		return
+	}
+
+	if err := dao.UpdateRepPolicyEnablement(pa.policyID, e.Enabled); err != nil {
 		log.Errorf("Failed to update policy enablement in DB, error: %v", err)
 		pa.RenderError(http.StatusInternalServerError, "Internal Error")
 		return
 	}
+
+	if e.Enabled == 1 {
+		go func() {
+			if err := TriggerReplication(pa.policyID, "", nil, models.RepOpTransfer); err != nil {
+				log.Errorf("failed to trigger replication of %d: %v", pa.policyID, err)
+			} else {
+				log.Infof("replication of %d triggered", pa.policyID)
+			}
+		}()
+	}
 }

api/repository.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"net/http"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -30,7 +31,10 @@ import (
 	svc_utils "github.com/vmware/harbor/service/utils"
 	"github.com/vmware/harbor/utils/log"
 	"github.com/vmware/harbor/utils/registry"
+
 	registry_error "github.com/vmware/harbor/utils/registry/error"
+
+	"github.com/vmware/harbor/utils/registry/auth"
 )
 
 // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put
@@ -39,23 +43,13 @@ import (
 // the security of registry
 type RepositoryAPI struct {
 	BaseAPI
-	userID int
-}
-
-// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission.
-func (ra *RepositoryAPI) Prepare() {
-	if svc_utils.VerifySecret(ra.Ctx.Request) {
-		ra.userID = 1
-	} else {
-		ra.userID = ra.ValidateUser()
-	}
 }
 
 // Get ...
 func (ra *RepositoryAPI) Get() {
-	projectID, err0 := ra.GetInt64("project_id")
+	projectID, err := ra.GetInt64("project_id")
-	if err0 != nil {
+	if err != nil {
-		log.Errorf("Failed to get project id, error: %v", err0)
+		log.Errorf("Failed to get project id, error: %v", err)
 		ra.RenderError(http.StatusBadRequest, "Invalid project id")
 		return
 	}
@@ -69,9 +63,20 @@ func (ra *RepositoryAPI) Get() {
 		ra.RenderError(http.StatusNotFound, "")
 		return
 	}
-	if p.Public == 0 && !checkProjectPermission(ra.userID, projectID) {
+
-		ra.RenderError(http.StatusForbidden, "")
+	if p.Public == 0 {
-		return
+		var userID int
+
+		if svc_utils.VerifySecret(ra.Ctx.Request) {
+			userID = 1
+		} else {
+			userID = ra.ValidateUser()
+		}
+
+		if !checkProjectPermission(userID, projectID) {
+			ra.RenderError(http.StatusForbidden, "")
+			return
+		}
 	}
 
 	repoList, err := cache.GetRepoFromCache()
@@ -110,7 +115,7 @@ func (ra *RepositoryAPI) Delete() {
 		ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
 	}
 
-	rc, err := ra.initializeRepositoryClient(repoName)
+	rc, err := ra.initRepositoryClient(repoName)
 	if err != nil {
 		log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
 		ra.CustomAbort(http.StatusInternalServerError, "internal error")
@@ -143,6 +148,8 @@ func (ra *RepositoryAPI) Delete() {
 			ra.CustomAbort(http.StatusInternalServerError, "internal error")
 		}
 		log.Infof("delete tag: %s %s", repoName, t)
+		go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete)
+
 	}
 
 	go func() {
@@ -166,7 +173,7 @@ func (ra *RepositoryAPI) GetTags() {
 		ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
 	}
 
-	rc, err := ra.initializeRepositoryClient(repoName)
+	rc, err := ra.initRepositoryClient(repoName)
 	if err != nil {
 		log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
 		ra.CustomAbort(http.StatusInternalServerError, "internal error")
@@ -186,6 +193,8 @@ func (ra *RepositoryAPI) GetTags() {
 
 	tags = append(tags, ts...)
 
+	sort.Strings(tags)
+
 	ra.Data["json"] = tags
 	ra.ServeJSON()
 }
@@ -199,7 +208,7 @@ func (ra *RepositoryAPI) GetManifests() {
 		ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil")
 	}
 
-	rc, err := ra.initializeRepositoryClient(repoName)
+	rc, err := ra.initRepositoryClient(repoName)
 	if err != nil {
 		log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
 		ra.CustomAbort(http.StatusInternalServerError, "internal error")
@@ -238,16 +247,50 @@ func (ra *RepositoryAPI) GetManifests() {
 	ra.ServeJSON()
 }
 
-func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) {
+func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) {
-	u := models.User{
+	endpoint := os.Getenv("REGISTRY_URL")
-		UserID: ra.userID,
+
+	username, password, ok := ra.Ctx.Request.BasicAuth()
+	if ok {
+		credential := auth.NewBasicAuthCredential(username, password)
+		return registry.NewRepositoryWithCredential(repoName, endpoint, credential)
 	}
-	user, err := dao.GetUser(u)
+
+	username, err = ra.getUsername()
 	if err != nil {
 		return nil, err
 	}
 
-	endpoint := os.Getenv("REGISTRY_URL")
+	return registry.NewRepositoryWithUsername(repoName, endpoint, username)
-
+}
-	return registry.NewRepositoryWithUsername(repoName, endpoint, user.Username)
+
+func (ra *RepositoryAPI) getUsername() (string, error) {
+	// get username from session
+	sessionUsername := ra.GetSession("username")
+	if sessionUsername != nil {
+		username, ok := sessionUsername.(string)
+		if ok {
+			return username, nil
+		}
+	}
+
+	// if username does not exist in session, try to get userId from sessiion
+	// and then get username from DB according to the userId
+	sessionUserID := ra.GetSession("userId")
+	if sessionUserID != nil {
+		userID, ok := sessionUserID.(int)
+		if ok {
+			u := models.User{
+				UserID: userID,
+			}
+			user, err := dao.GetUser(u)
+			if err != nil {
+				return "", err
+			}
+
+			return user.Username, nil
+		}
+	}
+
+	return "", nil
 }

api/target.go

@@ -16,7 +16,6 @@
 package api
 
 import (
-	"encoding/base64"
 	"fmt"
 	"net"
 	"net/http"
@@ -25,6 +24,7 @@ import (
 
 	"github.com/vmware/harbor/dao"
 	"github.com/vmware/harbor/models"
+	"github.com/vmware/harbor/utils"
 	"github.com/vmware/harbor/utils/log"
 	registry_util "github.com/vmware/harbor/utils/registry"
 	"github.com/vmware/harbor/utils/registry/auth"
@@ -76,13 +76,11 @@ func (t *TargetAPI) Ping() {
 		password = target.Password
 
 		if len(password) != 0 {
-			b, err := base64.StdEncoding.DecodeString(password)
+			password, err = utils.ReversibleDecrypt(password)
 			if err != nil {
-				log.Errorf("failed to decode password: %v", err)
+				log.Errorf("failed to decrypt password: %v", err)
 				t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
 			}
-
-			password = string(b)
 		}
 	} else {
 		endpoint = t.GetString("endpoint")
@@ -142,12 +140,12 @@ func (t *TargetAPI) Get() {
 
 		for _, target := range targets {
 			if len(target.Password) != 0 {
-				b, err := base64.StdEncoding.DecodeString(target.Password)
+				str, err := utils.ReversibleDecrypt(target.Password)
 				if err != nil {
-					log.Errorf("failed to decode password: %v", err)
+					log.Errorf("failed to decrypt password: %v", err)
 					t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
 				}
-				target.Password = string(b)
+				target.Password = str
 			}
 		}
 
@@ -167,12 +165,12 @@ func (t *TargetAPI) Get() {
 	}
 
 	if len(target.Password) != 0 {
-		b, err := base64.StdEncoding.DecodeString(target.Password)
+		pwd, err := utils.ReversibleDecrypt(target.Password)
 		if err != nil {
-			log.Errorf("failed to decode password: %v", err)
+			log.Errorf("failed to decrypt password: %v", err)
 			t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
 		}
-		target.Password = string(b)
+		target.Password = pwd
 	}
 
 	t.Data["json"] = target
@@ -189,7 +187,7 @@ func (t *TargetAPI) Post() {
 	}
 
 	if len(target.Password) != 0 {
-		target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password))
+		target.Password = utils.ReversibleEncrypt(target.Password)
 	}
 
 	id, err := dao.AddRepTarget(*target)
@@ -216,7 +214,7 @@ func (t *TargetAPI) Put() {
 	}
 
 	if len(target.Password) != 0 {
-		target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password))
+		target.Password = utils.ReversibleEncrypt(target.Password)
 	}
 
 	if err := dao.UpdateRepTarget(*target); err != nil {

api/utils.go

@@ -16,26 +16,66 @@
 package api
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+
 	"github.com/vmware/harbor/dao"
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/utils/log"
 )
 
 func checkProjectPermission(userID int, projectID int64) bool {
-	exist, err := dao.IsAdminRole(userID)
+	roles, err := listRoles(userID, projectID)
 	if err != nil {
-		log.Errorf("Error occurred in IsAdminRole, error: %v", err)
+		log.Errorf("error occurred in getProjectPermission: %v", err)
 		return false
 	}
-	if exist {
+	return len(roles) > 0
-		return true
+}
-	}
+
-	roleList, err := dao.GetUserProjectRoles(userID, projectID)
+func hasProjectAdminRole(userID int, projectID int64) bool {
+	roles, err := listRoles(userID, projectID)
 	if err != nil {
-		log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
+		log.Errorf("error occurred in getProjectPermission: %v", err)
 		return false
 	}
-	return len(roleList) > 0
+
+	for _, role := range roles {
+		if role.RoleID == models.PROJECTADMIN {
+			return true
+		}
+	}
+
+	return false
+}
+
+//sysadmin has all privileges to all projects
+func listRoles(userID int, projectID int64) ([]models.Role, error) {
+	roles := make([]models.Role, 1)
+	isSysAdmin, err := dao.IsAdminRole(userID)
+	if err != nil {
+		return roles, err
+	}
+	if isSysAdmin {
+		role, err := dao.GetRoleByID(models.PROJECTADMIN)
+		if err != nil {
+			return roles, err
+		}
+		roles = append(roles, *role)
+		return roles, nil
+	}
+
+	rs, err := dao.GetUserProjectRoles(userID, projectID)
+	if err != nil {
+		return roles, err
+	}
+	roles = append(roles, rs...)
+	return roles, nil
 }
 
 func checkUserExists(name string) int {
@@ -49,3 +89,103 @@ func checkUserExists(name string) int {
 	}
 	return 0
 }
+
+// TriggerReplication triggers the replication according to the policy
+func TriggerReplication(policyID int64, repository string,
+	tags []string, operation string) error {
+	data := struct {
+		PolicyID  int64    `json:"policy_id"`
+		Repo      string   `json:"repository"`
+		Operation string   `json:"operation"`
+		TagList   []string `json:"tags"`
+	}{
+		PolicyID:  policyID,
+		Repo:      repository,
+		TagList:   tags,
+		Operation: operation,
+	}
+
+	b, err := json.Marshal(&data)
+	if err != nil {
+		return err
+	}
+
+	url := buildReplicationURL()
+
+	resp, err := http.DefaultClient.Post(url, "application/json", bytes.NewBuffer(b))
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		return nil
+	}
+
+	defer resp.Body.Close()
+
+	b, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return fmt.Errorf("%d %s", resp.StatusCode, string(b))
+}
+
+// GetPoliciesByRepository returns policies according the repository
+func GetPoliciesByRepository(repository string) ([]*models.RepPolicy, error) {
+	repository = strings.TrimSpace(repository)
+	repository = strings.TrimRight(repository, "/")
+	projectName := repository[:strings.LastIndex(repository, "/")]
+
+	project, err := dao.GetProjectByName(projectName)
+	if err != nil {
+		return nil, err
+	}
+
+	policies, err := dao.GetRepPolicyByProject(project.ProjectID)
+	if err != nil {
+		return nil, err
+	}
+
+	return policies, nil
+}
+
+func TriggerReplicationByRepository(repository string, tags []string, operation string) {
+	policies, err := GetPoliciesByRepository(repository)
+	if err != nil {
+		log.Errorf("failed to get policies for repository %s: %v", repository, err)
+		return
+	}
+
+	for _, policy := range policies {
+		if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil {
+			log.Errorf("failed to trigger replication of %d for %s: %v", policy.ID, repository, err)
+		} else {
+			log.Infof("replication of %d for %s triggered", policy.ID, repository)
+		}
+	}
+}
+
+func buildReplicationURL() string {
+	url := getJobServiceURL()
+	url = strings.TrimSpace(url)
+	url = strings.TrimRight(url, "/")
+
+	return fmt.Sprintf("%s/api/replicationJobs", url)
+}
+
+func buildJobLogURL(jobID string) string {
+	url := getJobServiceURL()
+	url = strings.TrimSpace(url)
+	url = strings.TrimRight(url, "/")
+
+	return fmt.Sprintf("%s/api/replicationJobs/%s/log", url, jobID)
+}
+
+func getJobServiceURL() string {
+	url := os.Getenv("JOB_SERVICE_URL")
+	if len(url) == 0 {
+		url = "http://job_service"
+	}
+	return url
+}

dao/project.go

@@ -249,7 +249,6 @@ func getProjects(public int, projectName string) ([]models.Project, error) {
 	}
 	sql += " order by name "
 	var projects []models.Project
-	log.Debugf("sql xxx", sql)
 	if _, err := o.Raw(sql, queryParam).QueryRows(&projects); err != nil {
 		return nil, err
 	}

dao/replication_job.go

@@ -3,9 +3,10 @@ package dao
 import (
 	"fmt"
 
+	"strings"
+
 	"github.com/astaxie/beego/orm"
 	"github.com/vmware/harbor/models"
-	"strings"
 )
 
 func AddRepTarget(target models.RepTarget) (int64, error) {
@@ -81,23 +82,21 @@ func DeleteRepPolicy(id int64) error {
 	_, err := o.Delete(&models.RepPolicy{ID: id})
 	return err
 }
-func updateRepPolicyEnablement(id int64, enabled int) error {
+func UpdateRepPolicyEnablement(id int64, enabled int) error {
 	o := orm.NewOrm()
 	p := models.RepPolicy{
 		ID:      id,
 		Enabled: enabled}
-	num, err := o.Update(&p, "Enabled")
+	_, err := o.Update(&p, "Enabled")
-	if num == 0 {
+
-		err = fmt.Errorf("Failed to update replication policy with id: %d", id)
-	}
 	return err
 }
 func EnableRepPolicy(id int64) error {
-	return updateRepPolicyEnablement(id, 1)
+	return UpdateRepPolicyEnablement(id, 1)
 }
 
 func DisableRepPolicy(id int64) error {
-	return updateRepPolicyEnablement(id, 0)
+	return UpdateRepPolicyEnablement(id, 0)
 }
 
 func AddRepJob(job models.RepJob) (int64, error) {

job/replication/delete.go

@@ -0,0 +1,119 @@
+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package replication
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	"github.com/vmware/harbor/models"
+	"github.com/vmware/harbor/utils/log"
+)
+
+const (
+	// StateDelete ...
+	StateDelete = "delete"
+)
+
+// Deleter deletes repository or tags
+type Deleter struct {
+	repository string // prject_name/repo_name
+	tags       []string
+
+	dstURL string // url of target registry
+	dstUsr string // username ...
+	dstPwd string // username ...
+
+	logger *log.Logger
+}
+
+// NewDeleter returns a Deleter
+func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, logger *log.Logger) *Deleter {
+	deleter := &Deleter{
+		repository: repository,
+		tags:       tags,
+		dstURL:     dstURL,
+		dstUsr:     dstUsr,
+		dstPwd:     dstPwd,
+		logger:     logger,
+	}
+	deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, destination user: %s",
+		deleter.repository, deleter.tags, deleter.dstURL, deleter.dstUsr)
+	return deleter
+}
+
+// Exit ...
+func (d *Deleter) Exit() error {
+	return nil
+}
+
+// Enter deletes repository or tags
+func (d *Deleter) Enter() (string, error) {
+	url := strings.TrimRight(d.dstURL, "/") + "/api/repositories/"
+
+	// delete repository
+	if len(d.tags) == 0 {
+		u := url + "?repo_name=" + d.repository
+		if err := del(u, d.dstUsr, d.dstPwd); err != nil {
+			d.logger.Errorf("an error occurred while deleting repository %s on %s with user %s: %v", d.repository, d.dstURL, d.dstUsr, err)
+			return "", err
+		}
+
+		d.logger.Infof("repository %s on %s has been deleted", d.repository, d.dstURL)
+
+		return models.JobFinished, nil
+	}
+
+	// delele tags
+	for _, tag := range d.tags {
+		u := url + "?repo_name=" + d.repository + "&tag=" + tag
+		if err := del(u, d.dstUsr, d.dstPwd); err != nil {
+			d.logger.Errorf("an error occurred while deleting repository %s:%s on %s with user %s: %v", d.repository, tag, d.dstURL, d.dstUsr, err)
+			return "", err
+		}
+
+		d.logger.Infof("repository %s:%s on %s has been deleted", d.repository, tag, d.dstURL)
+	}
+
+	return models.JobFinished, nil
+}
+
+func del(url, username, password string) error {
+	req, err := http.NewRequest("DELETE", url, nil)
+	if err != nil {
+		return err
+	}
+
+	req.SetBasicAuth(username, password)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode == http.StatusOK {
+		return nil
+	}
+
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return fmt.Errorf("%d %s", resp.StatusCode, string(b))
+}

job/replication/transfer.go

@@ -153,7 +153,7 @@ func (c *Checker) Enter() (string, error) {
 	c.logger.Infof("project %s already exists on %s", c.project, c.dstURL)
 
 	if !canWrite {
-		err = fmt.Errorf("the user %s has no write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL)
+		err = fmt.Errorf("the user %s is unauthorized to write to project %s on %s", c.dstUsr, c.project, c.dstURL)
 		c.logger.Errorf("%v", err)
 		return "", err
 	}

job/statemachine.go

@@ -9,6 +9,7 @@ import (
 	"github.com/vmware/harbor/job/replication"
 	"github.com/vmware/harbor/job/utils"
 	"github.com/vmware/harbor/models"
+	uti "github.com/vmware/harbor/utils"
 	"github.com/vmware/harbor/utils/log"
 )
 
@@ -18,6 +19,7 @@ type RepJobParm struct {
 	TargetUsername string
 	TargetPassword string
 	Repository     string
+	Tags           []string
 	Enabled        int
 	Operation      string
 }
@@ -182,6 +184,7 @@ func (sm *JobSM) Reset(jid int64) error {
 	sm.Parms = &RepJobParm{
 		LocalRegURL: config.LocalHarborURL(),
 		Repository:  job.Repository,
+		Tags:        job.TagList,
 		Enabled:     policy.Enabled,
 		Operation:   job.Operation,
 	}
@@ -198,35 +201,43 @@ func (sm *JobSM) Reset(jid int64) error {
 	}
 	sm.Parms.TargetURL = target.URL
 	sm.Parms.TargetUsername = target.Username
-	sm.Parms.TargetPassword = target.Password
+	pwd := target.Password
+
+	if len(pwd) != 0 {
+		pwd, err = uti.ReversibleDecrypt(pwd)
+		if err != nil {
+			return fmt.Errorf("failed to decrypt password: %v", err)
+		}
+	}
+
+	sm.Parms.TargetPassword = pwd
+
 	//init states handlers
 	sm.Logger = utils.NewLogger(sm.JobID)
+	sm.Handlers = make(map[string]StateHandler)
+	sm.Transitions = make(map[string]map[string]struct{})
 	sm.CurrentState = models.JobPending
 
 	sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning})
 	sm.Handlers[models.JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobError}
 	sm.Handlers[models.JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobStopped}
 
-	if sm.Parms.Operation == models.RepOpTransfer {
+	switch sm.Parms.Operation {
-		/*
+	case models.RepOpTransfer:
-			sm.AddTransition(models.JobRunning, "pull-img", ImgPuller{DummyHandler: DummyHandler{JobID: sm.JobID}, img: sm.Parms.Repository, logger: sm.Logger})
+		err = addImgTransferTransition(sm)
-			//only handle one target for now
+	case models.RepOpDelete:
-			sm.AddTransition("pull-img", "push-img", ImgPusher{DummyHandler: DummyHandler{JobID: sm.JobID}, targetURL: sm.Parms.TargetURL, logger: sm.Logger})
+		err = addImgDeleteTransition(sm)
-			sm.AddTransition("push-img", models.JobFinished, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished})
+	default:
-		*/
+		err = fmt.Errorf("unsupported operation: %s", sm.Parms.Operation)
-
-		if err = addImgOutTransition(sm); err != nil {
-			return err
-		}
-
 	}
-	return nil
+
+	return err
 }
 
-func addImgOutTransition(sm *JobSM) error {
+func addImgTransferTransition(sm *JobSM) error {
 	base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(),
 		sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword,
-		nil, sm.Logger)
+		sm.Parms.Tags, sm.Logger)
 	if err != nil {
 		return err
 	}
@@ -238,3 +249,13 @@ func addImgOutTransition(sm *JobSM) error {
 	sm.AddTransition(replication.StatePushManifest, replication.StatePullManifest, &replication.ManifestPuller{BaseHandler: base})
 	return nil
 }
+
+func addImgDeleteTransition(sm *JobSM) error {
+	deleter := replication.NewDeleter(sm.Parms.Repository, sm.Parms.Tags, sm.Parms.TargetURL,
+		sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Logger)
+
+	sm.AddTransition(models.JobRunning, replication.StateDelete, deleter)
+	sm.AddTransition(replication.StateDelete, models.JobFinished, &StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished})
+
+	return nil
+}

service/notification.go

@@ -20,6 +20,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/vmware/harbor/api"
 	"github.com/vmware/harbor/dao"
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/service/cache"
@@ -69,6 +70,11 @@ func (n *NotificationHandler) Post() {
 			if username == "" {
 				username = "anonymous"
 			}
+
+			if username == "job-service-user" {
+				return
+			}
+
 			go dao.AccessLog(username, project, repo, repoTag, action)
 			if action == "push" {
 				go func() {
@@ -77,6 +83,8 @@ func (n *NotificationHandler) Post() {
 						log.Errorf("Error happens when refreshing cache: %v", err2)
 					}
 				}()
+
+				go api.TriggerReplicationByRepository(repo, []string{repoTag}, models.RepOpTransfer)
 			}
 		}
 	}

service/token/authutils.go

@@ -46,10 +46,24 @@ func GetResourceActions(scopes []string) []*token.ResourceActions {
 			continue
 		}
 		items := strings.Split(s, ":")
+		length := len(items)
+
+		typee := items[0]
+
+		name := ""
+		if length > 1 {
+			name = items[1]
+		}
+
+		actions := []string{}
+		if length > 2 {
+			actions = strings.Split(items[2], ",")
+		}
+
 		res = append(res, &token.ResourceActions{
-			Type:    items[0],
+			Type:    typee,
-			Name:    items[1],
+			Name:    name,
-			Actions: strings.Split(items[2], ","),
+			Actions: actions,
 		})
 	}
 	return res

ui/router.go

@@ -53,6 +53,7 @@ func initRouters() {
 	//API:
 	beego.Router("/api/search", &api.SearchAPI{})
 	beego.Router("/api/projects/:pid/members/?:mid", &api.ProjectMemberAPI{})
+	beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List")
 	beego.Router("/api/projects/?:id", &api.ProjectAPI{})
 	beego.Router("/api/statistics", &api.StatisticAPI{})
 	beego.Router("/api/projects/:id/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog")
@@ -62,6 +63,10 @@ func initRouters() {
 	beego.Router("/api/repositories", &api.RepositoryAPI{})
 	beego.Router("/api/repositories/tags", &api.RepositoryAPI{}, "get:GetTags")
 	beego.Router("/api/repositories/manifests", &api.RepositoryAPI{}, "get:GetManifests")
+	beego.Router("/api/replicationJobs", &api.RepJobAPI{})
+	beego.Router("/api/replicationJobs/:id/log", &api.RepJobAPI{}, "get:GetLog")
+	beego.Router("/api/replicationPolicies", &api.RepPolicyAPI{})
+	beego.Router("/api/replicationPolicies/:id/enablement", &api.RepPolicyAPI{}, "put:UpdateEnablement")
 	beego.Router("/api/targets/?:id", &api.TargetAPI{})
 	beego.Router("/api/target_ping", &api.TargetAPI{}, "get:Ping")
 

utils/encrypt.go

@@ -17,6 +17,7 @@ package utils
 
 import (
 	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 
 	"golang.org/x/crypto/pbkdf2"
@@ -26,3 +27,14 @@ import (
 func Encrypt(content string, salt string) string {
 	return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, sha1.New))
 }
+
+// ReversibleEncrypt encrypts the str with base64
+func ReversibleEncrypt(str string) string {
+	return base64.StdEncoding.EncodeToString([]byte(str))
+}
+
+// ReversibleDecrypt decrypts the str with base64
+func ReversibleDecrypt(str string) (string, error) {
+	b, err := base64.StdEncoding.DecodeString(str)
+	return string(b), err
+}