From 18254b9170fe54940e16789b6cc28e6bc39653da Mon Sep 17 00:00:00 2001 From: yhua123 Date: Tue, 17 May 2016 10:09:34 +0800 Subject: [PATCH 01/16] Create rsyslog.conf Using static 'rsyslog.conf' instead 'sed' command in Dockerfile, this is the modified config file. --- Deploy/log/rsyslog.conf | 60 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 Deploy/log/rsyslog.conf diff --git a/Deploy/log/rsyslog.conf b/Deploy/log/rsyslog.conf new file mode 100644 index 000000000..8b5c4fd84 --- /dev/null +++ b/Deploy/log/rsyslog.conf @@ -0,0 +1,60 @@ +# /etc/rsyslog.conf Configuration file for rsyslog. +# +# For more information see +# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html +# +# Default logging rules can be found in /etc/rsyslog.d/50-default.conf + + +################# +#### MODULES #### +################# + +$ModLoad imuxsock # provides support for local system logging +#$ModLoad imklog # provides kernel logging support +#$ModLoad immark # provides --MARK-- message capability + +# provides UDP syslog reception +$ModLoad imudp +$UDPServerRun 514 + +# provides TCP syslog reception +$ModLoad imtcp +$InputTCPServerRun 514 + +# Enable non-kernel facility klog messages +#$KLogPermitNonKernelFacility on + +########################### +#### GLOBAL DIRECTIVES #### +########################### + +# +# Use traditional timestamp format. +# To enable high precision timestamps, comment out the following line. +# +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# Filter duplicated messages +$RepeatedMsgReduction on + +# +# Set the default permissions for all log files. +# +$FileOwner syslog +$FileGroup adm +$FileCreateMode 0640 +$DirCreateMode 0755 +$Umask 0022 +#$PrivDropToUser syslog +#$PrivDropToGroup syslog + +# +# Where to place spool and state files +# +$WorkDirectory /var/spool/rsyslog + +# +# Include all config files in /etc/rsyslog.d/ +# +$IncludeConfig /etc/rsyslog.d/*.conf From 0c75672f590afc76bafe8985fbbac004391c1f24 Mon Sep 17 00:00:00 2001 From: yhua123 Date: Tue, 17 May 2016 11:24:02 +0800 Subject: [PATCH 02/16] Update Dockerfile --- Deploy/log/Dockerfile | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/Deploy/log/Dockerfile b/Deploy/log/Dockerfile index 18a86b0af..3f02ad542 100644 --- a/Deploy/log/Dockerfile +++ b/Deploy/log/Dockerfile @@ -2,15 +2,9 @@ FROM library/ubuntu:14.04 # run logrotate hourly, disable imklog model, provides TCP/UDP syslog reception RUN mv /etc/cron.daily/logrotate /etc/cron.hourly/ \ - && sed 's/$ModLoad imklog/#$ModLoad imklog/' -i /etc/rsyslog.conf \ - && sed 's/$KLogPermitNonKernelFacility on/#$KLogPermitNonKernelFacility on/' -i /etc/rsyslog.conf \ - && sed 's/#$ModLoad imudp/$ModLoad imudp/' -i /etc/rsyslog.conf \ - && sed 's/#$UDPServerRun 514/$UDPServerRun 514/' -i /etc/rsyslog.conf \ - && sed 's/#$ModLoad imtcp/$ModLoad imtcp/' -i /etc/rsyslog.conf \ - && sed 's/#$InputTCPServerRun 514/$InputTCPServerRun 514/' -i /etc/rsyslog.conf \ - && sed 's/$PrivDropToUser syslog/#$PrivDropToUser syslog/' -i /etc/rsyslog.conf \ - && sed 's/$PrivDropToGroup syslog/#$PrivDropToGroup syslog/' -i /etc/rsyslog.conf \ - && rm /etc/rsyslog.d/* + && rm /etc/rsyslog.d/* \ + && rm /etc/rsyslog.conf +ADD rsyslog.conf /etc/rsyslog.conf # logrotate configuration file for docker ADD logrotate_docker.conf /etc/logrotate.d/ @@ -23,4 +17,3 @@ VOLUME /var/log/docker/ EXPOSE 514 CMD cron && rsyslogd -n - From 6cd1813b86acfdf8a8ddd0a3a9dcc5b79d638914 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 19 May 2016 18:36:40 +0800 Subject: [PATCH 03/16] 1. user can get repositories of public project without login 2. add basic auth support for repository API 3. system admin can CRUD members of all projects --- api/member.go | 97 ++++++++++------------------------------------- api/repository.go | 68 ++++++++++++++++++++++++--------- api/utils.go | 48 +++++++++++++++++++---- 3 files changed, 112 insertions(+), 101 deletions(-) diff --git a/api/member.go b/api/member.go index 2ae7c9c53..b86e36f5d 100644 --- a/api/member.go +++ b/api/member.go @@ -114,23 +114,10 @@ func (pma *ProjectMemberAPI) Get() { // Post ... func (pma *ProjectMemberAPI) Post() { - pid := pma.project.ProjectID - - //userQuery := models.User{UserID: pma.currentUserID, RoleID: models.PROJECTADMIN} - rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid) - if err != nil { - log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) - pma.CustomAbort(http.StatusInternalServerError, "Internal error.") - } - hasProjectAdminRole := false - for _, role := range rolelist { - if role.RoleID == models.PROJECTADMIN { - hasProjectAdminRole = true - break - } - } - if !hasProjectAdminRole { - log.Warningf("Current user, id: %d does not have project admin role for project, id:", pma.currentUserID, pid) + currentUserID := pma.currentUserID + projectID := pma.project.ProjectID + if !hasProjectAdminRole(currentUserID, projectID) { + log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, projectID) pma.RenderError(http.StatusForbidden, "") return } @@ -144,21 +131,21 @@ func (pma *ProjectMemberAPI) Post() { pma.RenderError(http.StatusNotFound, "User does not exist") return } - rolelist, err = dao.GetUserProjectRoles(userID, pid) + rolelist, err := dao.GetUserProjectRoles(userID, projectID) if err != nil { log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) pma.CustomAbort(http.StatusInternalServerError, "Internal error.") } if len(rolelist) > 0 { - log.Warningf("user is already added to project, user id: %d, project id: %d", userID, pid) + log.Warningf("user is already added to project, user id: %d, project id: %d", userID, projectID) pma.RenderError(http.StatusConflict, "user is ready in project") return } for _, rid := range req.Roles { - err = dao.AddProjectMember(pid, userID, int(rid)) + err = dao.AddProjectMember(projectID, userID, int(rid)) if err != nil { - log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", pid, userID, rid) + log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", projectID, userID, rid) pma.RenderError(http.StatusInternalServerError, "Failed to update data in database") return } @@ -167,27 +154,16 @@ func (pma *ProjectMemberAPI) Post() { // Put ... func (pma *ProjectMemberAPI) Put() { + currentUserID := pma.currentUserID pid := pma.project.ProjectID - mid := pma.memberID - - rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid) - if err != nil { - log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) - pma.CustomAbort(http.StatusInternalServerError, "Internal error.") - } - hasProjectAdminRole := false - for _, role := range rolelist { - if role.RoleID == models.PROJECTADMIN { - hasProjectAdminRole = true - break - } - } - - if !hasProjectAdminRole { - log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid) + if !hasProjectAdminRole(currentUserID, pid) { + log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid) pma.RenderError(http.StatusForbidden, "") return } + + mid := pma.memberID + var req memberReq pma.DecodeJSONReq(&req) roleList, err := dao.GetUserProjectRoles(mid, pid) @@ -217,51 +193,20 @@ func (pma *ProjectMemberAPI) Put() { // Delete ... func (pma *ProjectMemberAPI) Delete() { + currentUserID := pma.currentUserID pid := pma.project.ProjectID - mid := pma.memberID - - rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid) - hasProjectAdminRole := false - for _, role := range rolelist { - if role.RoleID == models.PROJECTADMIN { - hasProjectAdminRole = true - break - } - } - - if !hasProjectAdminRole { - log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid) + if !hasProjectAdminRole(currentUserID, pid) { + log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid) pma.RenderError(http.StatusForbidden, "") return } - err = dao.DeleteProjectMember(pid, mid) + + mid := pma.memberID + + err := dao.DeleteProjectMember(pid, mid) if err != nil { log.Errorf("Failed to delete project roles for user, user id: %d, project id: %d, error: %v", mid, pid, err) pma.RenderError(http.StatusInternalServerError, "Failed to update data in DB") return } } - -//sysadmin has all privileges to all projects -func listRoles(userID int, projectID int64) ([]models.Role, error) { - roles := make([]models.Role, 1) - isSysAdmin, err := dao.IsAdminRole(userID) - if err != nil { - return roles, err - } - if isSysAdmin { - role, err := dao.GetRoleByID(models.PROJECTADMIN) - if err != nil { - return roles, err - } - roles = append(roles, *role) - return roles, nil - } - - rs, err := dao.GetUserProjectRoles(userID, projectID) - if err != nil { - return roles, err - } - roles = append(roles, rs...) - return roles, nil -} diff --git a/api/repository.go b/api/repository.go index a827d58bc..ddf66c335 100644 --- a/api/repository.go +++ b/api/repository.go @@ -38,19 +38,19 @@ import ( // the security of registry type RepositoryAPI struct { BaseAPI - userID int + //userID int } // Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. -func (ra *RepositoryAPI) Prepare() { - ra.userID = ra.ValidateUser() -} +//func (ra *RepositoryAPI) Prepare() { +// ra.userID = ra.ValidateUser() +//} // Get ... func (ra *RepositoryAPI) Get() { - projectID, err0 := ra.GetInt64("project_id") - if err0 != nil { - log.Errorf("Failed to get project id, error: %v", err0) + projectID, err := ra.GetInt64("project_id") + if err != nil { + log.Errorf("Failed to get project id, error: %v", err) ra.RenderError(http.StatusBadRequest, "Invalid project id") return } @@ -64,9 +64,14 @@ func (ra *RepositoryAPI) Get() { ra.RenderError(http.StatusNotFound, "") return } - if p.Public == 0 && !checkProjectPermission(ra.userID, projectID) { - ra.RenderError(http.StatusForbidden, "") - return + + if p.Public == 0 { + userID := ra.ValidateUser() + + if !checkProjectPermission(userID, projectID) { + ra.RenderError(http.StatusForbidden, "") + return + } } repoList, err := svc_utils.GetRepoFromCache() @@ -239,15 +244,44 @@ func (ra *RepositoryAPI) GetManifests() { } func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) { - u := models.User{ - UserID: ra.userID, - } - user, err := dao.GetUser(u) - if err != nil { - return nil, err + var username string + var sessionUsername, sessionUserID interface{} + + // get username from basic auth + username, _, ok := ra.Ctx.Request.BasicAuth() + if ok { + goto enter } + // get username from session + sessionUsername = ra.GetSession("username") + if sessionUsername != nil { + username, ok = sessionUsername.(string) + if ok { + goto enter + } + } + + // if username does not exist in session, try to get userId from sessiion + // and then get username from DB according to the userId + sessionUserID = ra.GetSession("userId") + if sessionUserID != nil { + userID, ok := sessionUserID.(int) + if ok { + u := models.User{ + UserID: userID, + } + user, err := dao.GetUser(u) + if err != nil { + return nil, err + } + username = user.Username + goto enter + } + } + +enter: endpoint := os.Getenv("REGISTRY_URL") - return registry.NewRepositoryWithUsername(repoName, endpoint, user.Username) + return registry.NewRepositoryWithUsername(repoName, endpoint, username) } diff --git a/api/utils.go b/api/utils.go index 826484ac3..97799cc55 100644 --- a/api/utils.go +++ b/api/utils.go @@ -22,20 +22,52 @@ import ( ) func checkProjectPermission(userID int, projectID int64) bool { - exist, err := dao.IsAdminRole(userID) + roles, err := listRoles(userID, projectID) if err != nil { - log.Errorf("Error occurred in IsAdminRole, error: %v", err) + log.Errorf("error occurred in getProjectPermission: %v", err) return false } - if exist { - return true - } - roleList, err := dao.GetUserProjectRoles(userID, projectID) + return len(roles) > 0 +} + +func hasProjectAdminRole(userID int, projectID int64) bool { + roles, err := listRoles(userID, projectID) if err != nil { - log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) + log.Errorf("error occurred in getProjectPermission: %v", err) return false } - return len(roleList) > 0 + + for _, role := range roles { + if role.RoleID == models.PROJECTADMIN { + return true + } + } + + return false +} + +//sysadmin has all privileges to all projects +func listRoles(userID int, projectID int64) ([]models.Role, error) { + roles := make([]models.Role, 1) + isSysAdmin, err := dao.IsAdminRole(userID) + if err != nil { + return roles, err + } + if isSysAdmin { + role, err := dao.GetRoleByID(models.PROJECTADMIN) + if err != nil { + return roles, err + } + roles = append(roles, *role) + return roles, nil + } + + rs, err := dao.GetUserProjectRoles(userID, projectID) + if err != nil { + return roles, err + } + roles = append(roles, rs...) + return roles, nil } func checkUserExists(name string) int { From 8c86e786c233a7c1239c36f7645ebc6f4d2ce6d8 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 20 May 2016 11:16:19 +0800 Subject: [PATCH 04/16] user can get public projects without login --- api/project.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/api/project.go b/api/project.go index 90ccbdb8c..cd65dce5f 100644 --- a/api/project.go +++ b/api/project.go @@ -43,7 +43,6 @@ const projectNameMaxLen int = 30 // Prepare validates the URL and the user func (p *ProjectAPI) Prepare() { - p.userID = p.ValidateUser() idStr := p.Ctx.Input.Param(":id") if len(idStr) > 0 { var err error @@ -65,6 +64,8 @@ func (p *ProjectAPI) Prepare() { // Post ... func (p *ProjectAPI) Post() { + p.userID = p.ValidateUser() + var req projectReq var public int p.DecodeJSONReq(&req) @@ -99,15 +100,24 @@ func (p *ProjectAPI) Post() { // Head ... func (p *ProjectAPI) Head() { projectName := p.GetString("project_name") - result, err := dao.ProjectExists(projectName) + project, err := dao.GetProjectByName(projectName) if err != nil { - log.Errorf("Error while communicating with DB, error: %v", err) - p.RenderError(http.StatusInternalServerError, "Error while communicating with DB") + log.Errorf("error occurred in GetProjectByName: %v", err) + p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + // only public project can be Headed by user without login + if project != nil && project.Public == 1 { return } - if !result { - p.RenderError(http.StatusNotFound, "") - return + + userID := p.ValidateUser() + if project == nil { + p.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound)) + } + + if !checkProjectPermission(userID, project.ProjectID) { + p.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden)) } } @@ -132,6 +142,8 @@ func (p *ProjectAPI) Get() { if public == 1 { projectList, err = dao.GetPublicProjects(projectName) } else { + //if the request is not for public projects, user must login or provide credential + p.userID = p.ValidateUser() isAdmin, err = dao.IsAdminRole(p.userID) if err != nil { log.Errorf("Error occured in check admin, error: %v", err) @@ -164,6 +176,8 @@ func (p *ProjectAPI) Get() { // Put ... func (p *ProjectAPI) Put() { + p.userID = p.ValidateUser() + var req projectReq var public int @@ -192,6 +206,7 @@ func (p *ProjectAPI) Put() { // FilterAccessLog handles GET to /api/projects/{}/logs func (p *ProjectAPI) FilterAccessLog() { + p.userID = p.ValidateUser() var filter models.AccessLog p.DecodeJSONReq(&filter) From 792080373549e9857562d802ec554e76a3b4499c Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 20 May 2016 11:32:12 +0800 Subject: [PATCH 05/16] handle project name is nil --- api/project.go | 4 ++++ dao/project.go | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api/project.go b/api/project.go index cd65dce5f..0a80db759 100644 --- a/api/project.go +++ b/api/project.go @@ -100,6 +100,10 @@ func (p *ProjectAPI) Post() { // Head ... func (p *ProjectAPI) Head() { projectName := p.GetString("project_name") + if len(projectName) == 0 { + p.CustomAbort(http.StatusBadRequest, "project_name is needed") + } + project, err := dao.GetProjectByName(projectName) if err != nil { log.Errorf("error occurred in GetProjectByName: %v", err) diff --git a/dao/project.go b/dao/project.go index 730b1f331..399cad6aa 100644 --- a/dao/project.go +++ b/dao/project.go @@ -250,7 +250,6 @@ func getProjects(public int, projectName string) ([]models.Project, error) { } sql += " order by name " var projects []models.Project - log.Debugf("sql xxx", sql) if _, err := o.Raw(sql, queryParam).QueryRows(&projects); err != nil { return nil, err } From 1919f19459d5e0a588d45fedbc472db302599be3 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 20 May 2016 11:38:24 +0800 Subject: [PATCH 06/16] remove useless codes --- api/repository.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/api/repository.go b/api/repository.go index ddf66c335..b860687fb 100644 --- a/api/repository.go +++ b/api/repository.go @@ -38,14 +38,8 @@ import ( // the security of registry type RepositoryAPI struct { BaseAPI - //userID int } -// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. -//func (ra *RepositoryAPI) Prepare() { -// ra.userID = ra.ValidateUser() -//} - // Get ... func (ra *RepositoryAPI) Get() { projectID, err := ra.GetInt64("project_id") From 0b30d04455f47af8f4ce41cce0ac984792096c02 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 20 May 2016 14:02:44 +0800 Subject: [PATCH 07/16] sort tags by name --- api/repository.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/repository.go b/api/repository.go index b860687fb..961da1a88 100644 --- a/api/repository.go +++ b/api/repository.go @@ -19,6 +19,7 @@ import ( "encoding/json" "net/http" "os" + "sort" "strconv" "strings" "time" @@ -184,6 +185,8 @@ func (ra *RepositoryAPI) GetTags() { tags = append(tags, ts...) + sort.Strings(tags) + ra.Data["json"] = tags ra.ServeJSON() } From 0b57345a4dfccd164ad22f566791b968447c1991 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 25 May 2016 15:25:16 +0800 Subject: [PATCH 08/16] 1.add API for getting logs of job 2.trigger replication by creating policy or enabling policy 3.trigger replication by webhook --- api/replication_job.go | 36 +++++++++++++++- api/replication_policy.go | 25 ++++++++++- api/utils.go | 88 +++++++++++++++++++++++++++++++++++++++ service/notification.go | 3 ++ 4 files changed, 149 insertions(+), 3 deletions(-) diff --git a/api/replication_job.go b/api/replication_job.go index 0c6f0f67b..c15e1f1f8 100644 --- a/api/replication_job.go +++ b/api/replication_job.go @@ -1,9 +1,11 @@ package api import ( + "io/ioutil" + "net/http" + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/utils/log" - "net/http" ) type RepJobAPI struct { @@ -19,6 +21,7 @@ func (ja *RepJobAPI) Prepare() { if !isAdmin { ja.CustomAbort(http.StatusForbidden, "") } + } func (ja *RepJobAPI) Get() { @@ -38,4 +41,35 @@ func (ja *RepJobAPI) Get() { ja.ServeJSON() } +// GetLog ... +func (ja *RepJobAPI) GetLog() { + id := ja.Ctx.Input.Param(":id") + if len(id) == 0 { + ja.CustomAbort(http.StatusBadRequest, "id is nil") + } + + resp, err := http.Get(buildJobLogURL(id)) + if err != nil { + log.Errorf("failed to get log for job %s: %v", id, err) + ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Errorf("failed to read response body for job %s: %v", id, err) + ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if resp.StatusCode == http.StatusOK { + if _, err = ja.Ctx.ResponseWriter.Write(b); err != nil { + log.Errorf("failed to write log to response; %v", err) + ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + return + } + + ja.CustomAbort(resp.StatusCode, string(b)) +} + //TODO:add Post handler to call job service API to submit jobs by policy diff --git a/api/replication_policy.go b/api/replication_policy.go index 174e898f0..2e78100e8 100644 --- a/api/replication_policy.go +++ b/api/replication_policy.go @@ -2,11 +2,13 @@ package api import ( "fmt" + + "net/http" + "strconv" + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" - "net/http" - "strconv" ) type RepPolicyAPI struct { @@ -70,6 +72,15 @@ func (pa *RepPolicyAPI) Post() { pa.RenderError(http.StatusInternalServerError, "Internal Error") return } + + go func() { + if err := TriggerReplication(pid, "", models.RepOpTransfer); err != nil { + log.Errorf("failed to trigger replication of %d: %v", pid, err) + } else { + log.Infof("replication of %d triggered", pid) + } + }() + pa.Redirect(http.StatusCreated, strconv.FormatInt(pid, 10)) } @@ -94,4 +105,14 @@ func (pa *RepPolicyAPI) UpdateEnablement() { pa.RenderError(http.StatusInternalServerError, "Internal Error") return } + + if e.Enabled == 1 { + go func() { + if err := TriggerReplication(pa.policyID, "", models.RepOpTransfer); err != nil { + log.Errorf("failed to trigger replication of %d: %v", pa.policyID, err) + } else { + log.Infof("replication of %d triggered", pa.policyID) + } + }() + } } diff --git a/api/utils.go b/api/utils.go index 826484ac3..92590c097 100644 --- a/api/utils.go +++ b/api/utils.go @@ -16,6 +16,13 @@ package api import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" @@ -49,3 +56,84 @@ func checkUserExists(name string) int { } return 0 } + +// TriggerReplication triggers the replication according to the policy +func TriggerReplication(policyID int64, repository, operation string) error { + data := struct { + PolicyID int64 `json:"policy_id"` + Repo string `json:"repository"` + Operation string `json:"operation"` + }{ + PolicyID: policyID, + Repo: repository, + Operation: operation, + } + + b, err := json.Marshal(data) + if err != nil { + return err + } + + url := buildReplicationURL() + + resp, err := http.DefaultClient.Post(url, "application/json", bytes.NewBuffer(b)) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusOK { + return nil + } + + defer resp.Body.Close() + + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return fmt.Errorf("%d %s", resp.StatusCode, string(b)) +} + +// GetPoliciesByRepository returns policies according the repository +func GetPoliciesByRepository(repository string) ([]*models.RepPolicy, error) { + repository = strings.TrimSpace(repository) + repository = strings.TrimRight(repository, "/") + projectName := repository[:strings.LastIndex(repository, "/")] + + project, err := dao.GetProjectByName(projectName) + if err != nil { + return nil, err + } + + policies, err := dao.GetRepPolicyByProject(project.ProjectID) + if err != nil { + return nil, err + } + + return policies, nil +} + +func TriggerReplicationByRepository(repository, operation string) { + policies, err := GetPoliciesByRepository(repository) + if err != nil { + log.Errorf("failed to get policies for repository %s: %v", repository, err) + return + } + + for _, policy := range policies { + if err := TriggerReplication(policy.ProjectID, repository, operation); err != nil { + log.Errorf("failed to trigger replication of %d for %s: %v", policy.ID, repository, err) + } else { + log.Infof("replication of %d for %s triggered", policy.ID, repository) + } + } +} + +func buildReplicationURL() string { + return "http://job_service/api/replicationJobs" +} + +func buildJobLogURL(jobID string) string { + return fmt.Sprintf("http://job_service/api/replicationJobs/%s/log", jobID) +} diff --git a/service/notification.go b/service/notification.go index 916b4901b..629bda987 100644 --- a/service/notification.go +++ b/service/notification.go @@ -20,6 +20,7 @@ import ( "regexp" "strings" + "github.com/vmware/harbor/api" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" @@ -77,6 +78,8 @@ func (n *NotificationHandler) Post() { log.Errorf("Error happens when refreshing cache: %v", err2) } }() + + go api.TriggerReplicationByRepository(repo, models.RepOpTransfer) } } } From 3f95a954dc23639813b12f37d42f15d29976f5db Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 25 May 2016 16:51:40 +0800 Subject: [PATCH 09/16] add state handler for deleting repository --- job/replication/deletestatehandlers.go | 104 +++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 job/replication/deletestatehandlers.go diff --git a/job/replication/deletestatehandlers.go b/job/replication/deletestatehandlers.go new file mode 100644 index 000000000..a4b07d8c6 --- /dev/null +++ b/job/replication/deletestatehandlers.go @@ -0,0 +1,104 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package replication + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" +) + +const ( + // StateDelete ... + StateDelete = "delete" +) + +// Deleter deletes repository or tags +type Deleter struct { + repository string // prject_name/repo_name + tags []string + + dstURL string // url of target registry + dstUsr string // username ... + dstPwd string // username ... + + logger *log.Logger +} + +// Exit ... +func (d *Deleter) Exit() error { + return nil +} + +// Enter deletes repository or tags +func (d *Deleter) Enter() (string, error) { + url := strings.TrimRight(d.dstURL, "/") + "/api/repositories/" + + // delete repository + if len(d.tags) == 0 { + u := url + "?repo_name=" + d.repository + if err := del(u, d.dstUsr, d.dstPwd); err != nil { + d.logger.Errorf("an error occurred while deleting repository %s on %s with user %s: %v", d.repository, d.dstURL, d.dstUsr, err) + return "", err + } + + d.logger.Infof("repository %s on %s has been deleted", d.repository, d.dstURL) + + return models.JobFinished, nil + } + + // delele tags + for _, tag := range d.tags { + u := url + "?repo_name=" + d.repository + "&tag=" + tag + if err := del(u, d.dstUsr, d.dstPwd); err != nil { + d.logger.Errorf("an error occurred while deleting repository %s:%s on %s with user %s: %v", d.repository, tag, d.dstURL, d.dstUsr, err) + return "", err + } + + d.logger.Infof("repository %s:%s on %s has been deleted", d.repository, tag, d.dstURL) + } + + return models.JobFinished, nil +} + +func del(url, username, password string) error { + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + req.SetBasicAuth(username, password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusOK { + return nil + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return fmt.Errorf("%d %s", resp.StatusCode, string(b)) +} From 535bea1b6b6ca9a476685716f77ce12e0e8e616b Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 25 May 2016 17:22:33 +0800 Subject: [PATCH 10/16] fix #268 --- service/token/authutils.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/service/token/authutils.go b/service/token/authutils.go index 3ea15c294..4c30648c4 100644 --- a/service/token/authutils.go +++ b/service/token/authutils.go @@ -46,10 +46,24 @@ func GetResourceActions(scopes []string) []*token.ResourceActions { continue } items := strings.Split(s, ":") + length := len(items) + + typee := items[0] + + name := "" + if length > 1 { + name = items[1] + } + + actions := []string{} + if length > 2 { + actions = strings.Split(items[2], ",") + } + res = append(res, &token.ResourceActions{ - Type: items[0], - Name: items[1], - Actions: strings.Split(items[2], ","), + Type: typee, + Name: name, + Actions: actions, }) } return res From 61e1b03a75e860906bdb0c991302843ca3bfabdc Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 26 May 2016 11:27:16 +0800 Subject: [PATCH 11/16] add delete state handlers --- .../{deletestatehandlers.go => delete.go} | 12 +++++++ .../{statehandlers.go => transfer.go} | 0 job/statemachine.go | 34 +++++++++++-------- 3 files changed, 32 insertions(+), 14 deletions(-) rename job/replication/{deletestatehandlers.go => delete.go} (89%) rename job/replication/{statehandlers.go => transfer.go} (100%) diff --git a/job/replication/deletestatehandlers.go b/job/replication/delete.go similarity index 89% rename from job/replication/deletestatehandlers.go rename to job/replication/delete.go index a4b07d8c6..99e2561bb 100644 --- a/job/replication/deletestatehandlers.go +++ b/job/replication/delete.go @@ -42,6 +42,18 @@ type Deleter struct { logger *log.Logger } +// NewDeleter returns a Deleter +func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, logger *log.Logger) *Deleter { + return &Deleter{ + repository: repository, + tags: tags, + dstURL: dstURL, + dstUsr: dstUsr, + dstPwd: dstPwd, + logger: logger, + } +} + // Exit ... func (d *Deleter) Exit() error { return nil diff --git a/job/replication/statehandlers.go b/job/replication/transfer.go similarity index 100% rename from job/replication/statehandlers.go rename to job/replication/transfer.go diff --git a/job/statemachine.go b/job/statemachine.go index 6a11391c9..9968629d6 100644 --- a/job/statemachine.go +++ b/job/statemachine.go @@ -207,23 +207,19 @@ func (sm *JobSM) Reset(jid int64) error { sm.Handlers[models.JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobError} sm.Handlers[models.JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobStopped} - if sm.Parms.Operation == models.RepOpTransfer { - /* - sm.AddTransition(models.JobRunning, "pull-img", ImgPuller{DummyHandler: DummyHandler{JobID: sm.JobID}, img: sm.Parms.Repository, logger: sm.Logger}) - //only handle one target for now - sm.AddTransition("pull-img", "push-img", ImgPusher{DummyHandler: DummyHandler{JobID: sm.JobID}, targetURL: sm.Parms.TargetURL, logger: sm.Logger}) - sm.AddTransition("push-img", models.JobFinished, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished}) - */ - - if err = addImgOutTransition(sm); err != nil { - return err - } - + switch sm.Parms.Operation { + case models.RepOpTransfer: + err = addImgTransferTransition(sm) + case models.RepOpDelete: + err = addImgDeleteTransition(sm) + default: + err = fmt.Errorf("unsupported operation: %s", sm.Parms.Operation) } - return nil + + return err } -func addImgOutTransition(sm *JobSM) error { +func addImgTransferTransition(sm *JobSM) error { base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(), sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword, nil, sm.Logger) @@ -238,3 +234,13 @@ func addImgOutTransition(sm *JobSM) error { sm.AddTransition(replication.StatePushManifest, replication.StatePullManifest, &replication.ManifestPuller{BaseHandler: base}) return nil } + +func addImgDeleteTransition(sm *JobSM) error { + deleter := replication.NewDeleter(sm.Parms.Repository, nil, sm.Parms.TargetURL, + sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Logger) + + sm.AddTransition(models.JobRunning, replication.StateDelete, deleter) + sm.AddTransition(replication.StateDelete, models.JobFinished, &StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished}) + + return nil +} From 2a0571fc64a8faf6f788d646f254236a3a81402b Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 26 May 2016 13:26:10 +0800 Subject: [PATCH 12/16] remove goto --- api/repository.go | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/api/repository.go b/api/repository.go index 961da1a88..ed40644a0 100644 --- a/api/repository.go +++ b/api/repository.go @@ -105,7 +105,7 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") } - rc, err := ra.initializeRepositoryClient(repoName) + rc, err := ra.initRepositoryClient(repoName) if err != nil { log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) ra.CustomAbort(http.StatusInternalServerError, "internal error") @@ -164,7 +164,7 @@ func (ra *RepositoryAPI) GetTags() { ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") } - rc, err := ra.initializeRepositoryClient(repoName) + rc, err := ra.initRepositoryClient(repoName) if err != nil { log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) ra.CustomAbort(http.StatusInternalServerError, "internal error") @@ -200,7 +200,7 @@ func (ra *RepositoryAPI) GetManifests() { ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil") } - rc, err := ra.initializeRepositoryClient(repoName) + rc, err := ra.initRepositoryClient(repoName) if err != nil { log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) ra.CustomAbort(http.StatusInternalServerError, "internal error") @@ -240,28 +240,36 @@ func (ra *RepositoryAPI) GetManifests() { ra.ServeJSON() } -func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) { - var username string - var sessionUsername, sessionUserID interface{} +func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { + username, err := ra.getUsername() + if err != nil { + return nil, err + } + endpoint := os.Getenv("REGISTRY_URL") + + return registry.NewRepositoryWithUsername(repoName, endpoint, username) +} + +func (ra *RepositoryAPI) getUsername() (string, error) { // get username from basic auth username, _, ok := ra.Ctx.Request.BasicAuth() if ok { - goto enter + return username, nil } // get username from session - sessionUsername = ra.GetSession("username") + sessionUsername := ra.GetSession("username") if sessionUsername != nil { username, ok = sessionUsername.(string) if ok { - goto enter + return username, nil } } // if username does not exist in session, try to get userId from sessiion // and then get username from DB according to the userId - sessionUserID = ra.GetSession("userId") + sessionUserID := ra.GetSession("userId") if sessionUserID != nil { userID, ok := sessionUserID.(int) if ok { @@ -270,15 +278,12 @@ func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registr } user, err := dao.GetUser(u) if err != nil { - return nil, err + return "", err } - username = user.Username - goto enter + + return user.Username, nil } } -enter: - endpoint := os.Getenv("REGISTRY_URL") - - return registry.NewRepositoryWithUsername(repoName, endpoint, username) + return "", nil } From 6b2b6dedd808d56619491634ceb9bc26e887cff1 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 26 May 2016 13:59:03 +0800 Subject: [PATCH 13/16] support Get /api/projects/:id --- api/project.go | 19 +++++++++++++++++++ ui/router.go | 1 + 2 files changed, 20 insertions(+) diff --git a/api/project.go b/api/project.go index 0a80db759..5d2a3cf1f 100644 --- a/api/project.go +++ b/api/project.go @@ -127,6 +127,25 @@ func (p *ProjectAPI) Head() { // Get ... func (p *ProjectAPI) Get() { + project, err := dao.GetProjectByID(p.projectID) + if err != nil { + log.Errorf("failed to get project %d: %v", p.projectID, err) + p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if project.Public == 0 { + userID := p.ValidateUser() + if !checkProjectPermission(userID, p.projectID) { + p.CustomAbort(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized)) + } + } + + p.Data["json"] = project + p.ServeJSON() +} + +// List ... +func (p *ProjectAPI) List() { var projectList []models.Project projectName := p.GetString("project_name") if len(projectName) > 0 { diff --git a/ui/router.go b/ui/router.go index 1218424c2..49c139288 100644 --- a/ui/router.go +++ b/ui/router.go @@ -53,6 +53,7 @@ func initRouters() { //API: beego.Router("/api/search", &api.SearchAPI{}) beego.Router("/api/projects/:pid/members/?:mid", &api.ProjectMemberAPI{}) + beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List") beego.Router("/api/projects/?:id", &api.ProjectAPI{}) beego.Router("/api/statistics", &api.StatisticAPI{}) beego.Router("/api/projects/:id/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog") From cdac1149555e55ef6caa97c2c3fb9204a239683c Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 27 May 2016 10:45:21 +0800 Subject: [PATCH 14/16] support tag transfer and deletion --- api/replication_policy.go | 4 ++-- api/repository.go | 2 ++ api/target.go | 24 +++++++++++------------ api/utils.go | 38 ++++++++++++++++++++++++++++--------- job/replication/transfer.go | 2 +- job/statemachine.go | 19 ++++++++++++++++--- service/notification.go | 2 +- ui/router.go | 4 ++++ utils/encrypt.go | 12 ++++++++++++ 9 files changed, 78 insertions(+), 29 deletions(-) diff --git a/api/replication_policy.go b/api/replication_policy.go index 2e78100e8..0324a968e 100644 --- a/api/replication_policy.go +++ b/api/replication_policy.go @@ -74,7 +74,7 @@ func (pa *RepPolicyAPI) Post() { } go func() { - if err := TriggerReplication(pid, "", models.RepOpTransfer); err != nil { + if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil { log.Errorf("failed to trigger replication of %d: %v", pid, err) } else { log.Infof("replication of %d triggered", pid) @@ -108,7 +108,7 @@ func (pa *RepPolicyAPI) UpdateEnablement() { if e.Enabled == 1 { go func() { - if err := TriggerReplication(pa.policyID, "", models.RepOpTransfer); err != nil { + if err := TriggerReplication(pa.policyID, "", nil, models.RepOpTransfer); err != nil { log.Errorf("failed to trigger replication of %d: %v", pa.policyID, err) } else { log.Infof("replication of %d triggered", pa.policyID) diff --git a/api/repository.go b/api/repository.go index 667f2105a..daef3c6f7 100644 --- a/api/repository.go +++ b/api/repository.go @@ -145,6 +145,8 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusInternalServerError, "internal error") } log.Infof("delete tag: %s %s", repoName, t) + go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete) + } go func() { diff --git a/api/target.go b/api/target.go index 7d925b3ca..c8cb443d9 100644 --- a/api/target.go +++ b/api/target.go @@ -16,7 +16,6 @@ package api import ( - "encoding/base64" "fmt" "net" "net/http" @@ -25,6 +24,7 @@ import ( "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils" "github.com/vmware/harbor/utils/log" registry_util "github.com/vmware/harbor/utils/registry" "github.com/vmware/harbor/utils/registry/auth" @@ -76,13 +76,11 @@ func (t *TargetAPI) Ping() { password = target.Password if len(password) != 0 { - b, err := base64.StdEncoding.DecodeString(password) + password, err = utils.ReversibleDecrypt(password) if err != nil { - log.Errorf("failed to decode password: %v", err) + log.Errorf("failed to decrypt password: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - - password = string(b) } } else { endpoint = t.GetString("endpoint") @@ -142,12 +140,12 @@ func (t *TargetAPI) Get() { for _, target := range targets { if len(target.Password) != 0 { - b, err := base64.StdEncoding.DecodeString(target.Password) + str, err := utils.ReversibleDecrypt(target.Password) if err != nil { - log.Errorf("failed to decode password: %v", err) + log.Errorf("failed to decrypt password: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - target.Password = string(b) + target.Password = str } } @@ -167,12 +165,12 @@ func (t *TargetAPI) Get() { } if len(target.Password) != 0 { - b, err := base64.StdEncoding.DecodeString(target.Password) + pwd, err := utils.ReversibleDecrypt(target.Password) if err != nil { - log.Errorf("failed to decode password: %v", err) + log.Errorf("failed to decrypt password: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - target.Password = string(b) + target.Password = pwd } t.Data["json"] = target @@ -189,7 +187,7 @@ func (t *TargetAPI) Post() { } if len(target.Password) != 0 { - target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password)) + target.Password = utils.ReversibleEncrypt(target.Password) } id, err := dao.AddRepTarget(*target) @@ -216,7 +214,7 @@ func (t *TargetAPI) Put() { } if len(target.Password) != 0 { - target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password)) + target.Password = utils.ReversibleEncrypt(target.Password) } if err := dao.UpdateRepTarget(*target); err != nil { diff --git a/api/utils.go b/api/utils.go index 575132d93..af1f1755a 100644 --- a/api/utils.go +++ b/api/utils.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "net/http" + "os" "strings" "github.com/vmware/harbor/dao" @@ -90,18 +91,21 @@ func checkUserExists(name string) int { } // TriggerReplication triggers the replication according to the policy -func TriggerReplication(policyID int64, repository, operation string) error { +func TriggerReplication(policyID int64, repository string, + tags []string, operation string) error { data := struct { - PolicyID int64 `json:"policy_id"` - Repo string `json:"repository"` - Operation string `json:"operation"` + PolicyID int64 `json:"policy_id"` + Repo string `json:"repository"` + Operation string `json:"operation"` + TagList []string `json:"tags"` }{ PolicyID: policyID, Repo: repository, + TagList: tags, Operation: operation, } - b, err := json.Marshal(data) + b, err := json.Marshal(&data) if err != nil { return err } @@ -146,7 +150,7 @@ func GetPoliciesByRepository(repository string) ([]*models.RepPolicy, error) { return policies, nil } -func TriggerReplicationByRepository(repository, operation string) { +func TriggerReplicationByRepository(repository string, tags []string, operation string) { policies, err := GetPoliciesByRepository(repository) if err != nil { log.Errorf("failed to get policies for repository %s: %v", repository, err) @@ -154,7 +158,7 @@ func TriggerReplicationByRepository(repository, operation string) { } for _, policy := range policies { - if err := TriggerReplication(policy.ProjectID, repository, operation); err != nil { + if err := TriggerReplication(policy.ProjectID, repository, tags, operation); err != nil { log.Errorf("failed to trigger replication of %d for %s: %v", policy.ID, repository, err) } else { log.Infof("replication of %d for %s triggered", policy.ID, repository) @@ -163,9 +167,25 @@ func TriggerReplicationByRepository(repository, operation string) { } func buildReplicationURL() string { - return "http://job_service/api/replicationJobs" + url := getJobServiceURL() + url = strings.TrimSpace(url) + url = strings.TrimRight(url, "/") + + return fmt.Sprintf("%s/api/replicationJobs", url) } func buildJobLogURL(jobID string) string { - return fmt.Sprintf("http://job_service/api/replicationJobs/%s/log", jobID) + url := getJobServiceURL() + url = strings.TrimSpace(url) + url = strings.TrimRight(url, "/") + + return fmt.Sprintf("%s/api/replicationJobs/%s/log", url, jobID) +} + +func getJobServiceURL() string { + url := os.Getenv("JOB_SERVICE_URL") + if len(url) == 0 { + url = "http://job_service" + } + return url } diff --git a/job/replication/transfer.go b/job/replication/transfer.go index 4fce8f1ca..4ff66b424 100644 --- a/job/replication/transfer.go +++ b/job/replication/transfer.go @@ -153,7 +153,7 @@ func (c *Checker) Enter() (string, error) { c.logger.Infof("project %s already exists on %s", c.project, c.dstURL) if !canWrite { - err = fmt.Errorf("the user %s has no write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL) + err = fmt.Errorf("the user %s is unauthorized to write to project %s on %s", c.dstUsr, c.project, c.dstURL) c.logger.Errorf("%v", err) return "", err } diff --git a/job/statemachine.go b/job/statemachine.go index 9968629d6..9a289994e 100644 --- a/job/statemachine.go +++ b/job/statemachine.go @@ -9,6 +9,7 @@ import ( "github.com/vmware/harbor/job/replication" "github.com/vmware/harbor/job/utils" "github.com/vmware/harbor/models" + uti "github.com/vmware/harbor/utils" "github.com/vmware/harbor/utils/log" ) @@ -18,6 +19,7 @@ type RepJobParm struct { TargetUsername string TargetPassword string Repository string + Tags []string Enabled int Operation string } @@ -182,6 +184,7 @@ func (sm *JobSM) Reset(jid int64) error { sm.Parms = &RepJobParm{ LocalRegURL: config.LocalHarborURL(), Repository: job.Repository, + Tags: job.TagList, Enabled: policy.Enabled, Operation: job.Operation, } @@ -198,7 +201,17 @@ func (sm *JobSM) Reset(jid int64) error { } sm.Parms.TargetURL = target.URL sm.Parms.TargetUsername = target.Username - sm.Parms.TargetPassword = target.Password + pwd := target.Password + + if len(pwd) != 0 { + pwd, err = uti.ReversibleDecrypt(pwd) + if err != nil { + return fmt.Errorf("failed to decrypt password: %v", err) + } + } + + sm.Parms.TargetPassword = pwd + //init states handlers sm.Logger = utils.NewLogger(sm.JobID) sm.CurrentState = models.JobPending @@ -222,7 +235,7 @@ func (sm *JobSM) Reset(jid int64) error { func addImgTransferTransition(sm *JobSM) error { base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(), sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword, - nil, sm.Logger) + sm.Parms.Tags, sm.Logger) if err != nil { return err } @@ -236,7 +249,7 @@ func addImgTransferTransition(sm *JobSM) error { } func addImgDeleteTransition(sm *JobSM) error { - deleter := replication.NewDeleter(sm.Parms.Repository, nil, sm.Parms.TargetURL, + deleter := replication.NewDeleter(sm.Parms.Repository, sm.Parms.Tags, sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Logger) sm.AddTransition(models.JobRunning, replication.StateDelete, deleter) diff --git a/service/notification.go b/service/notification.go index a464665a2..0f470e2db 100644 --- a/service/notification.go +++ b/service/notification.go @@ -79,7 +79,7 @@ func (n *NotificationHandler) Post() { } }() - go api.TriggerReplicationByRepository(repo, models.RepOpTransfer) + go api.TriggerReplicationByRepository(repo, []string{repoTag}, models.RepOpTransfer) } } } diff --git a/ui/router.go b/ui/router.go index 0b77df759..961e169ee 100644 --- a/ui/router.go +++ b/ui/router.go @@ -63,6 +63,10 @@ func initRouters() { beego.Router("/api/repositories", &api.RepositoryAPI{}) beego.Router("/api/repositories/tags", &api.RepositoryAPI{}, "get:GetTags") beego.Router("/api/repositories/manifests", &api.RepositoryAPI{}, "get:GetManifests") + beego.Router("/api/replicationJobs", &api.RepJobAPI{}) + beego.Router("/api/replicationJobs/:id/log", &api.RepJobAPI{}, "get:GetLog") + beego.Router("/api/replicationPolicies", &api.RepPolicyAPI{}) + beego.Router("/api/replicationPolicies/:id/enablement", &api.RepPolicyAPI{}, "put:UpdateEnablement") beego.Router("/api/targets/?:id", &api.TargetAPI{}) beego.Router("/api/target_ping", &api.TargetAPI{}, "get:Ping") diff --git a/utils/encrypt.go b/utils/encrypt.go index 2b85e8d97..e5ce44792 100644 --- a/utils/encrypt.go +++ b/utils/encrypt.go @@ -17,6 +17,7 @@ package utils import ( "crypto/sha1" + "encoding/base64" "fmt" "golang.org/x/crypto/pbkdf2" @@ -26,3 +27,14 @@ import ( func Encrypt(content string, salt string) string { return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, sha1.New)) } + +// ReversibleEncrypt encrypts the str with base64 +func ReversibleEncrypt(str string) string { + return base64.StdEncoding.EncodeToString([]byte(str)) +} + +// ReversibleDecrypt decrypts the str with base64 +func ReversibleDecrypt(str string) (string, error) { + b, err := base64.StdEncoding.DecodeString(str) + return string(b), err +} From 119dd50418eb9c0098d1ec083d4cfa3ec7d2de5c Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 27 May 2016 12:18:12 +0800 Subject: [PATCH 15/16] buf fix: create repository client with credential when basic auth is used --- api/repository.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/api/repository.go b/api/repository.go index ed40644a0..a355d73dc 100644 --- a/api/repository.go +++ b/api/repository.go @@ -30,6 +30,7 @@ import ( svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry" + "github.com/vmware/harbor/utils/registry/auth" "github.com/vmware/harbor/utils/registry/errors" ) @@ -241,27 +242,27 @@ func (ra *RepositoryAPI) GetManifests() { } func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { - username, err := ra.getUsername() + endpoint := os.Getenv("REGISTRY_URL") + + username, password, ok := ra.Ctx.Request.BasicAuth() + if ok { + credential := auth.NewBasicAuthCredential(username, password) + return registry.NewRepositoryWithCredential(repoName, endpoint, credential) + } + + username, err = ra.getUsername() if err != nil { return nil, err } - endpoint := os.Getenv("REGISTRY_URL") - return registry.NewRepositoryWithUsername(repoName, endpoint, username) } func (ra *RepositoryAPI) getUsername() (string, error) { - // get username from basic auth - username, _, ok := ra.Ctx.Request.BasicAuth() - if ok { - return username, nil - } - // get username from session sessionUsername := ra.GetSession("username") if sessionUsername != nil { - username, ok = sessionUsername.(string) + username, ok := sessionUsername.(string) if ok { return username, nil } From 1e09cf899bd993412c81e1fa3271121f7c4a6f6c Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 27 May 2016 15:04:20 +0800 Subject: [PATCH 16/16] fix --- api/replication_job.go | 4 ++++ api/replication_policy.go | 32 ++++++++++++++++++-------------- api/utils.go | 2 +- dao/replication_job.go | 15 +++++++-------- job/replication/delete.go | 5 ++++- job/statemachine.go | 2 ++ service/notification.go | 5 +++++ 7 files changed, 41 insertions(+), 24 deletions(-) diff --git a/api/replication_job.go b/api/replication_job.go index c15e1f1f8..877307f48 100644 --- a/api/replication_job.go +++ b/api/replication_job.go @@ -3,6 +3,7 @@ package api import ( "io/ioutil" "net/http" + "strconv" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/utils/log" @@ -62,6 +63,9 @@ func (ja *RepJobAPI) GetLog() { } if resp.StatusCode == http.StatusOK { + ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Disposition"), "attachment; filename=replication_job.log") + ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))) + ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(b))) if _, err = ja.Ctx.ResponseWriter.Write(b); err != nil { log.Errorf("failed to write log to response; %v", err) ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) diff --git a/api/replication_policy.go b/api/replication_policy.go index 0324a968e..6b27996bf 100644 --- a/api/replication_policy.go +++ b/api/replication_policy.go @@ -14,6 +14,7 @@ import ( type RepPolicyAPI struct { BaseAPI policyID int64 + policy *models.RepPolicy } func (pa *RepPolicyAPI) Prepare() { @@ -41,6 +42,7 @@ func (pa *RepPolicyAPI) Prepare() { if p == nil { pa.CustomAbort(http.StatusNotFound, fmt.Sprintf("policy does not exist, id: %v", pa.policyID)) } + pa.policy = p } } @@ -73,13 +75,15 @@ func (pa *RepPolicyAPI) Post() { return } - go func() { - if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil { - log.Errorf("failed to trigger replication of %d: %v", pid, err) - } else { - log.Infof("replication of %d triggered", pid) - } - }() + if policy.Enabled == 1 { + go func() { + if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil { + log.Errorf("failed to trigger replication of %d: %v", pid, err) + } else { + log.Infof("replication of %d triggered", pid) + } + }() + } pa.Redirect(http.StatusCreated, strconv.FormatInt(pid, 10)) } @@ -90,17 +94,17 @@ type enablementReq struct { func (pa *RepPolicyAPI) UpdateEnablement() { e := enablementReq{} - var err error pa.DecodeJSONReq(&e) - if e.Enabled == 1 { - err = dao.EnableRepPolicy(pa.policyID) - } else if e.Enabled == 0 { - err = dao.DisableRepPolicy(pa.policyID) - } else { + if e.Enabled != 0 && e.Enabled != 1 { pa.RenderError(http.StatusBadRequest, "invalid enabled value") return } - if err != nil { + + if pa.policy.Enabled == e.Enabled { + return + } + + if err := dao.UpdateRepPolicyEnablement(pa.policyID, e.Enabled); err != nil { log.Errorf("Failed to update policy enablement in DB, error: %v", err) pa.RenderError(http.StatusInternalServerError, "Internal Error") return diff --git a/api/utils.go b/api/utils.go index af1f1755a..2322dd089 100644 --- a/api/utils.go +++ b/api/utils.go @@ -158,7 +158,7 @@ func TriggerReplicationByRepository(repository string, tags []string, operation } for _, policy := range policies { - if err := TriggerReplication(policy.ProjectID, repository, tags, operation); err != nil { + if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil { log.Errorf("failed to trigger replication of %d for %s: %v", policy.ID, repository, err) } else { log.Infof("replication of %d for %s triggered", policy.ID, repository) diff --git a/dao/replication_job.go b/dao/replication_job.go index f4a005453..2a33d5df5 100644 --- a/dao/replication_job.go +++ b/dao/replication_job.go @@ -3,9 +3,10 @@ package dao import ( "fmt" + "strings" + "github.com/astaxie/beego/orm" "github.com/vmware/harbor/models" - "strings" ) func AddRepTarget(target models.RepTarget) (int64, error) { @@ -81,23 +82,21 @@ func DeleteRepPolicy(id int64) error { _, err := o.Delete(&models.RepPolicy{ID: id}) return err } -func updateRepPolicyEnablement(id int64, enabled int) error { +func UpdateRepPolicyEnablement(id int64, enabled int) error { o := orm.NewOrm() p := models.RepPolicy{ ID: id, Enabled: enabled} - num, err := o.Update(&p, "Enabled") - if num == 0 { - err = fmt.Errorf("Failed to update replication policy with id: %d", id) - } + _, err := o.Update(&p, "Enabled") + return err } func EnableRepPolicy(id int64) error { - return updateRepPolicyEnablement(id, 1) + return UpdateRepPolicyEnablement(id, 1) } func DisableRepPolicy(id int64) error { - return updateRepPolicyEnablement(id, 0) + return UpdateRepPolicyEnablement(id, 0) } func AddRepJob(job models.RepJob) (int64, error) { diff --git a/job/replication/delete.go b/job/replication/delete.go index 99e2561bb..cc4fd0d5a 100644 --- a/job/replication/delete.go +++ b/job/replication/delete.go @@ -44,7 +44,7 @@ type Deleter struct { // NewDeleter returns a Deleter func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, logger *log.Logger) *Deleter { - return &Deleter{ + deleter := &Deleter{ repository: repository, tags: tags, dstURL: dstURL, @@ -52,6 +52,9 @@ func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, dstPwd: dstPwd, logger: logger, } + deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, destination user: %s", + deleter.repository, deleter.tags, deleter.dstURL, deleter.dstUsr) + return deleter } // Exit ... diff --git a/job/statemachine.go b/job/statemachine.go index 9a289994e..99ef38124 100644 --- a/job/statemachine.go +++ b/job/statemachine.go @@ -214,6 +214,8 @@ func (sm *JobSM) Reset(jid int64) error { //init states handlers sm.Logger = utils.NewLogger(sm.JobID) + sm.Handlers = make(map[string]StateHandler) + sm.Transitions = make(map[string]map[string]struct{}) sm.CurrentState = models.JobPending sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning}) diff --git a/service/notification.go b/service/notification.go index 0f470e2db..586f5151d 100644 --- a/service/notification.go +++ b/service/notification.go @@ -70,6 +70,11 @@ func (n *NotificationHandler) Post() { if username == "" { username = "anonymous" } + + if username == "job-service-user" { + return + } + go dao.AccessLog(username, project, repo, repoTag, action) if action == "push" { go func() {