diff --git a/Deploy/log/Dockerfile b/Deploy/log/Dockerfile index 18a86b0af..3f02ad542 100644 --- a/Deploy/log/Dockerfile +++ b/Deploy/log/Dockerfile @@ -2,15 +2,9 @@ FROM library/ubuntu:14.04 # run logrotate hourly, disable imklog model, provides TCP/UDP syslog reception RUN mv /etc/cron.daily/logrotate /etc/cron.hourly/ \ - && sed 's/$ModLoad imklog/#$ModLoad imklog/' -i /etc/rsyslog.conf \ - && sed 's/$KLogPermitNonKernelFacility on/#$KLogPermitNonKernelFacility on/' -i /etc/rsyslog.conf \ - && sed 's/#$ModLoad imudp/$ModLoad imudp/' -i /etc/rsyslog.conf \ - && sed 's/#$UDPServerRun 514/$UDPServerRun 514/' -i /etc/rsyslog.conf \ - && sed 's/#$ModLoad imtcp/$ModLoad imtcp/' -i /etc/rsyslog.conf \ - && sed 's/#$InputTCPServerRun 514/$InputTCPServerRun 514/' -i /etc/rsyslog.conf \ - && sed 's/$PrivDropToUser syslog/#$PrivDropToUser syslog/' -i /etc/rsyslog.conf \ - && sed 's/$PrivDropToGroup syslog/#$PrivDropToGroup syslog/' -i /etc/rsyslog.conf \ - && rm /etc/rsyslog.d/* + && rm /etc/rsyslog.d/* \ + && rm /etc/rsyslog.conf +ADD rsyslog.conf /etc/rsyslog.conf # logrotate configuration file for docker ADD logrotate_docker.conf /etc/logrotate.d/ @@ -23,4 +17,3 @@ VOLUME /var/log/docker/ EXPOSE 514 CMD cron && rsyslogd -n - diff --git a/Deploy/log/rsyslog.conf b/Deploy/log/rsyslog.conf new file mode 100644 index 000000000..8b5c4fd84 --- /dev/null +++ b/Deploy/log/rsyslog.conf @@ -0,0 +1,60 @@ +# /etc/rsyslog.conf Configuration file for rsyslog. +# +# For more information see +# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html +# +# Default logging rules can be found in /etc/rsyslog.d/50-default.conf + + +################# +#### MODULES #### +################# + +$ModLoad imuxsock # provides support for local system logging +#$ModLoad imklog # provides kernel logging support +#$ModLoad immark # provides --MARK-- message capability + +# provides UDP syslog reception +$ModLoad imudp +$UDPServerRun 514 + +# provides TCP syslog reception +$ModLoad imtcp +$InputTCPServerRun 514 + +# Enable non-kernel facility klog messages +#$KLogPermitNonKernelFacility on + +########################### +#### GLOBAL DIRECTIVES #### +########################### + +# +# Use traditional timestamp format. +# To enable high precision timestamps, comment out the following line. +# +$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat + +# Filter duplicated messages +$RepeatedMsgReduction on + +# +# Set the default permissions for all log files. +# +$FileOwner syslog +$FileGroup adm +$FileCreateMode 0640 +$DirCreateMode 0755 +$Umask 0022 +#$PrivDropToUser syslog +#$PrivDropToGroup syslog + +# +# Where to place spool and state files +# +$WorkDirectory /var/spool/rsyslog + +# +# Include all config files in /etc/rsyslog.d/ +# +$IncludeConfig /etc/rsyslog.d/*.conf diff --git a/api/member.go b/api/member.go index 2ae7c9c53..b86e36f5d 100644 --- a/api/member.go +++ b/api/member.go @@ -114,23 +114,10 @@ func (pma *ProjectMemberAPI) Get() { // Post ... func (pma *ProjectMemberAPI) Post() { - pid := pma.project.ProjectID - - //userQuery := models.User{UserID: pma.currentUserID, RoleID: models.PROJECTADMIN} - rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid) - if err != nil { - log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) - pma.CustomAbort(http.StatusInternalServerError, "Internal error.") - } - hasProjectAdminRole := false - for _, role := range rolelist { - if role.RoleID == models.PROJECTADMIN { - hasProjectAdminRole = true - break - } - } - if !hasProjectAdminRole { - log.Warningf("Current user, id: %d does not have project admin role for project, id:", pma.currentUserID, pid) + currentUserID := pma.currentUserID + projectID := pma.project.ProjectID + if !hasProjectAdminRole(currentUserID, projectID) { + log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, projectID) pma.RenderError(http.StatusForbidden, "") return } @@ -144,21 +131,21 @@ func (pma *ProjectMemberAPI) Post() { pma.RenderError(http.StatusNotFound, "User does not exist") return } - rolelist, err = dao.GetUserProjectRoles(userID, pid) + rolelist, err := dao.GetUserProjectRoles(userID, projectID) if err != nil { log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) pma.CustomAbort(http.StatusInternalServerError, "Internal error.") } if len(rolelist) > 0 { - log.Warningf("user is already added to project, user id: %d, project id: %d", userID, pid) + log.Warningf("user is already added to project, user id: %d, project id: %d", userID, projectID) pma.RenderError(http.StatusConflict, "user is ready in project") return } for _, rid := range req.Roles { - err = dao.AddProjectMember(pid, userID, int(rid)) + err = dao.AddProjectMember(projectID, userID, int(rid)) if err != nil { - log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", pid, userID, rid) + log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", projectID, userID, rid) pma.RenderError(http.StatusInternalServerError, "Failed to update data in database") return } @@ -167,27 +154,16 @@ func (pma *ProjectMemberAPI) Post() { // Put ... func (pma *ProjectMemberAPI) Put() { + currentUserID := pma.currentUserID pid := pma.project.ProjectID - mid := pma.memberID - - rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid) - if err != nil { - log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) - pma.CustomAbort(http.StatusInternalServerError, "Internal error.") - } - hasProjectAdminRole := false - for _, role := range rolelist { - if role.RoleID == models.PROJECTADMIN { - hasProjectAdminRole = true - break - } - } - - if !hasProjectAdminRole { - log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid) + if !hasProjectAdminRole(currentUserID, pid) { + log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid) pma.RenderError(http.StatusForbidden, "") return } + + mid := pma.memberID + var req memberReq pma.DecodeJSONReq(&req) roleList, err := dao.GetUserProjectRoles(mid, pid) @@ -217,51 +193,20 @@ func (pma *ProjectMemberAPI) Put() { // Delete ... func (pma *ProjectMemberAPI) Delete() { + currentUserID := pma.currentUserID pid := pma.project.ProjectID - mid := pma.memberID - - rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid) - hasProjectAdminRole := false - for _, role := range rolelist { - if role.RoleID == models.PROJECTADMIN { - hasProjectAdminRole = true - break - } - } - - if !hasProjectAdminRole { - log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid) + if !hasProjectAdminRole(currentUserID, pid) { + log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid) pma.RenderError(http.StatusForbidden, "") return } - err = dao.DeleteProjectMember(pid, mid) + + mid := pma.memberID + + err := dao.DeleteProjectMember(pid, mid) if err != nil { log.Errorf("Failed to delete project roles for user, user id: %d, project id: %d, error: %v", mid, pid, err) pma.RenderError(http.StatusInternalServerError, "Failed to update data in DB") return } } - -//sysadmin has all privileges to all projects -func listRoles(userID int, projectID int64) ([]models.Role, error) { - roles := make([]models.Role, 1) - isSysAdmin, err := dao.IsAdminRole(userID) - if err != nil { - return roles, err - } - if isSysAdmin { - role, err := dao.GetRoleByID(models.PROJECTADMIN) - if err != nil { - return roles, err - } - roles = append(roles, *role) - return roles, nil - } - - rs, err := dao.GetUserProjectRoles(userID, projectID) - if err != nil { - return roles, err - } - roles = append(roles, rs...) - return roles, nil -} diff --git a/api/project.go b/api/project.go index 90ccbdb8c..5d2a3cf1f 100644 --- a/api/project.go +++ b/api/project.go @@ -43,7 +43,6 @@ const projectNameMaxLen int = 30 // Prepare validates the URL and the user func (p *ProjectAPI) Prepare() { - p.userID = p.ValidateUser() idStr := p.Ctx.Input.Param(":id") if len(idStr) > 0 { var err error @@ -65,6 +64,8 @@ func (p *ProjectAPI) Prepare() { // Post ... func (p *ProjectAPI) Post() { + p.userID = p.ValidateUser() + var req projectReq var public int p.DecodeJSONReq(&req) @@ -99,20 +100,52 @@ func (p *ProjectAPI) Post() { // Head ... func (p *ProjectAPI) Head() { projectName := p.GetString("project_name") - result, err := dao.ProjectExists(projectName) + if len(projectName) == 0 { + p.CustomAbort(http.StatusBadRequest, "project_name is needed") + } + + project, err := dao.GetProjectByName(projectName) if err != nil { - log.Errorf("Error while communicating with DB, error: %v", err) - p.RenderError(http.StatusInternalServerError, "Error while communicating with DB") + log.Errorf("error occurred in GetProjectByName: %v", err) + p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + // only public project can be Headed by user without login + if project != nil && project.Public == 1 { return } - if !result { - p.RenderError(http.StatusNotFound, "") - return + + userID := p.ValidateUser() + if project == nil { + p.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound)) + } + + if !checkProjectPermission(userID, project.ProjectID) { + p.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden)) } } // Get ... func (p *ProjectAPI) Get() { + project, err := dao.GetProjectByID(p.projectID) + if err != nil { + log.Errorf("failed to get project %d: %v", p.projectID, err) + p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if project.Public == 0 { + userID := p.ValidateUser() + if !checkProjectPermission(userID, p.projectID) { + p.CustomAbort(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized)) + } + } + + p.Data["json"] = project + p.ServeJSON() +} + +// List ... +func (p *ProjectAPI) List() { var projectList []models.Project projectName := p.GetString("project_name") if len(projectName) > 0 { @@ -132,6 +165,8 @@ func (p *ProjectAPI) Get() { if public == 1 { projectList, err = dao.GetPublicProjects(projectName) } else { + //if the request is not for public projects, user must login or provide credential + p.userID = p.ValidateUser() isAdmin, err = dao.IsAdminRole(p.userID) if err != nil { log.Errorf("Error occured in check admin, error: %v", err) @@ -164,6 +199,8 @@ func (p *ProjectAPI) Get() { // Put ... func (p *ProjectAPI) Put() { + p.userID = p.ValidateUser() + var req projectReq var public int @@ -192,6 +229,7 @@ func (p *ProjectAPI) Put() { // FilterAccessLog handles GET to /api/projects/{}/logs func (p *ProjectAPI) FilterAccessLog() { + p.userID = p.ValidateUser() var filter models.AccessLog p.DecodeJSONReq(&filter) diff --git a/api/replication_job.go b/api/replication_job.go index 0c6f0f67b..877307f48 100644 --- a/api/replication_job.go +++ b/api/replication_job.go @@ -1,9 +1,12 @@ package api import ( + "io/ioutil" + "net/http" + "strconv" + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/utils/log" - "net/http" ) type RepJobAPI struct { @@ -19,6 +22,7 @@ func (ja *RepJobAPI) Prepare() { if !isAdmin { ja.CustomAbort(http.StatusForbidden, "") } + } func (ja *RepJobAPI) Get() { @@ -38,4 +42,38 @@ func (ja *RepJobAPI) Get() { ja.ServeJSON() } +// GetLog ... +func (ja *RepJobAPI) GetLog() { + id := ja.Ctx.Input.Param(":id") + if len(id) == 0 { + ja.CustomAbort(http.StatusBadRequest, "id is nil") + } + + resp, err := http.Get(buildJobLogURL(id)) + if err != nil { + log.Errorf("failed to get log for job %s: %v", id, err) + ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Errorf("failed to read response body for job %s: %v", id, err) + ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + + if resp.StatusCode == http.StatusOK { + ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Disposition"), "attachment; filename=replication_job.log") + ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))) + ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(b))) + if _, err = ja.Ctx.ResponseWriter.Write(b); err != nil { + log.Errorf("failed to write log to response; %v", err) + ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) + } + return + } + + ja.CustomAbort(resp.StatusCode, string(b)) +} + //TODO:add Post handler to call job service API to submit jobs by policy diff --git a/api/replication_policy.go b/api/replication_policy.go index 174e898f0..6b27996bf 100644 --- a/api/replication_policy.go +++ b/api/replication_policy.go @@ -2,16 +2,19 @@ package api import ( "fmt" + + "net/http" + "strconv" + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" - "net/http" - "strconv" ) type RepPolicyAPI struct { BaseAPI policyID int64 + policy *models.RepPolicy } func (pa *RepPolicyAPI) Prepare() { @@ -39,6 +42,7 @@ func (pa *RepPolicyAPI) Prepare() { if p == nil { pa.CustomAbort(http.StatusNotFound, fmt.Sprintf("policy does not exist, id: %v", pa.policyID)) } + pa.policy = p } } @@ -70,6 +74,17 @@ func (pa *RepPolicyAPI) Post() { pa.RenderError(http.StatusInternalServerError, "Internal Error") return } + + if policy.Enabled == 1 { + go func() { + if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil { + log.Errorf("failed to trigger replication of %d: %v", pid, err) + } else { + log.Infof("replication of %d triggered", pid) + } + }() + } + pa.Redirect(http.StatusCreated, strconv.FormatInt(pid, 10)) } @@ -79,19 +94,29 @@ type enablementReq struct { func (pa *RepPolicyAPI) UpdateEnablement() { e := enablementReq{} - var err error pa.DecodeJSONReq(&e) - if e.Enabled == 1 { - err = dao.EnableRepPolicy(pa.policyID) - } else if e.Enabled == 0 { - err = dao.DisableRepPolicy(pa.policyID) - } else { + if e.Enabled != 0 && e.Enabled != 1 { pa.RenderError(http.StatusBadRequest, "invalid enabled value") return } - if err != nil { + + if pa.policy.Enabled == e.Enabled { + return + } + + if err := dao.UpdateRepPolicyEnablement(pa.policyID, e.Enabled); err != nil { log.Errorf("Failed to update policy enablement in DB, error: %v", err) pa.RenderError(http.StatusInternalServerError, "Internal Error") return } + + if e.Enabled == 1 { + go func() { + if err := TriggerReplication(pa.policyID, "", nil, models.RepOpTransfer); err != nil { + log.Errorf("failed to trigger replication of %d: %v", pa.policyID, err) + } else { + log.Infof("replication of %d triggered", pa.policyID) + } + }() + } } diff --git a/api/repository.go b/api/repository.go index b99685b65..3c0264ce3 100644 --- a/api/repository.go +++ b/api/repository.go @@ -19,6 +19,7 @@ import ( "encoding/json" "net/http" "os" + "sort" "strconv" "strings" "time" @@ -30,7 +31,10 @@ import ( svc_utils "github.com/vmware/harbor/service/utils" "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry" + registry_error "github.com/vmware/harbor/utils/registry/error" + + "github.com/vmware/harbor/utils/registry/auth" ) // RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put @@ -39,23 +43,13 @@ import ( // the security of registry type RepositoryAPI struct { BaseAPI - userID int -} - -// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. -func (ra *RepositoryAPI) Prepare() { - if svc_utils.VerifySecret(ra.Ctx.Request) { - ra.userID = 1 - } else { - ra.userID = ra.ValidateUser() - } } // Get ... func (ra *RepositoryAPI) Get() { - projectID, err0 := ra.GetInt64("project_id") - if err0 != nil { - log.Errorf("Failed to get project id, error: %v", err0) + projectID, err := ra.GetInt64("project_id") + if err != nil { + log.Errorf("Failed to get project id, error: %v", err) ra.RenderError(http.StatusBadRequest, "Invalid project id") return } @@ -69,9 +63,20 @@ func (ra *RepositoryAPI) Get() { ra.RenderError(http.StatusNotFound, "") return } - if p.Public == 0 && !checkProjectPermission(ra.userID, projectID) { - ra.RenderError(http.StatusForbidden, "") - return + + if p.Public == 0 { + var userID int + + if svc_utils.VerifySecret(ra.Ctx.Request) { + userID = 1 + } else { + userID = ra.ValidateUser() + } + + if !checkProjectPermission(userID, projectID) { + ra.RenderError(http.StatusForbidden, "") + return + } } repoList, err := cache.GetRepoFromCache() @@ -110,7 +115,7 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") } - rc, err := ra.initializeRepositoryClient(repoName) + rc, err := ra.initRepositoryClient(repoName) if err != nil { log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) ra.CustomAbort(http.StatusInternalServerError, "internal error") @@ -143,6 +148,8 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusInternalServerError, "internal error") } log.Infof("delete tag: %s %s", repoName, t) + go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete) + } go func() { @@ -166,7 +173,7 @@ func (ra *RepositoryAPI) GetTags() { ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") } - rc, err := ra.initializeRepositoryClient(repoName) + rc, err := ra.initRepositoryClient(repoName) if err != nil { log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) ra.CustomAbort(http.StatusInternalServerError, "internal error") @@ -186,6 +193,8 @@ func (ra *RepositoryAPI) GetTags() { tags = append(tags, ts...) + sort.Strings(tags) + ra.Data["json"] = tags ra.ServeJSON() } @@ -199,7 +208,7 @@ func (ra *RepositoryAPI) GetManifests() { ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil") } - rc, err := ra.initializeRepositoryClient(repoName) + rc, err := ra.initRepositoryClient(repoName) if err != nil { log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) ra.CustomAbort(http.StatusInternalServerError, "internal error") @@ -238,16 +247,50 @@ func (ra *RepositoryAPI) GetManifests() { ra.ServeJSON() } -func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) { - u := models.User{ - UserID: ra.userID, +func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) { + endpoint := os.Getenv("REGISTRY_URL") + + username, password, ok := ra.Ctx.Request.BasicAuth() + if ok { + credential := auth.NewBasicAuthCredential(username, password) + return registry.NewRepositoryWithCredential(repoName, endpoint, credential) } - user, err := dao.GetUser(u) + + username, err = ra.getUsername() if err != nil { return nil, err } - endpoint := os.Getenv("REGISTRY_URL") - - return registry.NewRepositoryWithUsername(repoName, endpoint, user.Username) + return registry.NewRepositoryWithUsername(repoName, endpoint, username) +} + +func (ra *RepositoryAPI) getUsername() (string, error) { + // get username from session + sessionUsername := ra.GetSession("username") + if sessionUsername != nil { + username, ok := sessionUsername.(string) + if ok { + return username, nil + } + } + + // if username does not exist in session, try to get userId from sessiion + // and then get username from DB according to the userId + sessionUserID := ra.GetSession("userId") + if sessionUserID != nil { + userID, ok := sessionUserID.(int) + if ok { + u := models.User{ + UserID: userID, + } + user, err := dao.GetUser(u) + if err != nil { + return "", err + } + + return user.Username, nil + } + } + + return "", nil } diff --git a/api/target.go b/api/target.go index 7d925b3ca..c8cb443d9 100644 --- a/api/target.go +++ b/api/target.go @@ -16,7 +16,6 @@ package api import ( - "encoding/base64" "fmt" "net" "net/http" @@ -25,6 +24,7 @@ import ( "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils" "github.com/vmware/harbor/utils/log" registry_util "github.com/vmware/harbor/utils/registry" "github.com/vmware/harbor/utils/registry/auth" @@ -76,13 +76,11 @@ func (t *TargetAPI) Ping() { password = target.Password if len(password) != 0 { - b, err := base64.StdEncoding.DecodeString(password) + password, err = utils.ReversibleDecrypt(password) if err != nil { - log.Errorf("failed to decode password: %v", err) + log.Errorf("failed to decrypt password: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - - password = string(b) } } else { endpoint = t.GetString("endpoint") @@ -142,12 +140,12 @@ func (t *TargetAPI) Get() { for _, target := range targets { if len(target.Password) != 0 { - b, err := base64.StdEncoding.DecodeString(target.Password) + str, err := utils.ReversibleDecrypt(target.Password) if err != nil { - log.Errorf("failed to decode password: %v", err) + log.Errorf("failed to decrypt password: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - target.Password = string(b) + target.Password = str } } @@ -167,12 +165,12 @@ func (t *TargetAPI) Get() { } if len(target.Password) != 0 { - b, err := base64.StdEncoding.DecodeString(target.Password) + pwd, err := utils.ReversibleDecrypt(target.Password) if err != nil { - log.Errorf("failed to decode password: %v", err) + log.Errorf("failed to decrypt password: %v", err) t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError)) } - target.Password = string(b) + target.Password = pwd } t.Data["json"] = target @@ -189,7 +187,7 @@ func (t *TargetAPI) Post() { } if len(target.Password) != 0 { - target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password)) + target.Password = utils.ReversibleEncrypt(target.Password) } id, err := dao.AddRepTarget(*target) @@ -216,7 +214,7 @@ func (t *TargetAPI) Put() { } if len(target.Password) != 0 { - target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password)) + target.Password = utils.ReversibleEncrypt(target.Password) } if err := dao.UpdateRepTarget(*target); err != nil { diff --git a/api/utils.go b/api/utils.go index 826484ac3..2322dd089 100644 --- a/api/utils.go +++ b/api/utils.go @@ -16,26 +16,66 @@ package api import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" ) func checkProjectPermission(userID int, projectID int64) bool { - exist, err := dao.IsAdminRole(userID) + roles, err := listRoles(userID, projectID) if err != nil { - log.Errorf("Error occurred in IsAdminRole, error: %v", err) + log.Errorf("error occurred in getProjectPermission: %v", err) return false } - if exist { - return true - } - roleList, err := dao.GetUserProjectRoles(userID, projectID) + return len(roles) > 0 +} + +func hasProjectAdminRole(userID int, projectID int64) bool { + roles, err := listRoles(userID, projectID) if err != nil { - log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err) + log.Errorf("error occurred in getProjectPermission: %v", err) return false } - return len(roleList) > 0 + + for _, role := range roles { + if role.RoleID == models.PROJECTADMIN { + return true + } + } + + return false +} + +//sysadmin has all privileges to all projects +func listRoles(userID int, projectID int64) ([]models.Role, error) { + roles := make([]models.Role, 1) + isSysAdmin, err := dao.IsAdminRole(userID) + if err != nil { + return roles, err + } + if isSysAdmin { + role, err := dao.GetRoleByID(models.PROJECTADMIN) + if err != nil { + return roles, err + } + roles = append(roles, *role) + return roles, nil + } + + rs, err := dao.GetUserProjectRoles(userID, projectID) + if err != nil { + return roles, err + } + roles = append(roles, rs...) + return roles, nil } func checkUserExists(name string) int { @@ -49,3 +89,103 @@ func checkUserExists(name string) int { } return 0 } + +// TriggerReplication triggers the replication according to the policy +func TriggerReplication(policyID int64, repository string, + tags []string, operation string) error { + data := struct { + PolicyID int64 `json:"policy_id"` + Repo string `json:"repository"` + Operation string `json:"operation"` + TagList []string `json:"tags"` + }{ + PolicyID: policyID, + Repo: repository, + TagList: tags, + Operation: operation, + } + + b, err := json.Marshal(&data) + if err != nil { + return err + } + + url := buildReplicationURL() + + resp, err := http.DefaultClient.Post(url, "application/json", bytes.NewBuffer(b)) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusOK { + return nil + } + + defer resp.Body.Close() + + b, err = ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return fmt.Errorf("%d %s", resp.StatusCode, string(b)) +} + +// GetPoliciesByRepository returns policies according the repository +func GetPoliciesByRepository(repository string) ([]*models.RepPolicy, error) { + repository = strings.TrimSpace(repository) + repository = strings.TrimRight(repository, "/") + projectName := repository[:strings.LastIndex(repository, "/")] + + project, err := dao.GetProjectByName(projectName) + if err != nil { + return nil, err + } + + policies, err := dao.GetRepPolicyByProject(project.ProjectID) + if err != nil { + return nil, err + } + + return policies, nil +} + +func TriggerReplicationByRepository(repository string, tags []string, operation string) { + policies, err := GetPoliciesByRepository(repository) + if err != nil { + log.Errorf("failed to get policies for repository %s: %v", repository, err) + return + } + + for _, policy := range policies { + if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil { + log.Errorf("failed to trigger replication of %d for %s: %v", policy.ID, repository, err) + } else { + log.Infof("replication of %d for %s triggered", policy.ID, repository) + } + } +} + +func buildReplicationURL() string { + url := getJobServiceURL() + url = strings.TrimSpace(url) + url = strings.TrimRight(url, "/") + + return fmt.Sprintf("%s/api/replicationJobs", url) +} + +func buildJobLogURL(jobID string) string { + url := getJobServiceURL() + url = strings.TrimSpace(url) + url = strings.TrimRight(url, "/") + + return fmt.Sprintf("%s/api/replicationJobs/%s/log", url, jobID) +} + +func getJobServiceURL() string { + url := os.Getenv("JOB_SERVICE_URL") + if len(url) == 0 { + url = "http://job_service" + } + return url +} diff --git a/dao/project.go b/dao/project.go index e444f8a26..60f32e367 100644 --- a/dao/project.go +++ b/dao/project.go @@ -249,7 +249,6 @@ func getProjects(public int, projectName string) ([]models.Project, error) { } sql += " order by name " var projects []models.Project - log.Debugf("sql xxx", sql) if _, err := o.Raw(sql, queryParam).QueryRows(&projects); err != nil { return nil, err } diff --git a/dao/replication_job.go b/dao/replication_job.go index f4a005453..2a33d5df5 100644 --- a/dao/replication_job.go +++ b/dao/replication_job.go @@ -3,9 +3,10 @@ package dao import ( "fmt" + "strings" + "github.com/astaxie/beego/orm" "github.com/vmware/harbor/models" - "strings" ) func AddRepTarget(target models.RepTarget) (int64, error) { @@ -81,23 +82,21 @@ func DeleteRepPolicy(id int64) error { _, err := o.Delete(&models.RepPolicy{ID: id}) return err } -func updateRepPolicyEnablement(id int64, enabled int) error { +func UpdateRepPolicyEnablement(id int64, enabled int) error { o := orm.NewOrm() p := models.RepPolicy{ ID: id, Enabled: enabled} - num, err := o.Update(&p, "Enabled") - if num == 0 { - err = fmt.Errorf("Failed to update replication policy with id: %d", id) - } + _, err := o.Update(&p, "Enabled") + return err } func EnableRepPolicy(id int64) error { - return updateRepPolicyEnablement(id, 1) + return UpdateRepPolicyEnablement(id, 1) } func DisableRepPolicy(id int64) error { - return updateRepPolicyEnablement(id, 0) + return UpdateRepPolicyEnablement(id, 0) } func AddRepJob(job models.RepJob) (int64, error) { diff --git a/job/replication/delete.go b/job/replication/delete.go new file mode 100644 index 000000000..cc4fd0d5a --- /dev/null +++ b/job/replication/delete.go @@ -0,0 +1,119 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package replication + +import ( + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" +) + +const ( + // StateDelete ... + StateDelete = "delete" +) + +// Deleter deletes repository or tags +type Deleter struct { + repository string // prject_name/repo_name + tags []string + + dstURL string // url of target registry + dstUsr string // username ... + dstPwd string // username ... + + logger *log.Logger +} + +// NewDeleter returns a Deleter +func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, logger *log.Logger) *Deleter { + deleter := &Deleter{ + repository: repository, + tags: tags, + dstURL: dstURL, + dstUsr: dstUsr, + dstPwd: dstPwd, + logger: logger, + } + deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, destination user: %s", + deleter.repository, deleter.tags, deleter.dstURL, deleter.dstUsr) + return deleter +} + +// Exit ... +func (d *Deleter) Exit() error { + return nil +} + +// Enter deletes repository or tags +func (d *Deleter) Enter() (string, error) { + url := strings.TrimRight(d.dstURL, "/") + "/api/repositories/" + + // delete repository + if len(d.tags) == 0 { + u := url + "?repo_name=" + d.repository + if err := del(u, d.dstUsr, d.dstPwd); err != nil { + d.logger.Errorf("an error occurred while deleting repository %s on %s with user %s: %v", d.repository, d.dstURL, d.dstUsr, err) + return "", err + } + + d.logger.Infof("repository %s on %s has been deleted", d.repository, d.dstURL) + + return models.JobFinished, nil + } + + // delele tags + for _, tag := range d.tags { + u := url + "?repo_name=" + d.repository + "&tag=" + tag + if err := del(u, d.dstUsr, d.dstPwd); err != nil { + d.logger.Errorf("an error occurred while deleting repository %s:%s on %s with user %s: %v", d.repository, tag, d.dstURL, d.dstUsr, err) + return "", err + } + + d.logger.Infof("repository %s:%s on %s has been deleted", d.repository, tag, d.dstURL) + } + + return models.JobFinished, nil +} + +func del(url, username, password string) error { + req, err := http.NewRequest("DELETE", url, nil) + if err != nil { + return err + } + + req.SetBasicAuth(username, password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusOK { + return nil + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return fmt.Errorf("%d %s", resp.StatusCode, string(b)) +} diff --git a/job/replication/statehandlers.go b/job/replication/transfer.go similarity index 99% rename from job/replication/statehandlers.go rename to job/replication/transfer.go index 4fce8f1ca..4ff66b424 100644 --- a/job/replication/statehandlers.go +++ b/job/replication/transfer.go @@ -153,7 +153,7 @@ func (c *Checker) Enter() (string, error) { c.logger.Infof("project %s already exists on %s", c.project, c.dstURL) if !canWrite { - err = fmt.Errorf("the user %s has no write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL) + err = fmt.Errorf("the user %s is unauthorized to write to project %s on %s", c.dstUsr, c.project, c.dstURL) c.logger.Errorf("%v", err) return "", err } diff --git a/job/statemachine.go b/job/statemachine.go index 6a11391c9..99ef38124 100644 --- a/job/statemachine.go +++ b/job/statemachine.go @@ -9,6 +9,7 @@ import ( "github.com/vmware/harbor/job/replication" "github.com/vmware/harbor/job/utils" "github.com/vmware/harbor/models" + uti "github.com/vmware/harbor/utils" "github.com/vmware/harbor/utils/log" ) @@ -18,6 +19,7 @@ type RepJobParm struct { TargetUsername string TargetPassword string Repository string + Tags []string Enabled int Operation string } @@ -182,6 +184,7 @@ func (sm *JobSM) Reset(jid int64) error { sm.Parms = &RepJobParm{ LocalRegURL: config.LocalHarborURL(), Repository: job.Repository, + Tags: job.TagList, Enabled: policy.Enabled, Operation: job.Operation, } @@ -198,35 +201,43 @@ func (sm *JobSM) Reset(jid int64) error { } sm.Parms.TargetURL = target.URL sm.Parms.TargetUsername = target.Username - sm.Parms.TargetPassword = target.Password + pwd := target.Password + + if len(pwd) != 0 { + pwd, err = uti.ReversibleDecrypt(pwd) + if err != nil { + return fmt.Errorf("failed to decrypt password: %v", err) + } + } + + sm.Parms.TargetPassword = pwd + //init states handlers sm.Logger = utils.NewLogger(sm.JobID) + sm.Handlers = make(map[string]StateHandler) + sm.Transitions = make(map[string]map[string]struct{}) sm.CurrentState = models.JobPending sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning}) sm.Handlers[models.JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobError} sm.Handlers[models.JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobStopped} - if sm.Parms.Operation == models.RepOpTransfer { - /* - sm.AddTransition(models.JobRunning, "pull-img", ImgPuller{DummyHandler: DummyHandler{JobID: sm.JobID}, img: sm.Parms.Repository, logger: sm.Logger}) - //only handle one target for now - sm.AddTransition("pull-img", "push-img", ImgPusher{DummyHandler: DummyHandler{JobID: sm.JobID}, targetURL: sm.Parms.TargetURL, logger: sm.Logger}) - sm.AddTransition("push-img", models.JobFinished, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished}) - */ - - if err = addImgOutTransition(sm); err != nil { - return err - } - + switch sm.Parms.Operation { + case models.RepOpTransfer: + err = addImgTransferTransition(sm) + case models.RepOpDelete: + err = addImgDeleteTransition(sm) + default: + err = fmt.Errorf("unsupported operation: %s", sm.Parms.Operation) } - return nil + + return err } -func addImgOutTransition(sm *JobSM) error { +func addImgTransferTransition(sm *JobSM) error { base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(), sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword, - nil, sm.Logger) + sm.Parms.Tags, sm.Logger) if err != nil { return err } @@ -238,3 +249,13 @@ func addImgOutTransition(sm *JobSM) error { sm.AddTransition(replication.StatePushManifest, replication.StatePullManifest, &replication.ManifestPuller{BaseHandler: base}) return nil } + +func addImgDeleteTransition(sm *JobSM) error { + deleter := replication.NewDeleter(sm.Parms.Repository, sm.Parms.Tags, sm.Parms.TargetURL, + sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Logger) + + sm.AddTransition(models.JobRunning, replication.StateDelete, deleter) + sm.AddTransition(replication.StateDelete, models.JobFinished, &StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished}) + + return nil +} diff --git a/service/notification.go b/service/notification.go index 4bfcf2e42..586f5151d 100644 --- a/service/notification.go +++ b/service/notification.go @@ -20,6 +20,7 @@ import ( "regexp" "strings" + "github.com/vmware/harbor/api" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/service/cache" @@ -69,6 +70,11 @@ func (n *NotificationHandler) Post() { if username == "" { username = "anonymous" } + + if username == "job-service-user" { + return + } + go dao.AccessLog(username, project, repo, repoTag, action) if action == "push" { go func() { @@ -77,6 +83,8 @@ func (n *NotificationHandler) Post() { log.Errorf("Error happens when refreshing cache: %v", err2) } }() + + go api.TriggerReplicationByRepository(repo, []string{repoTag}, models.RepOpTransfer) } } } diff --git a/service/token/authutils.go b/service/token/authutils.go index 3ea15c294..4c30648c4 100644 --- a/service/token/authutils.go +++ b/service/token/authutils.go @@ -46,10 +46,24 @@ func GetResourceActions(scopes []string) []*token.ResourceActions { continue } items := strings.Split(s, ":") + length := len(items) + + typee := items[0] + + name := "" + if length > 1 { + name = items[1] + } + + actions := []string{} + if length > 2 { + actions = strings.Split(items[2], ",") + } + res = append(res, &token.ResourceActions{ - Type: items[0], - Name: items[1], - Actions: strings.Split(items[2], ","), + Type: typee, + Name: name, + Actions: actions, }) } return res diff --git a/ui/router.go b/ui/router.go index 052a942a8..961e169ee 100644 --- a/ui/router.go +++ b/ui/router.go @@ -53,6 +53,7 @@ func initRouters() { //API: beego.Router("/api/search", &api.SearchAPI{}) beego.Router("/api/projects/:pid/members/?:mid", &api.ProjectMemberAPI{}) + beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List") beego.Router("/api/projects/?:id", &api.ProjectAPI{}) beego.Router("/api/statistics", &api.StatisticAPI{}) beego.Router("/api/projects/:id/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog") @@ -62,6 +63,10 @@ func initRouters() { beego.Router("/api/repositories", &api.RepositoryAPI{}) beego.Router("/api/repositories/tags", &api.RepositoryAPI{}, "get:GetTags") beego.Router("/api/repositories/manifests", &api.RepositoryAPI{}, "get:GetManifests") + beego.Router("/api/replicationJobs", &api.RepJobAPI{}) + beego.Router("/api/replicationJobs/:id/log", &api.RepJobAPI{}, "get:GetLog") + beego.Router("/api/replicationPolicies", &api.RepPolicyAPI{}) + beego.Router("/api/replicationPolicies/:id/enablement", &api.RepPolicyAPI{}, "put:UpdateEnablement") beego.Router("/api/targets/?:id", &api.TargetAPI{}) beego.Router("/api/target_ping", &api.TargetAPI{}, "get:Ping") diff --git a/utils/encrypt.go b/utils/encrypt.go index 2b85e8d97..e5ce44792 100644 --- a/utils/encrypt.go +++ b/utils/encrypt.go @@ -17,6 +17,7 @@ package utils import ( "crypto/sha1" + "encoding/base64" "fmt" "golang.org/x/crypto/pbkdf2" @@ -26,3 +27,14 @@ import ( func Encrypt(content string, salt string) string { return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, sha1.New)) } + +// ReversibleEncrypt encrypts the str with base64 +func ReversibleEncrypt(str string) string { + return base64.StdEncoding.EncodeToString([]byte(str)) +} + +// ReversibleDecrypt decrypts the str with base64 +func ReversibleDecrypt(str string) (string, error) { + b, err := base64.StdEncoding.DecodeString(str) + return string(b), err +}