Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-23 02:35:17 +01:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/job-service' into job-service

Browse Source
This commit is contained in:
Tan Jiang 2016-05-27 16:21:55 +08:00
commit 8817c24a27
18 changed files with 636 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
Deploy/log/Dockerfile
View File

@ -2,15 +2,9 @@ FROM library/ubuntu:14.04
# run logrotate hourly, disable imklog model, provides TCP/UDP syslog reception
RUN mv /etc/cron.daily/logrotate /etc/cron.hourly/ \
&& sed 's/$ModLoad imklog/#$ModLoad imklog/' -i /etc/rsyslog.conf \
&& sed 's/$KLogPermitNonKernelFacility on/#$KLogPermitNonKernelFacility on/' -i /etc/rsyslog.conf \
&& sed 's/#$ModLoad imudp/$ModLoad imudp/' -i /etc/rsyslog.conf \
&& sed 's/#$UDPServerRun 514/$UDPServerRun 514/' -i /etc/rsyslog.conf \
&& sed 's/#$ModLoad imtcp/$ModLoad imtcp/' -i /etc/rsyslog.conf \
&& sed 's/#$InputTCPServerRun 514/$InputTCPServerRun 514/' -i /etc/rsyslog.conf \
&& sed 's/$PrivDropToUser syslog/#$PrivDropToUser syslog/' -i /etc/rsyslog.conf \
&& sed 's/$PrivDropToGroup syslog/#$PrivDropToGroup syslog/' -i /etc/rsyslog.conf \
&& rm /etc/rsyslog.d/*
&& rm /etc/rsyslog.d/* \
&& rm /etc/rsyslog.conf
ADD rsyslog.conf /etc/rsyslog.conf
# logrotate configuration file for docker
ADD logrotate_docker.conf /etc/logrotate.d/
@ -23,4 +17,3 @@ VOLUME /var/log/docker/
EXPOSE 514
CMD cron && rsyslogd -n

60
Deploy/log/rsyslog.conf Normal file
View File

@ -0,0 +1,60 @@
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
#
# Default logging rules can be found in /etc/rsyslog.d/50-default.conf
#################
#### MODULES ####
#################
$ModLoad imuxsock # provides support for local system logging
#$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability
# provides UDP syslog reception
$ModLoad imudp
$UDPServerRun 514
# provides TCP syslog reception
$ModLoad imtcp
$InputTCPServerRun 514
# Enable non-kernel facility klog messages
#$KLogPermitNonKernelFacility on
###########################
#### GLOBAL DIRECTIVES ####
###########################
#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
# Filter duplicated messages
$RepeatedMsgReduction on
#
# Set the default permissions for all log files.
#
$FileOwner syslog
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
#$PrivDropToUser syslog
#$PrivDropToGroup syslog
#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog
#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf

97
api/member.go
View File

@ -114,23 +114,10 @@ func (pma *ProjectMemberAPI) Get() {
// Post ...
func (pma *ProjectMemberAPI) Post() {
pid := pma.project.ProjectID
//userQuery := models.User{UserID: pma.currentUserID, RoleID: models.PROJECTADMIN}
rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid)
if err != nil {
log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
pma.CustomAbort(http.StatusInternalServerError, "Internal error.")
}
hasProjectAdminRole := false
for _, role := range rolelist {
if role.RoleID == models.PROJECTADMIN {
hasProjectAdminRole = true
break
}
}
if !hasProjectAdminRole {
log.Warningf("Current user, id: %d does not have project admin role for project, id:", pma.currentUserID, pid)
currentUserID := pma.currentUserID
projectID := pma.project.ProjectID
if !hasProjectAdminRole(currentUserID, projectID) {
log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, projectID)
pma.RenderError(http.StatusForbidden, "")
return
}
@ -144,21 +131,21 @@ func (pma *ProjectMemberAPI) Post() {
pma.RenderError(http.StatusNotFound, "User does not exist")
return
}
rolelist, err = dao.GetUserProjectRoles(userID, pid)
rolelist, err := dao.GetUserProjectRoles(userID, projectID)
if err != nil {
log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
pma.CustomAbort(http.StatusInternalServerError, "Internal error.")
}
if len(rolelist) > 0 {
log.Warningf("user is already added to project, user id: %d, project id: %d", userID, pid)
log.Warningf("user is already added to project, user id: %d, project id: %d", userID, projectID)
pma.RenderError(http.StatusConflict, "user is ready in project")
return
}
for _, rid := range req.Roles {
err = dao.AddProjectMember(pid, userID, int(rid))
err = dao.AddProjectMember(projectID, userID, int(rid))
if err != nil {
log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", pid, userID, rid)
log.Errorf("Failed to update DB to add project user role, project id: %d, user id: %d, role id: %d", projectID, userID, rid)
pma.RenderError(http.StatusInternalServerError, "Failed to update data in database")
return
}
@ -167,27 +154,16 @@ func (pma *ProjectMemberAPI) Post() {
// Put ...
func (pma *ProjectMemberAPI) Put() {
currentUserID := pma.currentUserID
pid := pma.project.ProjectID
mid := pma.memberID
rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid)
if err != nil {
log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
pma.CustomAbort(http.StatusInternalServerError, "Internal error.")
}
hasProjectAdminRole := false
for _, role := range rolelist {
if role.RoleID == models.PROJECTADMIN {
hasProjectAdminRole = true
break
}
}
if !hasProjectAdminRole {
log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid)
if !hasProjectAdminRole(currentUserID, pid) {
log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid)
pma.RenderError(http.StatusForbidden, "")
return
}
mid := pma.memberID
var req memberReq
pma.DecodeJSONReq(&req)
roleList, err := dao.GetUserProjectRoles(mid, pid)
@ -217,51 +193,20 @@ func (pma *ProjectMemberAPI) Put() {
// Delete ...
func (pma *ProjectMemberAPI) Delete() {
currentUserID := pma.currentUserID
pid := pma.project.ProjectID
mid := pma.memberID
rolelist, err := dao.GetUserProjectRoles(pma.currentUserID, pid)
hasProjectAdminRole := false
for _, role := range rolelist {
if role.RoleID == models.PROJECTADMIN {
hasProjectAdminRole = true
break
}
}
if !hasProjectAdminRole {
log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", pma.currentUserID, pid)
if !hasProjectAdminRole(currentUserID, pid) {
log.Warningf("Current user, id: %d does not have project admin role for project, id:", currentUserID, pid)
pma.RenderError(http.StatusForbidden, "")
return
}
err = dao.DeleteProjectMember(pid, mid)
mid := pma.memberID
err := dao.DeleteProjectMember(pid, mid)
if err != nil {
log.Errorf("Failed to delete project roles for user, user id: %d, project id: %d, error: %v", mid, pid, err)
pma.RenderError(http.StatusInternalServerError, "Failed to update data in DB")
return
}
}
//sysadmin has all privileges to all projects
func listRoles(userID int, projectID int64) ([]models.Role, error) {
roles := make([]models.Role, 1)
isSysAdmin, err := dao.IsAdminRole(userID)
if err != nil {
return roles, err
}
if isSysAdmin {
role, err := dao.GetRoleByID(models.PROJECTADMIN)
if err != nil {
return roles, err
}
roles = append(roles, *role)
return roles, nil
}
rs, err := dao.GetUserProjectRoles(userID, projectID)
if err != nil {
return roles, err
}
roles = append(roles, rs...)
return roles, nil
}

52
api/project.go
View File

@ -43,7 +43,6 @@ const projectNameMaxLen int = 30
// Prepare validates the URL and the user
func (p *ProjectAPI) Prepare() {
p.userID = p.ValidateUser()
idStr := p.Ctx.Input.Param(":id")
if len(idStr) > 0 {
var err error
@ -65,6 +64,8 @@ func (p *ProjectAPI) Prepare() {
// Post ...
func (p *ProjectAPI) Post() {
p.userID = p.ValidateUser()
var req projectReq
var public int
p.DecodeJSONReq(&req)
@ -99,20 +100,52 @@ func (p *ProjectAPI) Post() {
// Head ...
func (p *ProjectAPI) Head() {
projectName := p.GetString("project_name")
result, err := dao.ProjectExists(projectName)
if len(projectName) == 0 {
p.CustomAbort(http.StatusBadRequest, "project_name is needed")
}
project, err := dao.GetProjectByName(projectName)
if err != nil {
log.Errorf("Error while communicating with DB, error: %v", err)
p.RenderError(http.StatusInternalServerError, "Error while communicating with DB")
log.Errorf("error occurred in GetProjectByName: %v", err)
p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
// only public project can be Headed by user without login
if project != nil && project.Public == 1 {
return
}
if !result {
p.RenderError(http.StatusNotFound, "")
return
userID := p.ValidateUser()
if project == nil {
p.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
}
if !checkProjectPermission(userID, project.ProjectID) {
p.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden))
}
}
// Get ...
func (p *ProjectAPI) Get() {
project, err := dao.GetProjectByID(p.projectID)
if err != nil {
log.Errorf("failed to get project %d: %v", p.projectID, err)
p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if project.Public == 0 {
userID := p.ValidateUser()
if !checkProjectPermission(userID, p.projectID) {
p.CustomAbort(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized))
}
}
p.Data["json"] = project
p.ServeJSON()
}
// List ...
func (p *ProjectAPI) List() {
var projectList []models.Project
projectName := p.GetString("project_name")
if len(projectName) > 0 {
@ -132,6 +165,8 @@ func (p *ProjectAPI) Get() {
if public == 1 {
projectList, err = dao.GetPublicProjects(projectName)
} else {
//if the request is not for public projects, user must login or provide credential
p.userID = p.ValidateUser()
isAdmin, err = dao.IsAdminRole(p.userID)
if err != nil {
log.Errorf("Error occured in check admin, error: %v", err)
@ -164,6 +199,8 @@ func (p *ProjectAPI) Get() {
// Put ...
func (p *ProjectAPI) Put() {
p.userID = p.ValidateUser()
var req projectReq
var public int
@ -192,6 +229,7 @@ func (p *ProjectAPI) Put() {
// FilterAccessLog handles GET to /api/projects/{}/logs
func (p *ProjectAPI) FilterAccessLog() {
p.userID = p.ValidateUser()
var filter models.AccessLog
p.DecodeJSONReq(&filter)

40
api/replication_job.go
View File

@ -1,9 +1,12 @@
package api
import (
"io/ioutil"
"net/http"
"strconv"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/utils/log"
"net/http"
)
type RepJobAPI struct {
@ -19,6 +22,7 @@ func (ja *RepJobAPI) Prepare() {
if !isAdmin {
ja.CustomAbort(http.StatusForbidden, "")
}
}
func (ja *RepJobAPI) Get() {
@ -38,4 +42,38 @@ func (ja *RepJobAPI) Get() {
ja.ServeJSON()
}
// GetLog ...
func (ja *RepJobAPI) GetLog() {
id := ja.Ctx.Input.Param(":id")
if len(id) == 0 {
ja.CustomAbort(http.StatusBadRequest, "id is nil")
}
resp, err := http.Get(buildJobLogURL(id))
if err != nil {
log.Errorf("failed to get log for job %s: %v", id, err)
ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Errorf("failed to read response body for job %s: %v", id, err)
ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if resp.StatusCode == http.StatusOK {
ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Disposition"), "attachment; filename=replication_job.log")
ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), resp.Header.Get(http.CanonicalHeaderKey("Content-Type")))
ja.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(b)))
if _, err = ja.Ctx.ResponseWriter.Write(b); err != nil {
log.Errorf("failed to write log to response; %v", err)
ja.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
return
}
ja.CustomAbort(resp.StatusCode, string(b))
}
//TODO:add Post handler to call job service API to submit jobs by policy

43
api/replication_policy.go
View File

@ -2,16 +2,19 @@ package api
import (
"fmt"
"net/http"
"strconv"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"net/http"
"strconv"
)
type RepPolicyAPI struct {
BaseAPI
policyID int64
policy *models.RepPolicy
}
func (pa *RepPolicyAPI) Prepare() {
@ -39,6 +42,7 @@ func (pa *RepPolicyAPI) Prepare() {
if p == nil {
pa.CustomAbort(http.StatusNotFound, fmt.Sprintf("policy does not exist, id: %v", pa.policyID))
}
pa.policy = p
}
}
@ -70,6 +74,17 @@ func (pa *RepPolicyAPI) Post() {
pa.RenderError(http.StatusInternalServerError, "Internal Error")
return
}
if policy.Enabled == 1 {
go func() {
if err := TriggerReplication(pid, "", nil, models.RepOpTransfer); err != nil {
log.Errorf("failed to trigger replication of %d: %v", pid, err)
} else {
log.Infof("replication of %d triggered", pid)
}
}()
}
pa.Redirect(http.StatusCreated, strconv.FormatInt(pid, 10))
}
@ -79,19 +94,29 @@ type enablementReq struct {
func (pa *RepPolicyAPI) UpdateEnablement() {
e := enablementReq{}
var err error
pa.DecodeJSONReq(&e)
if e.Enabled == 1 {
err = dao.EnableRepPolicy(pa.policyID)
} else if e.Enabled == 0 {
err = dao.DisableRepPolicy(pa.policyID)
} else {
if e.Enabled != 0 && e.Enabled != 1 {
pa.RenderError(http.StatusBadRequest, "invalid enabled value")
return
}
if err != nil {
if pa.policy.Enabled == e.Enabled {
return
}
if err := dao.UpdateRepPolicyEnablement(pa.policyID, e.Enabled); err != nil {
log.Errorf("Failed to update policy enablement in DB, error: %v", err)
pa.RenderError(http.StatusInternalServerError, "Internal Error")
return
}
if e.Enabled == 1 {
go func() {
if err := TriggerReplication(pa.policyID, "", nil, models.RepOpTransfer); err != nil {
log.Errorf("failed to trigger replication of %d: %v", pa.policyID, err)
} else {
log.Infof("replication of %d triggered", pa.policyID)
}
}()
}
}

95
api/repository.go
View File

@ -19,6 +19,7 @@ import (
"encoding/json"
"net/http"
"os"
"sort"
"strconv"
"strings"
"time"
@ -30,7 +31,10 @@ import (
svc_utils "github.com/vmware/harbor/service/utils"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry"
registry_error "github.com/vmware/harbor/utils/registry/error"
"github.com/vmware/harbor/utils/registry/auth"
)
// RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put
@ -39,23 +43,13 @@ import (
// the security of registry
type RepositoryAPI struct {
BaseAPI
userID int
}
// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission.
func (ra *RepositoryAPI) Prepare() {
if svc_utils.VerifySecret(ra.Ctx.Request) {
ra.userID = 1
} else {
ra.userID = ra.ValidateUser()
}
}
// Get ...
func (ra *RepositoryAPI) Get() {
projectID, err0 := ra.GetInt64("project_id")
if err0 != nil {
log.Errorf("Failed to get project id, error: %v", err0)
projectID, err := ra.GetInt64("project_id")
if err != nil {
log.Errorf("Failed to get project id, error: %v", err)
ra.RenderError(http.StatusBadRequest, "Invalid project id")
return
}
@ -69,9 +63,20 @@ func (ra *RepositoryAPI) Get() {
ra.RenderError(http.StatusNotFound, "")
return
}
if p.Public == 0 && !checkProjectPermission(ra.userID, projectID) {
ra.RenderError(http.StatusForbidden, "")
return
if p.Public == 0 {
var userID int
if svc_utils.VerifySecret(ra.Ctx.Request) {
userID = 1
} else {
userID = ra.ValidateUser()
}
if !checkProjectPermission(userID, projectID) {
ra.RenderError(http.StatusForbidden, "")
return
}
}
repoList, err := cache.GetRepoFromCache()
@ -110,7 +115,7 @@ func (ra *RepositoryAPI) Delete() {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
rc, err := ra.initRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
@ -143,6 +148,8 @@ func (ra *RepositoryAPI) Delete() {
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
log.Infof("delete tag: %s %s", repoName, t)
go TriggerReplicationByRepository(repoName, []string{t}, models.RepOpDelete)
}
go func() {
@ -166,7 +173,7 @@ func (ra *RepositoryAPI) GetTags() {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
rc, err := ra.initRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
@ -186,6 +193,8 @@ func (ra *RepositoryAPI) GetTags() {
tags = append(tags, ts...)
sort.Strings(tags)
ra.Data["json"] = tags
ra.ServeJSON()
}
@ -199,7 +208,7 @@ func (ra *RepositoryAPI) GetManifests() {
ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
rc, err := ra.initRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
@ -238,16 +247,50 @@ func (ra *RepositoryAPI) GetManifests() {
ra.ServeJSON()
}
func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) {
u := models.User{
UserID: ra.userID,
func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) {
endpoint := os.Getenv("REGISTRY_URL")
username, password, ok := ra.Ctx.Request.BasicAuth()
if ok {
credential := auth.NewBasicAuthCredential(username, password)
return registry.NewRepositoryWithCredential(repoName, endpoint, credential)
}
user, err := dao.GetUser(u)
username, err = ra.getUsername()
if err != nil {
return nil, err
}
endpoint := os.Getenv("REGISTRY_URL")
return registry.NewRepositoryWithUsername(repoName, endpoint, user.Username)
return registry.NewRepositoryWithUsername(repoName, endpoint, username)
}
func (ra *RepositoryAPI) getUsername() (string, error) {
// get username from session
sessionUsername := ra.GetSession("username")
if sessionUsername != nil {
username, ok := sessionUsername.(string)
if ok {
return username, nil
}
}
// if username does not exist in session, try to get userId from sessiion
// and then get username from DB according to the userId
sessionUserID := ra.GetSession("userId")
if sessionUserID != nil {
userID, ok := sessionUserID.(int)
if ok {
u := models.User{
UserID: userID,
}
user, err := dao.GetUser(u)
if err != nil {
return "", err
}
return user.Username, nil
}
}
return "", nil
}

24
api/target.go
View File

@ -16,7 +16,6 @@
package api
import (
"encoding/base64"
"fmt"
"net"
"net/http"
@ -25,6 +24,7 @@ import (
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils"
"github.com/vmware/harbor/utils/log"
registry_util "github.com/vmware/harbor/utils/registry"
"github.com/vmware/harbor/utils/registry/auth"
@ -76,13 +76,11 @@ func (t *TargetAPI) Ping() {
password = target.Password
if len(password) != 0 {
b, err := base64.StdEncoding.DecodeString(password)
password, err = utils.ReversibleDecrypt(password)
if err != nil {
log.Errorf("failed to decode password: %v", err)
log.Errorf("failed to decrypt password: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
password = string(b)
}
} else {
endpoint = t.GetString("endpoint")
@ -142,12 +140,12 @@ func (t *TargetAPI) Get() {
for _, target := range targets {
if len(target.Password) != 0 {
b, err := base64.StdEncoding.DecodeString(target.Password)
str, err := utils.ReversibleDecrypt(target.Password)
if err != nil {
log.Errorf("failed to decode password: %v", err)
log.Errorf("failed to decrypt password: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
target.Password = string(b)
target.Password = str
}
}
@ -167,12 +165,12 @@ func (t *TargetAPI) Get() {
}
if len(target.Password) != 0 {
b, err := base64.StdEncoding.DecodeString(target.Password)
pwd, err := utils.ReversibleDecrypt(target.Password)
if err != nil {
log.Errorf("failed to decode password: %v", err)
log.Errorf("failed to decrypt password: %v", err)
t.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
target.Password = string(b)
target.Password = pwd
}
t.Data["json"] = target
@ -189,7 +187,7 @@ func (t *TargetAPI) Post() {
}
if len(target.Password) != 0 {
target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password))
target.Password = utils.ReversibleEncrypt(target.Password)
}
id, err := dao.AddRepTarget(*target)
@ -216,7 +214,7 @@ func (t *TargetAPI) Put() {
}
if len(target.Password) != 0 {
target.Password = base64.StdEncoding.EncodeToString([]byte(target.Password))
target.Password = utils.ReversibleEncrypt(target.Password)
}
if err := dao.UpdateRepTarget(*target); err != nil {

156
api/utils.go
View File

@ -16,26 +16,66 @@
package api
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"os"
"strings"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
)
func checkProjectPermission(userID int, projectID int64) bool {
exist, err := dao.IsAdminRole(userID)
roles, err := listRoles(userID, projectID)
if err != nil {
log.Errorf("Error occurred in IsAdminRole, error: %v", err)
log.Errorf("error occurred in getProjectPermission: %v", err)
return false
}
if exist {
return true
}
roleList, err := dao.GetUserProjectRoles(userID, projectID)
return len(roles) > 0
}
func hasProjectAdminRole(userID int, projectID int64) bool {
roles, err := listRoles(userID, projectID)
if err != nil {
log.Errorf("Error occurred in GetUserProjectRoles, error: %v", err)
log.Errorf("error occurred in getProjectPermission: %v", err)
return false
}
return len(roleList) > 0
for _, role := range roles {
if role.RoleID == models.PROJECTADMIN {
return true
}
}
return false
}
//sysadmin has all privileges to all projects
func listRoles(userID int, projectID int64) ([]models.Role, error) {
roles := make([]models.Role, 1)
isSysAdmin, err := dao.IsAdminRole(userID)
if err != nil {
return roles, err
}
if isSysAdmin {
role, err := dao.GetRoleByID(models.PROJECTADMIN)
if err != nil {
return roles, err
}
roles = append(roles, *role)
return roles, nil
}
rs, err := dao.GetUserProjectRoles(userID, projectID)
if err != nil {
return roles, err
}
roles = append(roles, rs...)
return roles, nil
}
func checkUserExists(name string) int {
@ -49,3 +89,103 @@ func checkUserExists(name string) int {
}
return 0
}
// TriggerReplication triggers the replication according to the policy
func TriggerReplication(policyID int64, repository string,
tags []string, operation string) error {
data := struct {
PolicyID int64 `json:"policy_id"`
Repo string `json:"repository"`
Operation string `json:"operation"`
TagList []string `json:"tags"`
}{
PolicyID: policyID,
Repo: repository,
TagList: tags,
Operation: operation,
}
b, err := json.Marshal(&data)
if err != nil {
return err
}
url := buildReplicationURL()
resp, err := http.DefaultClient.Post(url, "application/json", bytes.NewBuffer(b))
if err != nil {
return err
}
if resp.StatusCode == http.StatusOK {
return nil
}
defer resp.Body.Close()
b, err = ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf("%d %s", resp.StatusCode, string(b))
}
// GetPoliciesByRepository returns policies according the repository
func GetPoliciesByRepository(repository string) ([]*models.RepPolicy, error) {
repository = strings.TrimSpace(repository)
repository = strings.TrimRight(repository, "/")
projectName := repository[:strings.LastIndex(repository, "/")]
project, err := dao.GetProjectByName(projectName)
if err != nil {
return nil, err
}
policies, err := dao.GetRepPolicyByProject(project.ProjectID)
if err != nil {
return nil, err
}
return policies, nil
}
func TriggerReplicationByRepository(repository string, tags []string, operation string) {
policies, err := GetPoliciesByRepository(repository)
if err != nil {
log.Errorf("failed to get policies for repository %s: %v", repository, err)
return
}
for _, policy := range policies {
if err := TriggerReplication(policy.ID, repository, tags, operation); err != nil {
log.Errorf("failed to trigger replication of %d for %s: %v", policy.ID, repository, err)
} else {
log.Infof("replication of %d for %s triggered", policy.ID, repository)
}
}
}
func buildReplicationURL() string {
url := getJobServiceURL()
url = strings.TrimSpace(url)
url = strings.TrimRight(url, "/")
return fmt.Sprintf("%s/api/replicationJobs", url)
}
func buildJobLogURL(jobID string) string {
url := getJobServiceURL()
url = strings.TrimSpace(url)
url = strings.TrimRight(url, "/")
return fmt.Sprintf("%s/api/replicationJobs/%s/log", url, jobID)
}
func getJobServiceURL() string {
url := os.Getenv("JOB_SERVICE_URL")
if len(url) == 0 {
url = "http://job_service"
}
return url
}

1
dao/project.go
View File

@ -249,7 +249,6 @@ func getProjects(public int, projectName string) ([]models.Project, error) {
}
sql += " order by name "
var projects []models.Project
log.Debugf("sql xxx", sql)
if _, err := o.Raw(sql, queryParam).QueryRows(&projects); err != nil {
return nil, err
}

15
dao/replication_job.go
View File

@ -3,9 +3,10 @@ package dao
import (
"fmt"
"strings"
"github.com/astaxie/beego/orm"
"github.com/vmware/harbor/models"
"strings"
)
func AddRepTarget(target models.RepTarget) (int64, error) {
@ -81,23 +82,21 @@ func DeleteRepPolicy(id int64) error {
_, err := o.Delete(&models.RepPolicy{ID: id})
return err
}
func updateRepPolicyEnablement(id int64, enabled int) error {
func UpdateRepPolicyEnablement(id int64, enabled int) error {
o := orm.NewOrm()
p := models.RepPolicy{
ID: id,
Enabled: enabled}
num, err := o.Update(&p, "Enabled")
if num == 0 {
err = fmt.Errorf("Failed to update replication policy with id: %d", id)
}
_, err := o.Update(&p, "Enabled")
return err
}
func EnableRepPolicy(id int64) error {
return updateRepPolicyEnablement(id, 1)
return UpdateRepPolicyEnablement(id, 1)
}
func DisableRepPolicy(id int64) error {
return updateRepPolicyEnablement(id, 0)
return UpdateRepPolicyEnablement(id, 0)
}
func AddRepJob(job models.RepJob) (int64, error) {

119
job/replication/delete.go Normal file
View File

@ -0,0 +1,119 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package replication
import (
"fmt"
"io/ioutil"
"net/http"
"strings"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
)
const (
// StateDelete ...
StateDelete = "delete"
)
// Deleter deletes repository or tags
type Deleter struct {
repository string // prject_name/repo_name
tags []string
dstURL string // url of target registry
dstUsr string // username ...
dstPwd string // username ...
logger *log.Logger
}
// NewDeleter returns a Deleter
func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, logger *log.Logger) *Deleter {
deleter := &Deleter{
repository: repository,
tags: tags,
dstURL: dstURL,
dstUsr: dstUsr,
dstPwd: dstPwd,
logger: logger,
}
deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, destination user: %s",
deleter.repository, deleter.tags, deleter.dstURL, deleter.dstUsr)
return deleter
}
// Exit ...
func (d *Deleter) Exit() error {
return nil
}
// Enter deletes repository or tags
func (d *Deleter) Enter() (string, error) {
url := strings.TrimRight(d.dstURL, "/") + "/api/repositories/"
// delete repository
if len(d.tags) == 0 {
u := url + "?repo_name=" + d.repository
if err := del(u, d.dstUsr, d.dstPwd); err != nil {
d.logger.Errorf("an error occurred while deleting repository %s on %s with user %s: %v", d.repository, d.dstURL, d.dstUsr, err)
return "", err
}
d.logger.Infof("repository %s on %s has been deleted", d.repository, d.dstURL)
return models.JobFinished, nil
}
// delele tags
for _, tag := range d.tags {
u := url + "?repo_name=" + d.repository + "&tag=" + tag
if err := del(u, d.dstUsr, d.dstPwd); err != nil {
d.logger.Errorf("an error occurred while deleting repository %s:%s on %s with user %s: %v", d.repository, tag, d.dstURL, d.dstUsr, err)
return "", err
}
d.logger.Infof("repository %s:%s on %s has been deleted", d.repository, tag, d.dstURL)
}
return models.JobFinished, nil
}
func del(url, username, password string) error {
req, err := http.NewRequest("DELETE", url, nil)
if err != nil {
return err
}
req.SetBasicAuth(username, password)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusOK {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf("%d %s", resp.StatusCode, string(b))
}

2
job/replication/statehandlers.go → job/replication/transfer.go
View File

@ -153,7 +153,7 @@ func (c *Checker) Enter() (string, error) {
c.logger.Infof("project %s already exists on %s", c.project, c.dstURL)
if !canWrite {
err = fmt.Errorf("the user %s has no write privilege to project %s on %s", c.dstUsr, c.project, c.dstURL)
err = fmt.Errorf("the user %s is unauthorized to write to project %s on %s", c.dstUsr, c.project, c.dstURL)
c.logger.Errorf("%v", err)
return "", err
}

53
job/statemachine.go
View File

@ -9,6 +9,7 @@ import (
"github.com/vmware/harbor/job/replication"
"github.com/vmware/harbor/job/utils"
"github.com/vmware/harbor/models"
uti "github.com/vmware/harbor/utils"
"github.com/vmware/harbor/utils/log"
)
@ -18,6 +19,7 @@ type RepJobParm struct {
TargetUsername string
TargetPassword string
Repository string
Tags []string
Enabled int
Operation string
}
@ -182,6 +184,7 @@ func (sm *JobSM) Reset(jid int64) error {
sm.Parms = &RepJobParm{
LocalRegURL: config.LocalHarborURL(),
Repository: job.Repository,
Tags: job.TagList,
Enabled: policy.Enabled,
Operation: job.Operation,
}
@ -198,35 +201,43 @@ func (sm *JobSM) Reset(jid int64) error {
}
sm.Parms.TargetURL = target.URL
sm.Parms.TargetUsername = target.Username
sm.Parms.TargetPassword = target.Password
pwd := target.Password
if len(pwd) != 0 {
pwd, err = uti.ReversibleDecrypt(pwd)
if err != nil {
return fmt.Errorf("failed to decrypt password: %v", err)
}
}
sm.Parms.TargetPassword = pwd
//init states handlers
sm.Logger = utils.NewLogger(sm.JobID)
sm.Handlers = make(map[string]StateHandler)
sm.Transitions = make(map[string]map[string]struct{})
sm.CurrentState = models.JobPending
sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning})
sm.Handlers[models.JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobError}
sm.Handlers[models.JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobStopped}
if sm.Parms.Operation == models.RepOpTransfer {
/*
sm.AddTransition(models.JobRunning, "pull-img", ImgPuller{DummyHandler: DummyHandler{JobID: sm.JobID}, img: sm.Parms.Repository, logger: sm.Logger})
//only handle one target for now
sm.AddTransition("pull-img", "push-img", ImgPusher{DummyHandler: DummyHandler{JobID: sm.JobID}, targetURL: sm.Parms.TargetURL, logger: sm.Logger})
sm.AddTransition("push-img", models.JobFinished, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished})
*/
if err = addImgOutTransition(sm); err != nil {
return err
}
switch sm.Parms.Operation {
case models.RepOpTransfer:
err = addImgTransferTransition(sm)
case models.RepOpDelete:
err = addImgDeleteTransition(sm)
default:
err = fmt.Errorf("unsupported operation: %s", sm.Parms.Operation)
}
return nil
return err
}
func addImgOutTransition(sm *JobSM) error {
func addImgTransferTransition(sm *JobSM) error {
base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(),
sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword,
nil, sm.Logger)
sm.Parms.Tags, sm.Logger)
if err != nil {
return err
}
@ -238,3 +249,13 @@ func addImgOutTransition(sm *JobSM) error {
sm.AddTransition(replication.StatePushManifest, replication.StatePullManifest, &replication.ManifestPuller{BaseHandler: base})
return nil
}
func addImgDeleteTransition(sm *JobSM) error {
deleter := replication.NewDeleter(sm.Parms.Repository, sm.Parms.Tags, sm.Parms.TargetURL,
sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Logger)
sm.AddTransition(models.JobRunning, replication.StateDelete, deleter)
sm.AddTransition(replication.StateDelete, models.JobFinished, &StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished})
return nil
}

8
service/notification.go
View File

@ -20,6 +20,7 @@ import (
"regexp"
"strings"
"github.com/vmware/harbor/api"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/service/cache"
@ -69,6 +70,11 @@ func (n *NotificationHandler) Post() {
if username == "" {
username = "anonymous"
}
if username == "job-service-user" {
return
}
go dao.AccessLog(username, project, repo, repoTag, action)
if action == "push" {
go func() {
@ -77,6 +83,8 @@ func (n *NotificationHandler) Post() {
log.Errorf("Error happens when refreshing cache: %v", err2)
}
}()
go api.TriggerReplicationByRepository(repo, []string{repoTag}, models.RepOpTransfer)
}
}
}

20
service/token/authutils.go
View File

@ -46,10 +46,24 @@ func GetResourceActions(scopes []string) []*token.ResourceActions {
continue
}
items := strings.Split(s, ":")
length := len(items)
typee := items[0]
name := ""
if length > 1 {
name = items[1]
}
actions := []string{}
if length > 2 {
actions = strings.Split(items[2], ",")
}
res = append(res, &token.ResourceActions{
Type: items[0],
Name: items[1],
Actions: strings.Split(items[2], ","),
Type: typee,
Name: name,
Actions: actions,
})
}
return res

5
ui/router.go
View File

@ -53,6 +53,7 @@ func initRouters() {
//API:
beego.Router("/api/search", &api.SearchAPI{})
beego.Router("/api/projects/:pid/members/?:mid", &api.ProjectMemberAPI{})
beego.Router("/api/projects/", &api.ProjectAPI{}, "get:List")
beego.Router("/api/projects/?:id", &api.ProjectAPI{})
beego.Router("/api/statistics", &api.StatisticAPI{})
beego.Router("/api/projects/:id/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog")
@ -62,6 +63,10 @@ func initRouters() {
beego.Router("/api/repositories", &api.RepositoryAPI{})
beego.Router("/api/repositories/tags", &api.RepositoryAPI{}, "get:GetTags")
beego.Router("/api/repositories/manifests", &api.RepositoryAPI{}, "get:GetManifests")
beego.Router("/api/replicationJobs", &api.RepJobAPI{})
beego.Router("/api/replicationJobs/:id/log", &api.RepJobAPI{}, "get:GetLog")
beego.Router("/api/replicationPolicies", &api.RepPolicyAPI{})
beego.Router("/api/replicationPolicies/:id/enablement", &api.RepPolicyAPI{}, "put:UpdateEnablement")
beego.Router("/api/targets/?:id", &api.TargetAPI{})
beego.Router("/api/target_ping", &api.TargetAPI{}, "get:Ping")

12
utils/encrypt.go
View File

@ -17,6 +17,7 @@ package utils
import (
"crypto/sha1"
"encoding/base64"
"fmt"
"golang.org/x/crypto/pbkdf2"
@ -26,3 +27,14 @@ import (
func Encrypt(content string, salt string) string {
return fmt.Sprintf("%x", pbkdf2.Key([]byte(content), []byte(salt), 4096, 16, sha1.New))
}
// ReversibleEncrypt encrypts the str with base64
func ReversibleEncrypt(str string) string {
return base64.StdEncoding.EncodeToString([]byte(str))
}
// ReversibleDecrypt decrypts the str with base64
func ReversibleDecrypt(str string) (string, error) {
b, err := base64.StdEncoding.DecodeString(str)
return string(b), err
}