diff --git a/.gitignore b/.gitignore index 6b04c4ad3..b53274b48 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ Deploy/config/ui/app.conf Deploy/config/db/env Deploy/harbor.cfg ui/ui +*.pyc diff --git a/.travis.yml b/.travis.yml index 6e536c507..aff9e1f81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,14 @@ language: go go: - - 1.5.3 + - 1.6.2 go_import_path: github.com/vmware/harbor service: - mysql -env: GO15VENDOREXPERIMENT=1 DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD= +env: DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD= install: - sudo apt-get update && sudo apt-get install -y libldap2-dev diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index 522b2b012..0380e739c 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -103,19 +103,41 @@ create table access_log ( FOREIGN KEY (project_id) REFERENCES project (project_id) ); -create table job ( - job_id int NOT NULL AUTO_INCREMENT, - job_type varchar(64) NOT NULL, - status varchar(64) NOT NULL, - options text, - parms text, +create table replication_policy ( + id int NOT NULL AUTO_INCREMENT, + name varchar(256), + project_id int NOT NULL, + target_id int NOT NULL, enabled tinyint(1) NOT NULL DEFAULT 1, + description text, cron_str varchar(256), - triggered_by varchar(64), - creation_time timestamp, - update_time timestamp, - PRIMARY KEY (job_id) -); + start_time timestamp, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY (id) + ); + +create table replication_target ( + id int NOT NULL AUTO_INCREMENT, + name varchar(64), + url varchar(64), + username varchar(40), + password varchar(40), + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY (id) + ); + +create table replication_job ( + id int NOT NULL AUTO_INCREMENT, + status varchar(64) NOT NULL, + policy_id int NOT NULL, + repository varchar(256) NOT NULL, + operation varchar(64) NOT NULL, + creation_time timestamp default CURRENT_TIMESTAMP, + update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, + PRIMARY KEY (id) + ); create table job_log ( log_id int NOT NULL AUTO_INCREMENT, @@ -125,7 +147,7 @@ create table job_log ( creation_time timestamp, update_time timestamp, PRIMARY KEY (log_id), - FOREIGN KEY (job_id) REFERENCES job (job_id) + FOREIGN KEY (job_id) REFERENCES replication_job (id) ); create table properties ( @@ -136,3 +158,9 @@ create table properties ( insert into properties (k, v) values ('schema_version', '0.1.1'); + +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; + +insert into alembic_version values ('0.1.1'); diff --git a/Deploy/prepare b/Deploy/prepare index 33288d06e..edd31ea95 100755 --- a/Deploy/prepare +++ b/Deploy/prepare @@ -116,7 +116,7 @@ FNULL = open(os.devnull, 'w') from functools import wraps def stat_decorator(func): - #@wraps(func) + @wraps(func) def check_wrapper(*args, **kwargs): stat = func(*args, **kwargs) message = "Generated configuration file: %s" % kwargs['path'] \ diff --git a/Deploy/templates/ui/env b/Deploy/templates/ui/env index fe91f5965..d21e8fafc 100644 --- a/Deploy/templates/ui/env +++ b/Deploy/templates/ui/env @@ -12,4 +12,4 @@ LDAP_URL=$ldap_url LDAP_BASE_DN=$ldap_basedn SELF_REGISTRATION=$self_registration LOG_LEVEL=debug -GODEBUG=netdns=cgo \ No newline at end of file +GODEBUG=netdns=cgo diff --git a/Dockerfile.ui b/Dockerfile.ui index 459158279..c5b087bd3 100644 --- a/Dockerfile.ui +++ b/Dockerfile.ui @@ -1,4 +1,4 @@ -FROM golang:1.5.1 +FROM golang:1.6.2 MAINTAINER jiangd@vmware.com @@ -11,7 +11,6 @@ COPY . /go/src/github.com/vmware/harbor COPY ./vendor/golang.org /go/src/golang.org WORKDIR /go/src/github.com/vmware/harbor/ui -ENV GO15VENDOREXPERIMENT 1 RUN go get -d github.com/docker/distribution \ && go get -d github.com/docker/libtrust \ && go get -d github.com/go-sql-driver/mysql \ diff --git a/api/job.go b/api/job.go index 5d5066a5f..48930b7ef 100644 --- a/api/job.go +++ b/api/job.go @@ -1,5 +1,6 @@ package api +/* import ( "encoding/json" "fmt" @@ -15,6 +16,7 @@ type JobAPI struct { BaseAPI } + func (ja *JobAPI) Post() { var je models.JobEntry ja.DecodeJSONReq(&je) @@ -82,4 +84,4 @@ func (ja *JobAPI) Get() { ja.Data["json"] = jobs } ja.ServeJSON() -} +}*/ diff --git a/api/replication.go b/api/replication.go new file mode 100644 index 000000000..ba6f81471 --- /dev/null +++ b/api/replication.go @@ -0,0 +1,106 @@ +package api + +import ( + "encoding/json" + "fmt" + "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/job" + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" + "io/ioutil" + "net/http" + "net/http/httputil" + "os" + "strconv" + "strings" +) + +type ReplicationJob struct { + BaseAPI +} + +type ReplicationReq struct { + PolicyID int64 `json:"policy_id"` +} + +func (rj *ReplicationJob) Post() { + var data ReplicationReq + rj.DecodeJSONReq(&data) + log.Debugf("data: %+v", data) + p, err := dao.GetRepPolicy(data.PolicyID) + if err != nil { + log.Errorf("Failed to get policy, error: %v", err) + rj.RenderError(http.StatusInternalServerError, fmt.Sprintf("Failed to get policy, id: %d", data.PolicyID)) + return + } + repoList, err := getRepoList(p.ProjectID) + if err != nil { + log.Errorf("Failed to get repository list, project id: %d, error: %v", p.ProjectID, err) + rj.RenderError(http.StatusInternalServerError, err.Error()) + return + } + log.Debugf("repo list: %v", repoList) + for _, repo := range repoList { + j := models.RepJob{ + Repository: repo, + PolicyID: data.PolicyID, + Operation: models.RepOpTransfer, + } + log.Debugf("Creating job for repo: %s, policy: %d", repo, data.PolicyID) + id, err := dao.AddRepJob(j) + if err != nil { + log.Errorf("Failed to insert job record, error: %v", err) + rj.RenderError(http.StatusInternalServerError, err.Error()) + return + } + log.Debugf("Send job to scheduler, job id: %d", id) + job.Schedule(id) + } +} + +// calls the api from UI to get repo list +func getRepoList(projectID int64) ([]string, error) { + uiURL := os.Getenv("UI_URL") + if len(uiURL) == 0 { + uiURL = "ui" + } + if !strings.HasSuffix(uiURL, "/") { + uiURL += "/" + } + //TODO:Use secret key instead + uiUser := os.Getenv("UI_USR") + if len(uiUser) == 0 { + uiUser = "admin" + } + uiPwd := os.Getenv("UI_PWD") + if len(uiPwd) == 0 { + uiPwd = "Harbor12345" + } + client := &http.Client{} + req, err := http.NewRequest("GET", uiURL+"api/repositories?project_id="+strconv.Itoa(int(projectID)), nil) + if err != nil { + log.Errorf("Error when creating request: %v") + return nil, err + } + req.SetBasicAuth(uiUser, uiPwd) + resp, err := client.Do(req) + if err != nil { + log.Errorf("Error when calling UI api to get repositories, error: %v", err) + return nil, err + } + if resp.StatusCode != http.StatusOK { + log.Errorf("Unexpected status code: %d", resp.StatusCode) + dump, _ := httputil.DumpResponse(resp, true) + log.Debugf("response: %q", dump) + return nil, fmt.Errorf("Unexpected status code when getting repository list: %d", resp.StatusCode) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.Errorf("Failed to read the response body, error: %v") + return nil, err + } + var repoList []string + err = json.Unmarshal(body, &repoList) + return repoList, err +} diff --git a/api/repository.go b/api/repository.go index e259067a4..b0361af3a 100644 --- a/api/repository.go +++ b/api/repository.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/docker/distribution/manifest/schema1" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" @@ -40,61 +41,20 @@ type RepositoryAPI struct { BaseAPI userID int username string - registry *registry.Registry } // Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. func (ra *RepositoryAPI) Prepare() { userID, ok := ra.GetSession("userId").(int) if !ok { - ra.userID = dao.NonExistUserID - } else { - ra.userID = userID + userID = dao.NonExistUserID } + ra.userID = userID + username, ok := ra.GetSession("username").(string) - if !ok { - log.Warning("failed to get username from session") - ra.username = "" - } else { + if ok { ra.username = username } - - var client *http.Client - - //no session, initialize a standard auth handler - if ra.userID == dao.NonExistUserID && len(ra.username) == 0 { - username, password, _ := ra.Ctx.Request.BasicAuth() - - credential := auth.NewBasicAuthCredential(username, password) - client = registry.NewClientStandardAuthHandlerEmbeded(credential) - log.Debug("initializing standard auth handler") - - } else { - // session works, initialize a username auth handler - username := ra.username - if len(username) == 0 { - user, err := dao.GetUser(models.User{ - UserID: ra.userID, - }) - if err != nil { - log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err) - return - } - - username = user.Username - } - - client = registry.NewClientUsernameAuthHandlerEmbeded(username) - log.Debug("initializing username auth handler: %s", username) - } - - endpoint := os.Getenv("REGISTRY_URL") - r, err := registry.New(endpoint, client) - if err != nil { - log.Fatalf("error occurred while initializing auth handler for repository API: %v", err) - } - - ra.registry = r } // Get ... @@ -156,10 +116,16 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") } + rc, err := ra.initializeRepositoryClient(repoName) + if err != nil { + log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + tags := []string{} tag := ra.GetString("tag") if len(tag) == 0 { - tagList, err := ra.registry.ListTag(repoName) + tagList, err := rc.ListTag() if err != nil { e, ok := errors.ParseError(err) if ok { @@ -169,16 +135,14 @@ func (ra *RepositoryAPI) Delete() { log.Error(err) ra.CustomAbort(http.StatusInternalServerError, "internal error") } - } tags = append(tags, tagList...) - } else { tags = append(tags, tag) } for _, t := range tags { - if err := ra.registry.DeleteTag(repoName, t); err != nil { + if err := rc.DeleteTag(t); err != nil { e, ok := errors.ParseError(err) if ok { ra.CustomAbort(e.StatusCode, e.Message) @@ -206,15 +170,23 @@ type tag struct { // GetTags handles GET /api/repositories/tags func (ra *RepositoryAPI) GetTags() { - - var tags []string - repoName := ra.GetString("repo_name") - tags, err := ra.registry.ListTag(repoName) + if len(repoName) == 0 { + ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") + } + + rc, err := ra.initializeRepositoryClient(repoName) + if err != nil { + log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + + tags := []string{} + + ts, err := rc.ListTag() if err != nil { e, ok := errors.ParseError(err) if ok { - log.Info(e) ra.CustomAbort(e.StatusCode, e.Message) } else { log.Error(err) @@ -222,6 +194,8 @@ func (ra *RepositoryAPI) GetTags() { } } + tags = append(tags, ts...) + ra.Data["json"] = tags ra.ServeJSON() } @@ -231,13 +205,23 @@ func (ra *RepositoryAPI) GetManifests() { repoName := ra.GetString("repo_name") tag := ra.GetString("tag") + if len(repoName) == 0 || len(tag) == 0 { + ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil") + } + + rc, err := ra.initializeRepositoryClient(repoName) + if err != nil { + log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + item := models.RepoItem{} - _, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1) + mediaTypes := []string{schema1.MediaTypeManifest} + _, _, payload, err := rc.PullManifest(tag, mediaTypes) if err != nil { e, ok := errors.ParseError(err) if ok { - log.Info(e) ra.CustomAbort(e.StatusCode, e.Message) } else { log.Error(err) @@ -264,3 +248,31 @@ func (ra *RepositoryAPI) GetManifests() { ra.Data["json"] = item ra.ServeJSON() } + +func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) { + endpoint := os.Getenv("REGISTRY_URL") + + //no session, use basic auth + if ra.userID == dao.NonExistUserID { + username, password, _ := ra.Ctx.Request.BasicAuth() + credential := auth.NewBasicAuthCredential(username, password) + + return registry.NewRepositoryWithCredential(repoName, endpoint, credential) + + } + + //session exists, use username + if len(ra.username) == 0 { + u := models.User{ + UserID: ra.userID, + } + user, err := dao.GetUser(u) + if err != nil { + return nil, err + } + + ra.username = user.Username + } + + return registry.NewRepositoryWithUsername(repoName, endpoint, ra.username) +} diff --git a/api/user.go b/api/user.go index c9bb99800..45869fb6c 100644 --- a/api/user.go +++ b/api/user.go @@ -187,7 +187,9 @@ func (ua *UserAPI) Delete() { // ChangePassword handles PUT to /api/users/{}/password func (ua *UserAPI) ChangePassword() { - if !(ua.AuthMode == "db_auth") { + ldapAdminUser := (ua.AuthMode == "ldap_auth" && ua.userID == 1 && ua.userID == ua.currentUserID) + + if !(ua.AuthMode == "db_auth" || ldapAdminUser) { ua.CustomAbort(http.StatusForbidden, "") } diff --git a/controllers/base.go b/controllers/base.go index e76abe4be..7d4b3e535 100644 --- a/controllers/base.go +++ b/controllers/base.go @@ -41,6 +41,7 @@ type BaseController struct { beego.Controller i18n.Locale SelfRegistration bool + IsLdapAdminUser bool IsAdmin bool AuthMode string } @@ -115,7 +116,11 @@ func (b *BaseController) Prepare() { if sessionUserID != nil { b.Data["Username"] = b.GetSession("username") b.Data["UserId"] = sessionUserID.(int) - + + if (sessionUserID == 1 && b.AuthMode == "ldap_auth") { + b.IsLdapAdminUser = true + } + var err error b.IsAdmin, err = dao.IsAdminRole(sessionUserID.(int)) if err != nil { @@ -126,6 +131,7 @@ func (b *BaseController) Prepare() { b.Data["IsAdmin"] = b.IsAdmin b.Data["SelfRegistration"] = b.SelfRegistration + b.Data["IsLdapAdminUser"] = b.IsLdapAdminUser } diff --git a/dao/accesslog.go b/dao/accesslog.go index dbf447353..908fb9280 100644 --- a/dao/accesslog.go +++ b/dao/accesslog.go @@ -62,6 +62,14 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { sql += ` and u.username like ? ` queryParam = append(queryParam, accessLog.Username) } + if accessLog.RepoName != "" { + sql += ` and a.repo_name = ? ` + queryParam = append(queryParam, accessLog.RepoName) + } + if accessLog.RepoTag != "" { + sql += ` and a.repo_tag = ? ` + queryParam = append(queryParam, accessLog.RepoTag) + } if accessLog.Keywords != "" { sql += ` and a.operation in ( ` keywordList := strings.Split(accessLog.Keywords, "/") diff --git a/dao/dao_test.go b/dao/dao_test.go index da0ad6993..30b9d7961 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -20,11 +20,9 @@ import ( "testing" "time" - "github.com/vmware/harbor/utils/log" - - "github.com/vmware/harbor/models" - "github.com/astaxie/beego/orm" + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" ) func execUpdate(o orm.Ormer, sql string, params interface{}) error { @@ -102,6 +100,8 @@ func clearUp(username string) { const username string = "Tester01" const projectName string = "test_project" +const repoTag string = "test1.1" +const repoTag2 string = "test1.2" const SysAdmin int = 1 const projectAdmin int = 2 const developer int = 3 @@ -419,6 +419,66 @@ func TestGetAccessLog(t *testing.T) { } } +func TestAddAccessLog(t *testing.T) { + var err error + var accessLogList []models.AccessLog + accessLog := models.AccessLog{ + UserID: currentUser.UserID, + ProjectID: currentProject.ProjectID, + RepoName: currentProject.Name + "/", + RepoTag: repoTag, + GUID: "N/A", + Operation: "create", + OpTime: time.Now(), + } + err = AddAccessLog(accessLog) + if err != nil { + t.Errorf("Error occurred in AddAccessLog: %v", err) + } + accessLogList, err = GetAccessLogs(accessLog) + if err != nil { + t.Errorf("Error occurred in GetAccessLog: %v", err) + } + if len(accessLogList) != 1 { + t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList)) + } + if accessLogList[0].RepoName != projectName+"/" { + t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName) + } + if accessLogList[0].RepoTag != repoTag { + t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag, accessLogList[0].RepoTag) + } +} + +func TestAccessLog(t *testing.T) { + var err error + var accessLogList []models.AccessLog + accessLog := models.AccessLog{ + UserID: currentUser.UserID, + ProjectID: currentProject.ProjectID, + RepoName: currentProject.Name + "/", + RepoTag: repoTag2, + Operation: "create", + } + err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/", repoTag2, "create") + if err != nil { + t.Errorf("Error occurred in AccessLog: %v", err) + } + accessLogList, err = GetAccessLogs(accessLog) + if err != nil { + t.Errorf("Error occurred in GetAccessLog: %v", err) + } + if len(accessLogList) != 1 { + t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList)) + } + if accessLogList[0].RepoName != projectName+"/" { + t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName) + } + if accessLogList[0].RepoTag != repoTag2 { + t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag2, accessLogList[0].RepoTag) + } +} + func TestProjectExists(t *testing.T) { var exists bool var err error @@ -660,3 +720,227 @@ func TestDeleteUser(t *testing.T) { t.Errorf("user is not nil after deletion, user: %+v", user) } } + +var targetID, policyID, jobID int64 + +func TestAddRepTarget(t *testing.T) { + target := models.RepTarget{ + URL: "127.0.0.1:5000", + Username: "admin", + Password: "admin", + } + //_, err := AddRepTarget(target) + id, err := AddRepTarget(target) + t.Logf("added target, id: %d", id) + if err != nil { + t.Errorf("Error occurred in AddRepTarget: %v", err) + } else { + targetID = id + } + id2 := id + 99 + tgt, err := GetRepTarget(id2) + if err != nil { + t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id2) + } + if tgt != nil { + t.Errorf("There should not be a target with id: %d", id2) + } + tgt, err = GetRepTarget(id) + if err != nil { + t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id) + } + if tgt == nil { + t.Errorf("Unable to find a target with id: %d", id) + } + if tgt.URL != "127.0.0.1:5000" { + t.Errorf("Unexpected url in target: %s, expected 127.0.0.1:5000", tgt.URL) + } + if tgt.Username != "admin" { + t.Errorf("Unexpected username in target: %s, expected admin", tgt.Username) + } +} + +func TestAddRepPolicy(t *testing.T) { + policy := models.RepPolicy{ + ProjectID: 1, + Enabled: 1, + TargetID: targetID, + Description: "whatever", + Name: "mypolicy", + } + id, err := AddRepPolicy(policy) + t.Logf("added policy, id: %d", id) + if err != nil { + t.Errorf("Error occurred in AddRepPolicy: %v", err) + } else { + policyID = id + } + p, err := GetRepPolicy(id) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", id) + } + + if p.Name != "mypolicy" || p.TargetID != targetID || p.Enabled != 1 || p.Description != "whatever" { + t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Enabled: 1, Description: whatever;\n result: Name: %s, TargetID: %d, Enabled: %d, Description: %s", + targetID, p.Name, p.TargetID, p.Enabled, p.Description) + } + var tm time.Time = time.Now().AddDate(0, 0, -1) + if !p.StartTime.After(tm) { + t.Errorf("Unexpected start_time: %v", p.StartTime) + } + +} + +func TestDisableRepPolicy(t *testing.T) { + err := DisableRepPolicy(policyID) + if err != nil { + t.Errorf("Failed to disable policy, id: %d", policyID) + } + p, err := GetRepPolicy(policyID) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", policyID) + } + if p.Enabled == 1 { + t.Errorf("The Enabled value of replication policy is still 1 after disabled, id: %d", policyID) + } +} + +func TestEnableRepPolicy(t *testing.T) { + err := EnableRepPolicy(policyID) + if err != nil { + t.Errorf("Failed to disable policy, id: %d", policyID) + } + p, err := GetRepPolicy(policyID) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", policyID) + } + if p.Enabled == 0 { + t.Errorf("The Enabled value of replication policy is still 0 after disabled, id: %d", policyID) + } +} + +func TestAddRepPolicy2(t *testing.T) { + policy2 := models.RepPolicy{ + ProjectID: 3, + Enabled: 0, + TargetID: 3, + Description: "whatever", + Name: "mypolicy", + } + id, err := AddRepPolicy(policy2) + t.Logf("added policy, id: %d", id) + if err != nil { + t.Errorf("Error occurred in AddRepPolicy: %v", err) + } + p, err := GetRepPolicy(id) + if err != nil { + t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id) + } + if p == nil { + t.Errorf("Unable to find a policy with id: %d", id) + } + var tm time.Time + if p.StartTime.After(tm) { + t.Errorf("Unexpected start_time: %v", p.StartTime) + } +} + +func TestAddRepJob(t *testing.T) { + job := models.RepJob{ + Repository: "library/ubuntu", + PolicyID: policyID, + Operation: "transfer", + } + id, err := AddRepJob(job) + if err != nil { + t.Errorf("Error occurred in AddRepJob: %v", err) + } else { + jobID = id + } + j, err := GetRepJob(id) + if err != nil { + t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, id) + } + if j == nil { + t.Errorf("Unable to find a job with id: %d", id) + } + if j.Status != models.JobPending || j.Repository != "library/ubuntu" || j.PolicyID != policyID || j.Operation != "transfer" { + t.Errorf("Expected data of job, id: %d, Status: %s, Repository: library/ubuntu, PolicyID: %d, Operation: transfer, "+ + "but in returned data:, Status: %s, Repository: %s, Operation: %s, PolicyID: %d", id, models.JobPending, policyID, j.Status, j.Repository, j.Operation, j.PolicyID) + } +} + +func TestUpdateRepJobStatus(t *testing.T) { + err := UpdateRepJobStatus(jobID, models.JobFinished) + if err != nil { + t.Errorf("Error occured in UpdateRepJobStatus, error: %v, id: %d", err, jobID) + return + } + j, err := GetRepJob(jobID) + if err != nil { + t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, jobID) + } + if j == nil { + t.Errorf("Unable to find a job with id: %d", jobID) + } + if j.Status != models.JobFinished { + t.Errorf("Job's status: %s, expected: %s, id: %d", j.Status, models.JobFinished, jobID) + } +} + +func TestDeleteRepTarget(t *testing.T) { + err := DeleteRepTarget(targetID) + if err != nil { + t.Errorf("Error occured in DeleteRepTarget: %v, id: %d", err, targetID) + return + } + t.Logf("deleted target, id: %d", targetID) + tgt, err := GetRepTarget(targetID) + if err != nil { + t.Errorf("Error occurred in GetTarget: %v, id: %d", err, targetID) + } + if tgt != nil { + t.Errorf("Able to find target after deletion, id: %d", targetID) + } +} + +func TestDeleteRepPolicy(t *testing.T) { + err := DeleteRepPolicy(policyID) + if err != nil { + t.Errorf("Error occured in DeleteRepPolicy: %v, id: %d", err, policyID) + return + } + t.Logf("delete rep policy, id: %d", policyID) + p, err := GetRepPolicy(policyID) + if err != nil { + t.Errorf("Error occured in GetRepPolicy:%v", err) + } + if p != nil { + t.Errorf("Able to find rep policy after deletion, id: %d", policyID) + } +} + +func TestDeleteRepJob(t *testing.T) { + err := DeleteRepJob(jobID) + if err != nil { + t.Errorf("Error occured in DeleteRepJob: %v, id: %d", err, jobID) + return + } + t.Logf("deleted rep job, id: %d", jobID) + j, err := GetRepJob(jobID) + if err != nil { + t.Errorf("Error occured in GetRepJob:%v", err) + } + if j != nil { + t.Errorf("Able to find rep job after deletion, id: %d", jobID) + } +} diff --git a/dao/replication_job.go b/dao/replication_job.go new file mode 100644 index 000000000..695464621 --- /dev/null +++ b/dao/replication_job.go @@ -0,0 +1,113 @@ +package dao + +import ( + "fmt" + "github.com/astaxie/beego/orm" + "github.com/vmware/harbor/models" +) + +func AddRepTarget(target models.RepTarget) (int64, error) { + o := orm.NewOrm() + return o.Insert(&target) +} +func GetRepTarget(id int64) (*models.RepTarget, error) { + o := orm.NewOrm() + t := models.RepTarget{ID: id} + err := o.Read(&t) + if err == orm.ErrNoRows { + return nil, nil + } + return &t, err +} +func DeleteRepTarget(id int64) error { + o := orm.NewOrm() + _, err := o.Delete(&models.RepTarget{ID: id}) + return err +} + +func AddRepPolicy(policy models.RepPolicy) (int64, error) { + o := orm.NewOrm() + sqlTpl := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, %s, NOW(), NOW())` + var sql string + if policy.Enabled == 1 { + sql = fmt.Sprintf(sqlTpl, "NOW()") + } else { + sql = fmt.Sprintf(sqlTpl, "NULL") + } + p, err := o.Raw(sql).Prepare() + if err != nil { + return 0, err + } + r, err := p.Exec(policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr) + if err != nil { + return 0, err + } + id, err := r.LastInsertId() + return id, err +} +func GetRepPolicy(id int64) (*models.RepPolicy, error) { + o := orm.NewOrm() + p := models.RepPolicy{ID: id} + err := o.Read(&p) + if err == orm.ErrNoRows { + return nil, nil + } + return &p, err +} +func DeleteRepPolicy(id int64) error { + o := orm.NewOrm() + _, err := o.Delete(&models.RepPolicy{ID: id}) + return err +} +func updateRepPolicyEnablement(id int64, enabled int) error { + o := orm.NewOrm() + p := models.RepPolicy{ + ID: id, + Enabled: enabled} + num, err := o.Update(&p, "Enabled") + if num == 0 { + err = fmt.Errorf("Failed to update replication policy with id: %d", id) + } + return err +} +func EnableRepPolicy(id int64) error { + return updateRepPolicyEnablement(id, 1) +} + +func DisableRepPolicy(id int64) error { + return updateRepPolicyEnablement(id, 0) +} + +func AddRepJob(job models.RepJob) (int64, error) { + o := orm.NewOrm() + if len(job.Status) == 0 { + job.Status = models.JobPending + } + return o.Insert(&job) +} +func GetRepJob(id int64) (*models.RepJob, error) { + o := orm.NewOrm() + j := models.RepJob{ID: id} + err := o.Read(&j) + if err == orm.ErrNoRows { + return nil, nil + } + return &j, err +} +func DeleteRepJob(id int64) error { + o := orm.NewOrm() + _, err := o.Delete(&models.RepJob{ID: id}) + return err +} +func UpdateRepJobStatus(id int64, status string) error { + o := orm.NewOrm() + j := models.RepJob{ + ID: id, + Status: status, + } + num, err := o.Update(&j, "Status") + if num == 0 { + err = fmt.Errorf("Failed to update replication job with id: %d", id) + } + return err +} diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 404a0b431..20307cc49 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -109,7 +109,7 @@ paths: tags: - Products responses: - 200: + 201: description: Project created successfully. 400: description: Unsatisfied with constraints of the project creation. @@ -383,6 +383,28 @@ paths: description: User does not have permission of admin role. 500: description: Unexpected internal errors. + post: + summary: Creates a new user account. + description: | + This endpoint is to create a user if the user does not already exist. + parameters: + - name: user + in: body + description: New created user. + required: true + schema: + $ref: '#/definitions/User' + tags: + - Products + responses: + 201: + description: User created successfully. + 400: + description: Unsatisfied with constraints of the user creation. + 403: + description: User registration can only be used by admin role user when self-registration is off. + 500: + description: Unexpected internal errors. /users/{user_id}: put: summary: Update a registered user to change to be an administrator of Harbor. @@ -438,6 +460,37 @@ paths: description: User ID does not exist. 500: description: Unexpected internal errors. + /users/{user_id}/password: + put: + summary: Change the password on a user that already exists. + description: | + This endpoint is for user to update password. Users with the admin role can change any user's password. Guest users can change only their own password. + parameters: + - name: user_id + in: path + type: integer + format: int32 + required: true + description: Registered user ID. + - name: password + in: body + description: Password to be updated. + required: true + schema: + $ref: '#/definitions/Password' + tags: + - Products + responses: + 200: + description: Updated password successfully. + 400: + description: Invalid user ID; Old password is blank; New password is blank. + 401: + description: Old password is not correct. + 403: + description: Guests can only change their own account. + 500: + description: Unexpected internal errors. /repositories: get: summary: Get repositories accompany with relevant project and repo name. @@ -640,6 +693,15 @@ definitions: deleted: type: integer format: int32 + Password: + type: object + properties: + old_password: + type: string + description: The user's existing password. + new_password: + type: string + description: New password for marking as to be updated. AccessLog: type: object properties: diff --git a/job/imgout/runner.go b/job/imgout/runner.go index 54097ee33..d4311fb4a 100644 --- a/job/imgout/runner.go +++ b/job/imgout/runner.go @@ -1,5 +1,6 @@ package imgout +/* import ( "encoding/json" //"github.com/vmware/harbor/dao" @@ -72,3 +73,4 @@ func (r *Runner) init(je models.JobEntry) error { r.AddTransition("push-img", job.JobFinished, job.StatusUpdater{job.DummyHandler{JobID: r.JobID}, job.JobFinished}) return nil } +*/ diff --git a/job/scheduler.go b/job/scheduler.go index bb265e80a..80d9859fa 100644 --- a/job/scheduler.go +++ b/job/scheduler.go @@ -1,32 +1,31 @@ package job import ( + "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" - "os" - "strconv" ) -var lock chan bool +func Schedule(jobID int64) { + //TODO: introduce jobqueue to better control concurrent job numbers + go HandleRepJob(jobID) +} -const defaultMaxJobs int64 = 10 - -func init() { - maxJobsEnv := os.Getenv("MAX_CONCURRENT_JOB") - maxJobs, err := strconv.ParseInt(maxJobsEnv, 10, 32) +func HandleRepJob(id int64) { + sm := &JobSM{JobID: id} + err := sm.Init() if err != nil { - log.Warningf("Failed to parse max job setting, error: %v, the default value: %d will be used", err, defaultMaxJobs) - maxJobs = defaultMaxJobs + log.Errorf("Failed to initialize statemachine, error: %v") + err2 := dao.UpdateRepJobStatus(id, models.JobError) + if err2 != nil { + log.Errorf("Failed to update job status to ERROR, error:%v", err2) + } + return + } + if sm.Parms.Enabled == 0 { + log.Debugf("The policy of job:%d is disabled, will cancel the job") + _ = dao.UpdateRepJobStatus(id, models.JobCanceled) + } else { + sm.Start(models.JobRunning) } - lock = make(chan bool, maxJobs) -} -func Schedule(job models.JobEntry) { - log.Infof("job: %d will be scheduled", job.ID) - //TODO: add support for cron string when needed. - go func() { - lock <- true - defer func() { <-lock }() - log.Infof("running job: %d", job.ID) - run(job) - }() } diff --git a/job/statehandlers.go b/job/statehandlers.go new file mode 100644 index 000000000..9f68ff9bc --- /dev/null +++ b/job/statehandlers.go @@ -0,0 +1,73 @@ +package job + +import ( + "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" + "time" +) + +// StateHandler handles transition, it associates with each state, will be called when +// SM enters and exits a state during a transition. +type StateHandler interface { + // Enter returns the next state, if it returns empty string the SM will hold the current state or + // or decide the next state. + Enter() (string, error) + //Exit should be idempotent + Exit() error +} + +type DummyHandler struct { + JobID int64 +} + +func (dh DummyHandler) Enter() (string, error) { + return "", nil +} + +func (dh DummyHandler) Exit() error { + return nil +} + +type StatusUpdater struct { + DummyHandler + State string +} + +func (su StatusUpdater) Enter() (string, error) { + err := dao.UpdateRepJobStatus(su.JobID, su.State) + if err != nil { + log.Warningf("Failed to update state of job: %d, state: %s, error: %v", su.JobID, su.State, err) + } + var next string = models.JobContinue + if su.State == models.JobStopped || su.State == models.JobError || su.State == models.JobFinished { + next = "" + } + return next, err +} + +type ImgPuller struct { + DummyHandler + img string + logger Logger +} + +func (ip ImgPuller) Enter() (string, error) { + ip.logger.Infof("I'm pretending to pull img:%s, then sleep 30s", ip.img) + time.Sleep(30 * time.Second) + ip.logger.Infof("wake up from sleep....") + return "push-img", nil +} + +type ImgPusher struct { + DummyHandler + targetURL string + logger Logger +} + +func (ip ImgPusher) Enter() (string, error) { + ip.logger.Infof("I'm pretending to push img to:%s, then sleep 30s", ip.targetURL) + time.Sleep(30 * time.Second) + ip.logger.Infof("wake up from sleep....") + return models.JobContinue, nil +} diff --git a/job/statemachine.go b/job/statemachine.go index f6bc88b92..f4056cad9 100644 --- a/job/statemachine.go +++ b/job/statemachine.go @@ -3,59 +3,22 @@ package job import ( "fmt" "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" + "os" "sync" ) -// StateHandler handles transition, it associates with each state, will be called when -// SM enters and exits a state during a transition. -type StateHandler interface { - // Enter returns the next state, if it returns empty string the SM will hold the current state or - // or decide the next state. - Enter() (string, error) - //Exit should be idempotent - Exit() error +type RepJobParm struct { + LocalRegURL string + TargetURL string + TargetUsername string + TargetPassword string + Repository string + Enabled int + Operation string } -type DummyHandler struct { - JobID int64 -} - -func (dh DummyHandler) Enter() (string, error) { - return "", nil -} - -func (dh DummyHandler) Exit() error { - return nil -} - -type StatusUpdater struct { - DummyHandler - State string -} - -func (su StatusUpdater) Enter() (string, error) { - err := dao.UpdateJobStatus(su.JobID, su.State) - if err != nil { - log.Warningf("Failed to update state of job: %d, state: %s, error: %v", su.JobID, su.State, err) - } - var next string = JobContinue - if su.State == JobStopped || su.State == JobError || su.State == JobFinished { - next = "" - } - return next, err -} - -const ( - JobPending string = "pending" - JobRunning string = "running" - JobError string = "error" - JobStopped string = "stopped" - JobFinished string = "finished" - // statemachine will move to next possible state based on trasition table - JobContinue string = "_continue" -) - type JobSM struct { JobID int64 CurrentState string @@ -64,8 +27,10 @@ type JobSM struct { ForcedStates map[string]struct{} Transitions map[string]map[string]struct{} Handlers map[string]StateHandler - lock *sync.Mutex desiredState string + Logger Logger + Parms *RepJobParm + lock *sync.Mutex } // EnsterState transit the statemachine from the current state to the state in parameter. @@ -87,7 +52,7 @@ func (sm *JobSM) EnterState(s string) (string, error) { log.Debugf("No handler found for state:%s, skip", sm.CurrentState) } enterHandler, ok := sm.Handlers[s] - var next string = JobContinue + var next string = models.JobContinue var err error if ok { if next, err = enterHandler.Enter(); err != nil { @@ -115,14 +80,14 @@ func (sm *JobSM) Start(s string) { sm.setDesiredState("") continue } - if n == JobContinue && len(sm.Transitions[sm.CurrentState]) == 1 { + if n == models.JobContinue && len(sm.Transitions[sm.CurrentState]) == 1 { for n = range sm.Transitions[sm.CurrentState] { break } log.Debugf("Continue to state: %s", n) continue } - if n == JobContinue && len(sm.Transitions[sm.CurrentState]) != 1 { + if n == models.JobContinue && len(sm.Transitions[sm.CurrentState]) != 1 { log.Errorf("Next state is continue but there are %d possible next states in transition table", len(sm.Transitions[sm.CurrentState])) err = fmt.Errorf("Unable to continue") break @@ -132,7 +97,7 @@ func (sm *JobSM) Start(s string) { } if err != nil { log.Warningf("The statemachin will enter error state due to error: %v", err) - sm.EnterState(JobError) + sm.EnterState(models.JobError) } } @@ -154,7 +119,7 @@ func (sm *JobSM) RemoveTransition(from string, to string) { } func (sm *JobSM) Stop() { - sm.setDesiredState(JobStopped) + sm.setDesiredState(models.JobStopped) } func (sm *JobSM) getDesiredState() string { @@ -169,12 +134,60 @@ func (sm *JobSM) setDesiredState(s string) { sm.desiredState = s } -func (sm *JobSM) InitJobSM() { +func (sm *JobSM) Init() error { + //init parms + regURL := os.Getenv("LOCAL_REGISTRY_URL") + if len(regURL) == 0 { + regURL = "http://registry:5000/" + } + job, err := dao.GetRepJob(sm.JobID) + if err != nil { + return fmt.Errorf("Failed to get job, error: %v", err) + } + if job == nil { + return fmt.Errorf("The job doesn't exist in DB, job id: %d", sm.JobID) + } + policy, err := dao.GetRepPolicy(job.PolicyID) + if err != nil { + return fmt.Errorf("Failed to get policy, error: %v", err) + } + if policy == nil { + return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID) + } + sm.Parms = &RepJobParm{ + LocalRegURL: regURL, + Repository: job.Repository, + Enabled: policy.Enabled, + Operation: job.Operation, + } + if policy.Enabled == 0 { + //handler will cancel this job + return nil + } + target, err := dao.GetRepTarget(policy.TargetID) + if err != nil { + return fmt.Errorf("Failed to get target, error: %v", err) + } + if target == nil { + return fmt.Errorf("The target doesn't exist in DB, target id: %d", policy.TargetID) + } + sm.Parms.TargetURL = target.URL + sm.Parms.TargetUsername = target.Username + sm.Parms.TargetPassword = target.Password + //init states handlers sm.lock = &sync.Mutex{} sm.Handlers = make(map[string]StateHandler) sm.Transitions = make(map[string]map[string]struct{}) - sm.CurrentState = JobPending - sm.AddTransition(JobPending, JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, JobRunning}) - sm.Handlers[JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, JobError} - sm.Handlers[JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, JobStopped} + sm.Logger = Logger{sm.JobID} + sm.CurrentState = models.JobPending + sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning}) + sm.Handlers[models.JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobError} + sm.Handlers[models.JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobStopped} + if sm.Parms.Operation == models.RepOpTransfer { + sm.AddTransition(models.JobRunning, "pull-img", ImgPuller{DummyHandler: DummyHandler{JobID: sm.JobID}, img: sm.Parms.Repository, logger: sm.Logger}) + //only handle on target for now + sm.AddTransition("pull-img", "push-img", ImgPusher{DummyHandler: DummyHandler{JobID: sm.JobID}, targetURL: sm.Parms.TargetURL, logger: sm.Logger}) + sm.AddTransition("push-img", models.JobFinished, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished}) + } + return nil } diff --git a/jobservice/my_start.sh b/jobservice/my_start.sh index d2274be48..bbcff279c 100755 --- a/jobservice/my_start.sh +++ b/jobservice/my_start.sh @@ -3,5 +3,8 @@ export MYSQL_PORT=3306 export MYSQL_USR=root export MYSQL_PWD=root123 export LOG_LEVEL=debug +export UI_URL=http://127.0.0.1/ +export UI_USR=admin +export UI_PWD=Harbor12345 ./jobservice diff --git a/jobservice/newtest.json b/jobservice/newtest.json new file mode 100644 index 000000000..9a40fa461 --- /dev/null +++ b/jobservice/newtest.json @@ -0,0 +1 @@ +{"policy_id": 1} diff --git a/jobservice/populate.sql b/jobservice/populate.sql new file mode 100644 index 000000000..64fb2d360 --- /dev/null +++ b/jobservice/populate.sql @@ -0,0 +1,2 @@ +insert into replication_target (name, url, username, password) values ('test', '192.168.0.2:5000', 'testuser', 'passw0rd'); +insert into replication_policy (name, project_id, target_id, enabled, start_time) value ('test_policy', 1, 1, 1, NOW()); diff --git a/jobservice/router.go b/jobservice/router.go index eb70e52fa..a9c028038 100644 --- a/jobservice/router.go +++ b/jobservice/router.go @@ -7,5 +7,5 @@ import ( ) func initRouters() { - beego.Router("/api/jobs/?:id", &api.JobAPI{}) + beego.Router("/api/jobs/replication", &api.ReplicationJob{}) } diff --git a/migration/Dockerfile b/migration/Dockerfile new file mode 100644 index 000000000..507342170 --- /dev/null +++ b/migration/Dockerfile @@ -0,0 +1,23 @@ +FROM mysql:5.6 + +MAINTAINER bhe@vmware.com + +RUN sed -i -e 's/us.archive.ubuntu.com/archive.ubuntu.com/g' /etc/apt/sources.list + +RUN apt-get update + +RUN apt-get install -y curl python python-pip git python-mysqldb + +RUN pip install alembic + +RUN mkdir -p /harbor-migration + +WORKDIR /harbor-migration + +COPY ./ ./ + +COPY ./migration.cfg ./ + +RUN ./prepare.sh + +ENTRYPOINT ["./run.sh"] diff --git a/migration/README.md b/migration/README.md new file mode 100644 index 000000000..1d013e47f --- /dev/null +++ b/migration/README.md @@ -0,0 +1,51 @@ +# migration +Migration is a module for migrating database schema between different version of project [harbor](https://github.com/vmware/harbor) + +**WARNING!!** You must backup your data before migrating + +###installation +- step 1: modify migration.cfg +- step 2: build image from dockerfile + ``` + cd harbor-migration + + docker build -t your-image-name . + ``` + +###migration operation +- show instruction of harbor-migration + + ```docker run your-image-name help``` + +- create backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name backup + ``` + +- restore from backup file in `/path/to/backup` + + ``` + docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name restore + ``` + +- perform database schema upgrade + + ```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head``` + +- perform database schema downgrade(downgrade has been disabled) + + ```docker run -v /data/database:/var/lib/mysql your-image-name down base``` + +###migration step +- step 1: stop and remove harbor service + + ``` + docker-compose stop && docker-compose rm -f + ``` +- step 2: perform migration operation +- step 3: rebuild newest harbor images and restart service + + ``` + docker-compose build && docker-compose up -d + ``` diff --git a/migration/alembic.sql b/migration/alembic.sql new file mode 100644 index 000000000..21dc7a1de --- /dev/null +++ b/migration/alembic.sql @@ -0,0 +1,4 @@ +use `registry`; +CREATE TABLE IF NOT EXISTS `alembic_version` ( + `version_num` varchar(32) NOT NULL +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/migration/alembic.tpl b/migration/alembic.tpl new file mode 100644 index 000000000..548c6028d --- /dev/null +++ b/migration/alembic.tpl @@ -0,0 +1,68 @@ +echo " +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = migration_harbor + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to migration_harbor/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat migration_harbor/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = mysql://$db_username:$db_password@localhost:$db_port/$db_name + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S" diff --git a/migration/migration.cfg b/migration/migration.cfg new file mode 100644 index 000000000..a383853ac --- /dev/null +++ b/migration/migration.cfg @@ -0,0 +1,4 @@ +db_username="root" +db_password="root123" +db_port="3306" +db_name="registry" diff --git a/migration/migration_harbor/env.py b/migration/migration_harbor/env.py new file mode 100644 index 000000000..646f39862 --- /dev/null +++ b/migration/migration_harbor/env.py @@ -0,0 +1,85 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata + +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migration/migration_harbor/script.py.mako b/migration/migration_harbor/script.py.mako new file mode 100644 index 000000000..43c09401b --- /dev/null +++ b/migration/migration_harbor/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migration/migration_harbor/versions/0_1_1.py b/migration/migration_harbor/versions/0_1_1.py new file mode 100644 index 000000000..0f21b5436 --- /dev/null +++ b/migration/migration_harbor/versions/0_1_1.py @@ -0,0 +1,128 @@ +# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""0.1.0 to 0.1.1 + +Revision ID: 0.1.1 +Revises: +Create Date: 2016-04-18 18:32:14.101897 + +""" + +# revision identifiers, used by Alembic. +revision = '0.1.1' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker, relationship +from datetime import datetime + +Session = sessionmaker() + +Base = declarative_base() + +class Properties(Base): + __tablename__ = 'properties' + + k = sa.Column(sa.String(64), primary_key = True) + v = sa.Column(sa.String(128), nullable = False) + +class ProjectMember(Base): + __tablename__ = 'project_member' + + project_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), primary_key = True) + role = sa.Column(sa.Integer(), nullable = False) + creation_time = sa.Column(sa.DateTime(), nullable = True) + update_time = sa.Column(sa.DateTime(), nullable = True) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ), + sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ), + sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ), + +class UserProjectRole(Base): + __tablename__ = 'user_project_role' + + upr_id = sa.Column(sa.Integer(), primary_key = True) + user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id')) + pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id')) + project_role = relationship("ProjectRole") + +class ProjectRole(Base): + __tablename__ = 'project_role' + + pr_id = sa.Column(sa.Integer(), primary_key = True) + project_id = sa.Column(sa.Integer(), nullable = False) + role_id = sa.Column(sa.Integer(), nullable = False) + sa.ForeignKeyConstraint(['role_id'], [u'role.role_id']) + sa.ForeignKeyConstraint(['project_id'], [u'project.project_id']) + +class Access(Base): + __tablename__ = 'access' + + access_id = sa.Column(sa.Integer(), primary_key = True) + access_code = sa.Column(sa.String(1)) + comment = sa.Column(sa.String(30)) + +def upgrade(): + """ + update schema&data + """ + bind = op.get_bind() + session = Session(bind=bind) + + #delete M from table access + acc = session.query(Access).filter_by(access_id=1).first() + session.delete(acc) + + #create table property + Properties.__table__.create(bind) + session.add(Properties(k='schema_version', v='0.1.1')) + + #create table project_member + ProjectMember.__table__.create(bind) + + #fill data + join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all() + for result in join_result: + session.add(ProjectMember(project_id=result.project_role.project_id, \ + user_id=result.user_id, role=result.project_role.role_id, \ + creation_time=datetime.now(), update_time=datetime.now())) + + #drop user_project_role table before drop project_role + #because foreign key constraint + op.drop_table('user_project_role') + op.drop_table('project_role') + + #add column to table project + op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True)) + + #add column to table role + op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False)) + + #add column to table user + op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True)) + op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True)) + op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True)) + session.commit() + +def downgrade(): + """ + Downgrade has been disabled. + """ + pass diff --git a/migration/prepare.sh b/migration/prepare.sh new file mode 100755 index 000000000..4d707407c --- /dev/null +++ b/migration/prepare.sh @@ -0,0 +1,3 @@ +#!/bin/bash +source ./migration.cfg +source ./alembic.tpl > ./alembic.ini diff --git a/migration/run.sh b/migration/run.sh new file mode 100755 index 000000000..806378ff5 --- /dev/null +++ b/migration/run.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +source ./migration.cfg + +WAITTIME=60 + +DBCNF="-hlocalhost -u${db_username}" + +#prevent shell to print insecure message +export MYSQL_PWD="${db_password}" + +if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then + echo "Usage:" + echo "backup perform database backup" + echo "restore perform database restore" + echo "up, upgrade perform database schema upgrade" + echo "h, help usage help" + exit 0 +fi + +if [[ $1 = "up" || $1 = "upgrade" ]]; then + echo "Please backup before upgrade." + read -p "Enter y to continue updating or n to abort:" ans + case $ans in + [Yy]* ) + ;; + [Nn]* ) + exit 0 + ;; + * ) echo "illegal answer: $ans. Upgrade abort!!" + exit 1 + ;; + esac + +fi + +echo 'Trying to start mysql server...' +DBRUN=0 +nohup mysqld 2>&1 > ./nohup.log& +for i in $(seq 1 $WAITTIME); do + echo "$(/usr/sbin/service mysql status)" + if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then + sleep 1 + else + DBRUN=1 + break + fi +done + +if [[ $DBRUN -eq 0 ]]; then + echo "timeout. Can't run mysql server." + exit 1 +fi + +key="$1" +case $key in +up|upgrade) + VERSION="$2" + if [[ -z $VERSION ]]; then + VERSION="head" + echo "Version is not specified. Default version is head." + fi + echo "Performing upgrade ${VERSION}..." + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='alembic_version';") -eq 0 ]]; then + echo "table alembic_version does not exist. Trying to initial alembic_version." + mysql $DBCNF < ./alembic.sql + #compatible with version 0.1.0 and 0.1.1 + if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \ + where table_schema='registry' and table_name='properties'") -eq 0 ]]; then + echo "table properties does not exist. The version of registry is 0.1.0" + else + echo "The version of registry is 0.1.1" + mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')" + fi + fi + alembic -c ./alembic.ini upgrade ${VERSION} + echo "Upgrade performed." + ;; +backup) + echo "Performing backup..." + mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql + echo "Backup performed." + ;; +restore) + echo "Performing restore..." + mysql $DBCNF < ./backup/registry.sql + echo "Restore performed." + ;; +*) + echo "unknown option" + exit 0 + ;; +esac diff --git a/models/base.go b/models/base.go new file mode 100644 index 000000000..fdd0204ac --- /dev/null +++ b/models/base.go @@ -0,0 +1,11 @@ +package models + +import ( + "github.com/astaxie/beego/orm" +) + +func init() { + orm.RegisterModel(new(RepTarget), + new(RepPolicy), + new(RepJob)) +} diff --git a/models/replication_job.go b/models/replication_job.go new file mode 100644 index 000000000..885cb311c --- /dev/null +++ b/models/replication_job.go @@ -0,0 +1,65 @@ +package models + +import ( + "time" +) + +const ( + JobPending string = "pending" + JobRunning string = "running" + JobError string = "error" + JobStopped string = "stopped" + JobFinished string = "finished" + JobCanceled string = "canceled" + // statemachine will move to next possible state based on trasition table + JobContinue string = "_continue" + RepOpTransfer string = "transfer" + RepOpDelete string = "delete" +) + +type RepPolicy struct { + ID int64 `orm:"column(id)" json:"id"` + ProjectID int64 `orm:"column(project_id)" json:"project_id"` + TargetID int64 `orm:"column(target_id)" json:"target_id"` + Name string `orm:"column(name)" json:"name"` + Target RepTarget `orm:"-" json:"target"` + Enabled int `orm:"column(enabled)" json:"enabled"` + Description string `orm:"column(description)" json:"description"` + CronStr string `orm:"column(cron_str)" json:"cron_str"` + StartTime time.Time `orm:"column(start_time)" json:"start_time"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +type RepJob struct { + ID int64 `orm:"column(id)" json:"id"` + Status string `orm:"column(status)" json:"status"` + Repository string `orm:"column(repository)" json:"repository"` + PolicyID int64 `orm:"column(policy_id)" json:"policy_id"` + Operation string `orm:"column(operation)" json:"operation"` + Policy RepPolicy `orm:"-" json:"policy"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +type RepTarget struct { + ID int64 `orm:"column(id)" json:"id"` + URL string `orm:"column(url)" json:"url"` + Name string `orm:"column(name)" json:"name"` + Username string `orm:"column(username)" json:"username"` + Password string `orm:"column(password)" json:"password"` + CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"` + UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"` +} + +func (rt *RepTarget) TableName() string { + return "replication_target" +} + +func (rj *RepJob) TableName() string { + return "replication_job" +} + +func (rp *RepPolicy) TableName() string { + return "replication_policy" +} diff --git a/service/token/authutils.go b/service/token/authutils.go index bb9d8ad92..3ea15c294 100644 --- a/service/token/authutils.go +++ b/service/token/authutils.go @@ -39,6 +39,7 @@ const ( // GetResourceActions ... func GetResourceActions(scopes []string) []*token.ResourceActions { + log.Debugf("scopes: %+v", scopes) var res []*token.ResourceActions for _, s := range scopes { if s == "" { @@ -59,6 +60,7 @@ func GetResourceActions(scopes []string) []*token.ResourceActions { func FilterAccess(username string, authenticated bool, a *token.ResourceActions) { if a.Type == "registry" && a.Name == "catalog" { + log.Infof("current access, type: %s, name:%s, actions:%v \n", a.Type, a.Name, a.Actions) return } @@ -108,7 +110,7 @@ func FilterAccess(username string, authenticated bool, a *token.ResourceActions) } // GenTokenForUI is for the UI process to call, so it won't establish a https connection from UI to proxy. -func GenTokenForUI(username string, service string, scopes []string) (string, error) { +func GenTokenForUI(username string, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { access := GetResourceActions(scopes) for _, a := range access { FilterAccess(username, true, a) @@ -117,22 +119,22 @@ func GenTokenForUI(username string, service string, scopes []string) (string, er } // MakeToken makes a valid jwt token based on parms. -func MakeToken(username, service string, access []*token.ResourceActions) (string, error) { +func MakeToken(username, service string, access []*token.ResourceActions) (token string, expiresIn int, issuedAt *time.Time, err error) { pk, err := libtrust.LoadKeyFile(privateKey) if err != nil { - return "", err + return "", 0, nil, err } - tk, err := makeTokenCore(issuer, username, service, expiration, access, pk) + tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk) if err != nil { - return "", err + return "", 0, nil, err } rs := fmt.Sprintf("%s.%s", tk.Raw, base64UrlEncode(tk.Signature)) - return rs, nil + return rs, expiresIn, issuedAt, nil } //make token core func makeTokenCore(issuer, subject, audience string, expiration int, - access []*token.ResourceActions, signingKey libtrust.PrivateKey) (*token.Token, error) { + access []*token.ResourceActions, signingKey libtrust.PrivateKey) (t *token.Token, expiresIn int, issuedAt *time.Time, err error) { joseHeader := &token.Header{ Type: "JWT", @@ -142,10 +144,12 @@ func makeTokenCore(issuer, subject, audience string, expiration int, jwtID, err := randString(16) if err != nil { - return nil, fmt.Errorf("Error to generate jwt id: %s", err) + return nil, 0, nil, fmt.Errorf("Error to generate jwt id: %s", err) } - now := time.Now() + now := time.Now().UTC() + issuedAt = &now + expiresIn = expiration * 60 claimSet := &token.ClaimSet{ Issuer: issuer, @@ -161,10 +165,10 @@ func makeTokenCore(issuer, subject, audience string, expiration int, var joseHeaderBytes, claimSetBytes []byte if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil { - return nil, fmt.Errorf("unable to marshal jose header: %s", err) + return nil, 0, nil, fmt.Errorf("unable to marshal jose header: %s", err) } if claimSetBytes, err = json.Marshal(claimSet); err != nil { - return nil, fmt.Errorf("unable to marshal claim set: %s", err) + return nil, 0, nil, fmt.Errorf("unable to marshal claim set: %s", err) } encodedJoseHeader := base64UrlEncode(joseHeaderBytes) @@ -173,12 +177,13 @@ func makeTokenCore(issuer, subject, audience string, expiration int, var signatureBytes []byte if signatureBytes, _, err = signingKey.Sign(strings.NewReader(payload), crypto.SHA256); err != nil { - return nil, fmt.Errorf("unable to sign jwt payload: %s", err) + return nil, 0, nil, fmt.Errorf("unable to sign jwt payload: %s", err) } signature := base64UrlEncode(signatureBytes) tokenString := fmt.Sprintf("%s.%s", payload, signature) - return token.NewToken(tokenString) + t, err = token.NewToken(tokenString) + return } func randString(length int) (string, error) { diff --git a/service/token/token.go b/service/token/token.go index c12e4d7ad..2d4c77673 100644 --- a/service/token/token.go +++ b/service/token/token.go @@ -17,6 +17,7 @@ package token import ( "net/http" + "time" "github.com/vmware/harbor/auth" "github.com/vmware/harbor/models" @@ -43,7 +44,6 @@ func (h *Handler) Get() { authenticated := authenticate(username, password) service := h.GetString("service") scopes := h.GetStrings("scope") - log.Debugf("scopes: %+v", scopes) if len(scopes) == 0 && !authenticated { log.Info("login request with invalid credentials") @@ -59,14 +59,16 @@ func (h *Handler) Get() { func (h *Handler) serveToken(username, service string, access []*token.ResourceActions) { writer := h.Ctx.ResponseWriter //create token - rawToken, err := MakeToken(username, service, access) + rawToken, expiresIn, issuedAt, err := MakeToken(username, service, access) if err != nil { log.Errorf("Failed to make token, error: %v", err) writer.WriteHeader(http.StatusInternalServerError) return } - tk := make(map[string]string) + tk := make(map[string]interface{}) tk["token"] = rawToken + tk["expires_in"] = expiresIn + tk["issued_at"] = issuedAt.Format(time.RFC3339) h.Data["json"] = tk h.ServeJSON() } diff --git a/service/utils/cache.go b/service/utils/cache.go index a97a4599c..a19753f59 100644 --- a/service/utils/cache.go +++ b/service/utils/cache.go @@ -25,10 +25,14 @@ import ( "github.com/astaxie/beego/cache" ) -// Cache is the global cache in system. -var Cache cache.Cache - -var registryClient *registry.Registry +var ( + // Cache is the global cache in system. + Cache cache.Cache + endpoint string + username string + registryClient *registry.Registry + repositoryClients map[string]*registry.Repository +) const catalogKey string = "catalog" @@ -39,17 +43,25 @@ func init() { log.Errorf("Failed to initialize cache, error:%v", err) } - endpoint := os.Getenv("REGISTRY_URL") - client := registry.NewClientUsernameAuthHandlerEmbeded("admin") - registryClient, err = registry.New(endpoint, client) - if err != nil { - log.Fatalf("error occurred while initializing authentication handler used by cache: %v", err) - } + endpoint = os.Getenv("REGISTRY_URL") + username = "admin" + repositoryClients = make(map[string]*registry.Repository, 10) } // RefreshCatalogCache calls registry's API to get repository list and write it to cache. func RefreshCatalogCache() error { log.Debug("refreshing catalog cache...") + + if registryClient == nil { + var err error + registryClient, err = registry.NewRegistryWithUsername(endpoint, username) + if err != nil { + log.Errorf("error occurred while initializing registry client used by cache: %v", err) + return err + } + } + + var err error rs, err := registryClient.Catalog() if err != nil { return err @@ -58,10 +70,19 @@ func RefreshCatalogCache() error { repos := []string{} for _, repo := range rs { - tags, err := registryClient.ListTag(repo) + rc, ok := repositoryClients[repo] + if !ok { + rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username) + if err != nil { + log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err) + continue + } + repositoryClients[repo] = rc + } + tags, err := rc.ListTag() if err != nil { log.Errorf("error occurred while list tag for %s: %v", repo, err) - return err + continue } if len(tags) != 0 { diff --git a/utils/registry/auth/authorizer.go b/utils/registry/auth/authorizer.go new file mode 100644 index 000000000..cea731246 --- /dev/null +++ b/utils/registry/auth/authorizer.go @@ -0,0 +1,60 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package auth + +import ( + "net/http" + + au "github.com/docker/distribution/registry/client/auth" +) + +// Handler authorizes requests according to the schema +type Handler interface { + // Scheme : basic, bearer + Scheme() string + //AuthorizeRequest adds basic auth or token auth to the header of request + AuthorizeRequest(req *http.Request, params map[string]string) error +} + +// RequestAuthorizer holds a handler list, which will authorize request. +// Implements interface RequestModifier +type RequestAuthorizer struct { + handlers []Handler + challenges []au.Challenge +} + +// NewRequestAuthorizer ... +func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *RequestAuthorizer { + return &RequestAuthorizer{ + handlers: handlers, + challenges: challenges, + } +} + +// ModifyRequest adds authorization to the request +func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error { + for _, handler := range r.handlers { + for _, challenge := range r.challenges { + if handler.Scheme() == challenge.Scheme { + if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/utils/registry/auth/credential.go b/utils/registry/auth/credential.go new file mode 100644 index 000000000..6dd867136 --- /dev/null +++ b/utils/registry/auth/credential.go @@ -0,0 +1,44 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package auth + +import ( + "net/http" +) + +// Credential ... +type Credential interface { + // AddAuthorization adds authorization information to request + AddAuthorization(req *http.Request) +} + +// Implements interface Credential +type basicAuthCredential struct { + username string + password string +} + +// NewBasicAuthCredential ... +func NewBasicAuthCredential(username, password string) Credential { + return &basicAuthCredential{ + username: username, + password: password, + } +} + +func (b *basicAuthCredential) AddAuthorization(req *http.Request) { + req.SetBasicAuth(b.username, b.password) +} diff --git a/utils/registry/auth/handler.go b/utils/registry/auth/handler.go deleted file mode 100644 index 61550178d..000000000 --- a/utils/registry/auth/handler.go +++ /dev/null @@ -1,197 +0,0 @@ -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package auth - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "strings" - - token_util "github.com/vmware/harbor/service/token" - "github.com/vmware/harbor/utils/log" - registry_errors "github.com/vmware/harbor/utils/registry/errors" -) - -const ( - // credential type - basicAuth string = "basic_auth" - secretKey string = "secret_key" -) - -// Handler authorizes the request when encounters a 401 error -type Handler interface { - // Schema : basic, bearer - Schema() string - //AuthorizeRequest adds basic auth or token auth to the header of request - AuthorizeRequest(req *http.Request, params map[string]string) error -} - -// Credential ... -type Credential interface { - // AddAuthorization adds authorization information to request - AddAuthorization(req *http.Request) -} - -type basicAuthCredential struct { - username string - password string -} - -// NewBasicAuthCredential ... -func NewBasicAuthCredential(username, password string) Credential { - return &basicAuthCredential{ - username: username, - password: password, - } -} - -func (b *basicAuthCredential) AddAuthorization(req *http.Request) { - req.SetBasicAuth(b.username, b.password) -} - -type token struct { - Token string `json:"token"` -} - -type standardTokenHandler struct { - client *http.Client - credential Credential -} - -// NewStandardTokenHandler returns a standard token handler. The handler will request a token -// from token server whose URL is specified in the "WWW-authentication" header and add it to -// the origin request -// TODO deal with https -func NewStandardTokenHandler(credential Credential) Handler { - return &standardTokenHandler{ - client: &http.Client{ - Transport: http.DefaultTransport, - }, - credential: credential, - } -} - -// Schema implements the corresponding method in interface AuthHandler -func (t *standardTokenHandler) Schema() string { - return "bearer" -} - -// AuthorizeRequest implements the corresponding method in interface AuthHandler -func (t *standardTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { - realm, ok := params["realm"] - if !ok { - return errors.New("no realm") - } - - service := params["service"] - scope := params["scope"] - - u, err := url.Parse(realm) - if err != nil { - return err - } - - q := u.Query() - q.Add("service", service) - - for _, s := range strings.Split(scope, " ") { - q.Add("scope", s) - } - - u.RawQuery = q.Encode() - - r, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return err - } - - t.credential.AddAuthorization(r) - - resp, err := t.client.Do(r) - if err != nil { - return err - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - if resp.StatusCode != http.StatusOK { - return registry_errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - } - - decoder := json.NewDecoder(resp.Body) - - tk := &token{} - if err = decoder.Decode(tk); err != nil { - return err - } - - req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk.Token)) - - log.Debugf("standardTokenHandler generated token successfully | %s %s", req.Method, req.URL) - - return nil -} - -type usernameTokenHandler struct { - username string -} - -// NewUsernameTokenHandler returns a handler which will generate -// a token according the user's privileges -func NewUsernameTokenHandler(username string) Handler { - return &usernameTokenHandler{ - username: username, - } -} - -// Schema implements the corresponding method in interface AuthHandler -func (u *usernameTokenHandler) Schema() string { - return "bearer" -} - -// AuthorizeRequest implements the corresponding method in interface AuthHandler -func (u *usernameTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { - service := params["service"] - - scopes := []string{} - scope := params["scope"] - if len(scope) != 0 { - scopes = strings.Split(scope, " ") - } - - token, err := token_util.GenTokenForUI(u.username, service, scopes) - if err != nil { - return err - } - - req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token)) - - log.Debugf("usernameTokenHandler generated token successfully | %s %s", req.Method, req.URL) - - return nil -} diff --git a/utils/registry/auth/tokenhandler.go b/utils/registry/auth/tokenhandler.go new file mode 100644 index 000000000..f546bac0c --- /dev/null +++ b/utils/registry/auth/tokenhandler.go @@ -0,0 +1,230 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package auth + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + "time" + + token_util "github.com/vmware/harbor/service/token" + "github.com/vmware/harbor/utils/log" + registry_errors "github.com/vmware/harbor/utils/registry/errors" +) + +type scope struct { + Type string + Name string + Actions []string +} + +func (s *scope) string() string { + return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ",")) +} + +type tokenGenerator func(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) + +// Implements interface Handler +type tokenHandler struct { + scope *scope + tg tokenGenerator + cache string // cached token + expiresIn int // The duration in seconds since the token was issued that it will remain valid + issuedAt *time.Time // The RFC3339-serialized UTC standard time at which a given token was issued +} + +// Scheme returns the scheme that the handler can handle +func (t *tokenHandler) Scheme() string { + return "bearer" +} + +// AuthorizeRequest will add authorization header which contains a token before the request is sent +func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + var scopes []*scope + var token string + + hasFrom := false + from := req.URL.Query().Get("from") + if len(from) != 0 { + s := &scope{ + Type: "repository", + Name: from, + Actions: []string{"pull"}, + } + scopes = append(scopes, s) + // do not cache the token if "from" appears + hasFrom = true + } + + scopes = append(scopes, t.scope) + + expired := true + + if t.expiresIn != 0 && t.issuedAt != nil { + expired = t.issuedAt.Add(time.Duration(t.expiresIn) * time.Second).Before(time.Now().UTC()) + } + + if expired || hasFrom { + scopeStrs := []string{} + for _, scope := range scopes { + scopeStrs = append(scopeStrs, scope.string()) + } + to, expiresIn, issuedAt, err := t.tg(params["realm"], params["service"], scopeStrs) + if err != nil { + return err + } + token = to + + if !hasFrom { + t.cache = token + t.expiresIn = expiresIn + t.issuedAt = issuedAt + log.Debug("add token to cache") + } + } else { + token = t.cache + log.Debug("get token from cache") + } + + req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token)) + log.Debugf("add token to request: %s %s", req.Method, req.URL.String()) + + return nil +} + +// Implements interface Handler +type standardTokenHandler struct { + tokenHandler + client *http.Client + credential Credential +} + +// NewStandardTokenHandler returns a standard token handler. The handler will request a token +// from token server and add it to the origin request +// TODO deal with https +func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, scopeActions ...string) Handler { + handler := &standardTokenHandler{ + client: &http.Client{ + Transport: http.DefaultTransport, + }, + credential: credential, + } + + handler.scope = &scope{ + Type: scopeType, + Name: scopeName, + Actions: scopeActions, + } + handler.tg = handler.generateToken + + return handler +} + +func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { + u, err := url.Parse(realm) + if err != nil { + return + } + q := u.Query() + q.Add("service", service) + for _, scope := range scopes { + q.Add("scope", scope) + } + u.RawQuery = q.Encode() + r, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return + } + + s.credential.AddAuthorization(r) + + resp, err := s.client.Do(r) + if err != nil { + return + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + if resp.StatusCode != http.StatusOK { + err = registry_errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + return + } + + tk := struct { + Token string `json:"token"` + ExpiresIn int `json:"expires_in"` + IssuedAt string `json:"issued_at"` + }{} + if err = json.Unmarshal(b, &tk); err != nil { + return + } + + token = tk.Token + + expiresIn = tk.ExpiresIn + + t, err := time.Parse(time.RFC3339, tk.IssuedAt) + if err != nil { + log.Errorf("error occurred while parsing issued_at: %v", err) + err = nil + } else { + issuedAt = &t + } + + log.Debug("get token from token server") + + return +} + +// Implements interface Handler +type usernameTokenHandler struct { + tokenHandler + username string +} + +// NewUsernameTokenHandler returns a handler which will generate a token according to +// the user's privileges +func NewUsernameTokenHandler(username string, scopeType, scopeName string, scopeActions ...string) Handler { + handler := &usernameTokenHandler{ + username: username, + } + + handler.scope = &scope{ + Type: scopeType, + Name: scopeName, + Actions: scopeActions, + } + + handler.tg = handler.generateToken + + return handler +} + +func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) { + token, expiresIn, issuedAt, err = token_util.GenTokenForUI(u.username, service, scopes) + log.Debug("get token by calling GenTokenForUI directly") + return +} diff --git a/utils/registry/httpclient.go b/utils/registry/httpclient.go deleted file mode 100644 index f23a2c064..000000000 --- a/utils/registry/httpclient.go +++ /dev/null @@ -1,116 +0,0 @@ -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - "net/http" - - "github.com/vmware/harbor/utils/log" - "github.com/vmware/harbor/utils/registry/auth" -) - -// NewClient returns a http.Client according to the handlers provided -func NewClient(handlers []auth.Handler) *http.Client { - transport := NewAuthTransport(http.DefaultTransport, handlers) - - return &http.Client{ - Transport: transport, - } -} - -// NewClientStandardAuthHandlerEmbeded return a http.Client which will authorize the request -// according to the credential provided and send it again when encounters a 401 error -func NewClientStandardAuthHandlerEmbeded(credential auth.Credential) *http.Client { - handlers := []auth.Handler{} - - tokenHandler := auth.NewStandardTokenHandler(credential) - - handlers = append(handlers, tokenHandler) - - return NewClient(handlers) -} - -// NewClientUsernameAuthHandlerEmbeded return a http.Client which will authorize the request -// according to the user's privileges and send it again when encounters a 401 error -func NewClientUsernameAuthHandlerEmbeded(username string) *http.Client { - handlers := []auth.Handler{} - - tokenHandler := auth.NewUsernameTokenHandler(username) - - handlers = append(handlers, tokenHandler) - - return NewClient(handlers) -} - -type authTransport struct { - transport http.RoundTripper - handlers []auth.Handler -} - -// NewAuthTransport wraps the AuthHandlers to be http.RounTripper -func NewAuthTransport(transport http.RoundTripper, handlers []auth.Handler) http.RoundTripper { - return &authTransport{ - transport: transport, - handlers: handlers, - } -} - -// RoundTrip ... -func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { - originResp, originErr := a.transport.RoundTrip(req) - - if originErr != nil { - return originResp, originErr - } - - log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) - - if originResp.StatusCode != http.StatusUnauthorized { - return originResp, nil - } - - challenges := auth.ParseChallengeFromResponse(originResp) - - reqChanged := false - for _, challenge := range challenges { - - scheme := challenge.Scheme - - for _, handler := range a.handlers { - if scheme != handler.Schema() { - log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) - continue - } - - if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { - return nil, err - } - reqChanged = true - } - } - - if !reqChanged { - log.Warning("no handler match scheme") - return originResp, nil - } - - resp, err := a.transport.RoundTrip(req) - if err == nil { - log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) - } - - return resp, err -} diff --git a/utils/registry/registry.go b/utils/registry/registry.go index e390c4c0d..1ee01892e 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -21,57 +21,68 @@ import ( "io/ioutil" "net/http" "net/url" + "strings" - "github.com/docker/distribution/manifest" - "github.com/docker/distribution/manifest/schema1" - "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/utils/log" + "github.com/vmware/harbor/utils/registry/auth" "github.com/vmware/harbor/utils/registry/errors" ) -// Registry holds information of a registry entiry +// Registry holds information of a registry entity type Registry struct { Endpoint *url.URL client *http.Client - ub *uRLBuilder } -type uRLBuilder struct { - root *url.URL -} +// NewRegistry returns an instance of registry +func NewRegistry(endpoint string, client *http.Client) (*Registry, error) { + endpoint = strings.TrimRight(endpoint, "/") -var ( - // ManifestVersion1 : schema 1 - ManifestVersion1 = manifest.Versioned{ - SchemaVersion: 1, - MediaType: schema1.MediaTypeManifest, - } - // ManifestVersion2 : schema 2 - ManifestVersion2 = manifest.Versioned{ - SchemaVersion: 2, - MediaType: schema2.MediaTypeManifest, - } -) - -// New returns an instance of Registry -func New(endpoint string, client *http.Client) (*Registry, error) { u, err := url.Parse(endpoint) if err != nil { return nil, err } - return &Registry{ + registry := &Registry{ Endpoint: u, client: client, - ub: &uRLBuilder{ - root: u, - }, - }, nil + } + + log.Debugf("initialized a registry client: %s", endpoint) + + return registry, nil +} + +// NewRegistryWithUsername returns a Registry instance which will authorize the request +// according to the privileges of user +func NewRegistryWithUsername(endpoint, username string) (*Registry, error) { + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + client, err := newClient(endpoint, username, nil, "registry", "catalog", "*") + if err != nil { + return nil, err + } + + registry := &Registry{ + Endpoint: u, + client: client, + } + + log.Debugf("initialized a registry client with username: %s %s", endpoint, username) + + return registry, nil } // Catalog ... func (r *Registry) Catalog() ([]string, error) { repos := []string{} - req, err := http.NewRequest("GET", r.ub.buildCatalogURL(), nil) + + req, err := http.NewRequest("GET", buildCatalogURL(r.Endpoint.String()), nil) if err != nil { return repos, err } @@ -108,209 +119,34 @@ func (r *Registry) Catalog() ([]string, error) { } } -// ListTag ... -func (r *Registry) ListTag(name string) ([]string, error) { - tags := []string{} - req, err := http.NewRequest("GET", r.ub.buildTagListURL(name), nil) - if err != nil { - return tags, err - } - - resp, err := r.client.Do(req) - if err != nil { - return tags, err - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return tags, err - } - - if resp.StatusCode == http.StatusOK { - tagsResp := struct { - Tags []string `json:"tags"` - }{} - - if err := json.Unmarshal(b, &tagsResp); err != nil { - return tags, err - } - - tags = tagsResp.Tags - - return tags, nil - } - - return tags, errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - +func buildCatalogURL(endpoint string) string { + return fmt.Sprintf("%s/v2/_catalog", endpoint) } -// ManifestExist ... -func (r *Registry) ManifestExist(name, reference string) (digest string, exist bool, err error) { - req, err := http.NewRequest("HEAD", r.ub.buildManifestURL(name, reference), nil) +func newClient(endpoint, username string, credential auth.Credential, + scopeType, scopeName string, scopeActions ...string) (*http.Client, error) { + + endpoint = strings.TrimRight(endpoint, "/") + resp, err := http.Get(buildPingURL(endpoint)) if err != nil { - return + return nil, err } - // request Schema 2 manifest, if the registry does not support it, - // Schema 1 manifest will be returned - req.Header.Set(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest) - - resp, err := r.client.Do(req) - if err != nil { - return + var handlers []auth.Handler + var handler auth.Handler + if credential != nil { + handler = auth.NewStandardTokenHandler(credential, scopeType, scopeName, scopeActions...) + } else { + handler = auth.NewUsernameTokenHandler(username, scopeType, scopeName, scopeActions...) } - if resp.StatusCode == http.StatusOK { - exist = true - digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) - return - } + handlers = append(handlers, handler) - if resp.StatusCode == http.StatusNotFound { - return - } + challenges := auth.ParseChallengeFromResponse(resp) + authorizer := auth.NewRequestAuthorizer(handlers, challenges) - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - err = errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - return -} - -// PullManifest ... -func (r *Registry) PullManifest(name, reference string, version manifest.Versioned) (digest, mediaType string, payload []byte, err error) { - req, err := http.NewRequest("GET", r.ub.buildManifestURL(name, reference), nil) - if err != nil { - return - } - - // if the registry does not support schema 2, schema 1 manifest will be returned - req.Header.Set(http.CanonicalHeaderKey("Accept"), version.MediaType) - - resp, err := r.client.Do(req) - if err != nil { - return - } - - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode == http.StatusOK { - digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) - mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type")) - payload = b - return - } - - err = errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - - return -} - -// DeleteManifest ... -func (r *Registry) DeleteManifest(name, digest string) error { - req, err := http.NewRequest("DELETE", r.ub.buildManifestURL(name, digest), nil) - if err != nil { - return err - } - - resp, err := r.client.Do(req) - if err != nil { - return err - } - - if resp.StatusCode == http.StatusAccepted { - return nil - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - return errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } -} - -// DeleteTag ... -func (r *Registry) DeleteTag(name, tag string) error { - digest, exist, err := r.ManifestExist(name, tag) - if err != nil { - return err - } - - if !exist { - return errors.Error{ - StatusCode: http.StatusNotFound, - } - } - - return r.DeleteManifest(name, digest) -} - -// DeleteBlob ... -func (r *Registry) DeleteBlob(name, digest string) error { - req, err := http.NewRequest("DELETE", r.ub.buildBlobURL(name, digest), nil) - if err != nil { - return err - } - - resp, err := r.client.Do(req) - if err != nil { - return err - } - - if resp.StatusCode == http.StatusAccepted { - return nil - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - return errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } -} - -func (u *uRLBuilder) buildCatalogURL() string { - return fmt.Sprintf("%s/v2/_catalog", u.root.String()) -} - -func (u *uRLBuilder) buildTagListURL(name string) string { - return fmt.Sprintf("%s/v2/%s/tags/list", u.root.String(), name) -} - -func (u *uRLBuilder) buildManifestURL(name, reference string) string { - return fmt.Sprintf("%s/v2/%s/manifests/%s", u.root.String(), name, reference) -} - -func (u *uRLBuilder) buildBlobURL(name, reference string) string { - return fmt.Sprintf("%s/v2/%s/blobs/%s", u.root.String(), name, reference) + transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) + return &http.Client{ + Transport: transport, + }, nil } diff --git a/utils/registry/repository.go b/utils/registry/repository.go new file mode 100644 index 000000000..507634415 --- /dev/null +++ b/utils/registry/repository.go @@ -0,0 +1,505 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/utils/log" + "github.com/vmware/harbor/utils/registry/auth" + "github.com/vmware/harbor/utils/registry/errors" +) + +// Repository holds information of a repository entity +type Repository struct { + Name string + Endpoint *url.URL + client *http.Client +} + +// TODO add agent to header of request, notifications need it + +// NewRepository returns an instance of Repository +func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) { + name = strings.TrimSpace(name) + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + repository := &Repository{ + Name: name, + Endpoint: u, + client: client, + } + + return repository, nil +} + +// NewRepositoryWithCredential returns a Repository instance which will authorize the request +// according to the credenttial +func NewRepositoryWithCredential(name, endpoint string, credential auth.Credential) (*Repository, error) { + name = strings.TrimSpace(name) + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + client, err := newClient(endpoint, "", credential, "repository", name, "pull", "push") + + repository := &Repository{ + Name: name, + Endpoint: u, + client: client, + } + + log.Debugf("initialized a repository client with credential: %s %s", endpoint, name) + + return repository, nil +} + +// NewRepositoryWithUsername returns a Repository instance which will authorize the request +// according to the privileges of user +func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, error) { + name = strings.TrimSpace(name) + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + client, err := newClient(endpoint, username, nil, "repository", name, "pull", "push") + + repository := &Repository{ + Name: name, + Endpoint: u, + client: client, + } + + log.Debugf("initialized a repository client with username: %s %s %s", endpoint, name, username) + + return repository, nil +} + +// ListTag ... +func (r *Repository) ListTag() ([]string, error) { + tags := []string{} + req, err := http.NewRequest("GET", buildTagListURL(r.Endpoint.String(), r.Name), nil) + if err != nil { + return tags, err + } + + resp, err := r.client.Do(req) + if err != nil { + return tags, err + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return tags, err + } + + if resp.StatusCode == http.StatusOK { + tagsResp := struct { + Tags []string `json:"tags"` + }{} + + if err := json.Unmarshal(b, &tagsResp); err != nil { + return tags, err + } + + tags = tagsResp.Tags + + return tags, nil + } + + return tags, errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + +} + +// ManifestExist ... +func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) { + req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil) + if err != nil { + return + } + + req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest) + req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest) + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusOK { + exist = true + digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + return + } + + if resp.StatusCode == http.StatusNotFound { + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + return +} + +// PullManifest ... +func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) { + req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil) + if err != nil { + return + } + + for _, mediaType := range acceptMediaTypes { + req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType) + } + + resp, err := r.client.Do(req) + if err != nil { + return + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + if resp.StatusCode == http.StatusOK { + digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type")) + payload = b + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +// PushManifest ... +func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) { + req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference), + bytes.NewReader(payload)) + if err != nil { + return + } + req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType) + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusCreated { + digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +// DeleteManifest ... +func (r *Repository) DeleteManifest(digest string) error { + req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusAccepted { + return nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// DeleteTag ... +func (r *Repository) DeleteTag(tag string) error { + digest, exist, err := r.ManifestExist(tag) + if err != nil { + return err + } + + if !exist { + return errors.Error{ + StatusCode: http.StatusNotFound, + } + } + + return r.DeleteManifest(digest) +} + +// BlobExist ... +func (r *Repository) BlobExist(digest string) (bool, error) { + req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return false, err + } + + resp, err := r.client.Do(req) + if err != nil { + return false, err + } + + if resp.StatusCode == http.StatusOK { + return true, nil + } + + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, err + } + + return false, errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// PullBlob ... +func (r *Repository) PullBlob(digest string) (size int64, data []byte, err error) { + req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return + } + + resp, err := r.client.Do(req) + if err != nil { + return + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + if resp.StatusCode == http.StatusOK { + contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length")) + size, err = strconv.ParseInt(contengLength, 10, 64) + if err != nil { + return + } + data = b + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) { + req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil) + req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0") + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusAccepted { + location = resp.Header.Get(http.CanonicalHeaderKey("Location")) + uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID")) + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data []byte) error { + req, err := http.NewRequest("PUT", buildMonolithicBlobUploadURL(location, digest), bytes.NewReader(data)) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusCreated { + return nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// PushBlob ... +func (r *Repository) PushBlob(digest string, size int64, data []byte) error { + exist, err := r.BlobExist(digest) + if err != nil { + return err + } + + if exist { + log.Infof("blob already exists, skip pushing: %s %s", r.Name, digest) + return nil + } + + location, _, err := r.initiateBlobUpload(r.Name) + if err != nil { + return err + } + + return r.monolithicBlobUpload(location, digest, size, data) +} + +// DeleteBlob ... +func (r *Repository) DeleteBlob(digest string) error { + req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusAccepted { + return nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +func buildPingURL(endpoint string) string { + return fmt.Sprintf("%s/v2/", endpoint) +} + +func buildTagListURL(endpoint, repoName string) string { + return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName) +} + +func buildManifestURL(endpoint, repoName, reference string) string { + return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference) +} + +func buildBlobURL(endpoint, repoName, reference string) string { + return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference) +} + +func buildInitiateBlobUploadURL(endpoint, repoName string) string { + return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName) +} + +func buildMonolithicBlobUploadURL(location, digest string) string { + return fmt.Sprintf("%s&digest=%s", location, digest) +} diff --git a/utils/registry/transport.go b/utils/registry/transport.go new file mode 100644 index 000000000..f2569b3cd --- /dev/null +++ b/utils/registry/transport.go @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "net/http" + + "github.com/vmware/harbor/utils/log" +) + +// RequestModifier modifies request +type RequestModifier interface { + ModifyRequest(*http.Request) error +} + +// Transport holds information about base transport and modifiers +type Transport struct { + transport http.RoundTripper + modifiers []RequestModifier +} + +// NewTransport ... +func NewTransport(transport http.RoundTripper, modifiers []RequestModifier) *Transport { + return &Transport{ + transport: transport, + modifiers: modifiers, + } +} + +// RoundTrip ... +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + for _, modifier := range t.modifiers { + if err := modifier.ModifyRequest(req); err != nil { + return nil, err + } + } + + resp, err := t.transport.RoundTrip(req) + if err != nil { + return nil, err + } + + log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL.String()) + + return resp, err +} diff --git a/views/segment/header-content.tpl b/views/segment/header-content.tpl index 464cd09aa..368a64d8c 100644 --- a/views/segment/header-content.tpl +++ b/views/segment/header-content.tpl @@ -57,6 +57,10 @@