diff --git a/AUTHORS b/AUTHORS index 8e9802ff4..e03722c33 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,17 +1,27 @@ # This file lists all individuals having contributed content to the repository. Alexander Zeitler +Alexey Erkak +Allen Heavey Amanda Zhang Benniu Ji Bobby Zhang Chaofeng Wu Daniel Jiang +Deshi Xiao +Guangping Fu Haining Henry Zhang Hao Xia Jack Liu +Jessy Zhang Kun Wang +Mahesh Paolini-Subramanya +Meng Wei +Nagarjun G Peng Zhao +Robin Naundorf Shan Zhu Victoria Zheng Wenkai Yin +Yahao He Yan Wang diff --git a/Deploy/db/registry.sql b/Deploy/db/registry.sql index bcda445b6..e38bb402b 100644 --- a/Deploy/db/registry.sql +++ b/Deploy/db/registry.sql @@ -138,17 +138,6 @@ create table replication_job ( update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP, PRIMARY KEY (id) ); - -create table job_log ( - log_id int NOT NULL AUTO_INCREMENT, - job_id int NOT NULL, - level varchar(64) NOT NULL, - message text, - creation_time timestamp, - update_time timestamp, - PRIMARY KEY (log_id), - FOREIGN KEY (job_id) REFERENCES replication_job (id) - ); create table properties ( k varchar(64) NOT NULL, diff --git a/README.md b/README.md index 950891463..479707edd 100644 --- a/README.md +++ b/README.md @@ -9,18 +9,18 @@ Project Harbor is an enterprise-class registry server, which extends the open source Docker Registry server by adding the functionality usually required by an enterprise, such as security, control, and management. Harbor is primarily designed to be a private registry - providing the needed security and control that enterprises require. It also helps minimize bandwidth usage, which is helpful to both improve productivity (local network access) as well as performance (for those with poor internet connectivity). ### Features -* **Role Based Access Control**: Users and docker repositories are organized via "projects", a user can have different permission for images under a namespace. -* **Graphical user portal**: User can easily browse, search docker repositories, manage projects/namespaces. +* **Role Based Access Control**: Users and Docker repositories are organized via "projects", a user can have different permission for images under a project. +* **Graphical user portal**: User can easily browse, search Docker repositories, manage projects/namespaces. * **AD/LDAP support**: Harbor integrates with existing enterprise AD/LDAP for user authentication and management. * **Auditing**: All the operations to the repositories are tracked. -* **Internationalization**: Already Localized for English, Chinese and German. More languages can be added. +* **Internationalization**: Already localized for English, Chinese, German and Russian. More languages can be added. * **RESTful API**: RESTful APIs for most administrative operations, easing intergration with external management platforms. ### Getting Started Harbor is self-contained and can be easily deployed via docker-compose (Quick-Start steps below). Refer to the [Installation and Configuration Guide](docs/installation_guide.md) for detailed information. **System requirements:** -Harbor only works with docker 1.10+ and docker-compose 1.6.0+, and an internet-connected host +Harbor only works with docker 1.10+ and docker-compose 1.6.0+, and an internet-connected host. 1. Get the source code: @@ -30,7 +30,7 @@ Harbor only works with docker 1.10+ and docker-compose 1.6.0+, and an internet-c 2. Edit the file **Deploy/harbor.cfg**, make necessary configuration changes such as hostname, admin password and mail server. Refer to [Installation and Configuration Guide](docs/installation_guide.md) for more info. -3. Install Harbor with the following commands. Note that the docker-compose process can take a while! +3. Install Harbor with the following commands. Note that the docker-compose process can take a while. ```sh $ cd Deploy @@ -56,9 +56,6 @@ For those who don't want to clone the source, or need to install Harbor on a ser For information on how to use Harbor, please see [User Guide](docs/user_guide.md) . -### Deploy Harbor on Kubernetes -Detailed instruction about deploying Harbor on Kubernetes is available [here](docs/kubernetes_deployment.md). - ### Contribution We welcome contributions from the community. If you wish to contribute code and you have not signed our contributor license agreement (CLA), our bot will update the issue when you open a pull request. For any questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq). @@ -67,6 +64,7 @@ Harbor is available under the [Apache 2 license](LICENSE). ### Partners DataMan     SlamTec +    CaiCloud ### Users MaDaiLiCai diff --git a/api/jobs/replication.go b/api/jobs/replication.go index 3cd63206f..e9a7c4afe 100644 --- a/api/jobs/replication.go +++ b/api/jobs/replication.go @@ -6,6 +6,7 @@ import ( "github.com/vmware/harbor/api" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/job" + "github.com/vmware/harbor/job/utils" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" "io/ioutil" @@ -21,7 +22,9 @@ type ReplicationJob struct { } type ReplicationReq struct { - PolicyID int64 `json:"policy_id"` + PolicyID int64 `json:"policy_id"` + Repo string `json:"repository"` + Operation string `json:"operation"` } func (rj *ReplicationJob) Post() { @@ -39,31 +42,54 @@ func (rj *ReplicationJob) Post() { rj.RenderError(http.StatusNotFound, fmt.Sprintf("Policy not found, id: %d", data.PolicyID)) return } - repoList, err := getRepoList(p.ProjectID) - if err != nil { - log.Errorf("Failed to get repository list, project id: %d, error: %v", p.ProjectID, err) - rj.RenderError(http.StatusInternalServerError, err.Error()) - return - } - log.Debugf("repo list: %v", repoList) - for _, repo := range repoList { - j := models.RepJob{ - Repository: repo, - PolicyID: data.PolicyID, - Operation: models.RepOpTransfer, + if len(data.Repo) == 0 { // sync all repositories + repoList, err := getRepoList(p.ProjectID) + if err != nil { + log.Errorf("Failed to get repository list, project id: %d, error: %v", p.ProjectID, err) + rj.RenderError(http.StatusInternalServerError, err.Error()) + return } - log.Debugf("Creating job for repo: %s, policy: %d", repo, data.PolicyID) - id, err := dao.AddRepJob(j) + log.Debugf("repo list: %v", repoList) + for _, repo := range repoList { + err := rj.addJob(repo, data.PolicyID, models.RepOpTransfer) + if err != nil { + log.Errorf("Failed to insert job record, error: %v", err) + rj.RenderError(http.StatusInternalServerError, err.Error()) + return + } + } + } else { // sync a single repository + var op string + if len(data.Operation) > 0 { + op = data.Operation + } else { + op = models.RepOpTransfer + } + err := rj.addJob(data.Repo, data.PolicyID, op) if err != nil { log.Errorf("Failed to insert job record, error: %v", err) rj.RenderError(http.StatusInternalServerError, err.Error()) return } - log.Debugf("Send job to scheduler, job id: %d", id) - job.Schedule(id) } } +func (rj *ReplicationJob) addJob(repo string, policyID int64, operation string) error { + j := models.RepJob{ + Repository: repo, + PolicyID: policyID, + Operation: operation, + } + log.Debugf("Creating job for repo: %s, policy: %d", repo, policyID) + id, err := dao.AddRepJob(j) + if err != nil { + return err + } + log.Debugf("Send job to scheduler, job id: %d", id) + job.Schedule(id) + return nil +} + type RepActionReq struct { PolicyID int64 `json:"policy_id"` Action string `json:"action"` @@ -91,6 +117,18 @@ func (rj *ReplicationJob) HandleAction() { job.WorkerPool.StopJobs(jobIDList) } +func (rj *ReplicationJob) GetLog() { + idStr := rj.Ctx.Input.Param(":id") + jid, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + log.Errorf("Error parsing job id: %s, error: %v", idStr, err) + rj.RenderError(http.StatusBadRequest, "Invalid job id") + return + } + logFile := utils.GetJobLogPath(jid) + rj.Ctx.Output.Download(logFile) +} + // calls the api from UI to get repo list func getRepoList(projectID int64) ([]string, error) { uiURL := os.Getenv("UI_URL") diff --git a/api/project.go b/api/project.go index b0c404c2b..90ccbdb8c 100644 --- a/api/project.go +++ b/api/project.go @@ -113,23 +113,50 @@ func (p *ProjectAPI) Head() { // Get ... func (p *ProjectAPI) Get() { - queryProject := models.Project{UserID: p.userID} + var projectList []models.Project projectName := p.GetString("project_name") if len(projectName) > 0 { - queryProject.Name = "%" + projectName + "%" + projectName = "%" + projectName + "%" + } + var public int + var err error + isPublic := p.GetString("is_public") + if len(isPublic) > 0 { + public, err = strconv.Atoi(isPublic) + if err != nil { + log.Errorf("Error parsing public property: %d, error: %v", isPublic, err) + p.CustomAbort(http.StatusBadRequest, "invalid project Id") + } + } + isAdmin := false + if public == 1 { + projectList, err = dao.GetPublicProjects(projectName) + } else { + isAdmin, err = dao.IsAdminRole(p.userID) + if err != nil { + log.Errorf("Error occured in check admin, error: %v", err) + p.CustomAbort(http.StatusInternalServerError, "Internal error.") + } + if isAdmin { + projectList, err = dao.GetAllProjects(projectName) + } else { + projectList, err = dao.GetUserRelevantProjects(p.userID, projectName) + } } - public, _ := p.GetInt("is_public") - queryProject.Public = public - - projectList, err := dao.QueryProject(queryProject) if err != nil { - log.Errorf("Error occurred in QueryProject, error: %v", err) + log.Errorf("Error occured in get projects info, error: %v", err) p.CustomAbort(http.StatusInternalServerError, "Internal error.") } for i := 0; i < len(projectList); i++ { - if isProjectAdmin(p.userID, projectList[i].ProjectID) { - projectList[i].Togglable = true + if public != 1 { + if isAdmin { + projectList[i].Role = models.PROJECTADMIN + } + if projectList[i].Role == models.PROJECTADMIN { + projectList[i].Togglable = true + } } + projectList[i].RepoCount = getRepoCountByProject(projectList[i].Name) } p.Data["json"] = projectList p.ServeJSON() diff --git a/api/search.go b/api/search.go index d47bb5890..3acdc7ca8 100644 --- a/api/search.go +++ b/api/search.go @@ -55,13 +55,13 @@ func (s *SearchAPI) Get() { var projects []models.Project if isSysAdmin { - projects, err = dao.GetAllProjects() + projects, err = dao.GetAllProjects("") if err != nil { log.Errorf("failed to get all projects: %v", err) s.CustomAbort(http.StatusInternalServerError, "internal error") } } else { - projects, err = dao.GetUserRelevantProjects(userID) + projects, err = dao.SearchProjects(userID) if err != nil { log.Errorf("failed to get user %d 's relevant projects: %v", userID, err) s.CustomAbort(http.StatusInternalServerError, "internal error") diff --git a/api/statistic.go b/api/statistic.go new file mode 100644 index 000000000..67a7cc388 --- /dev/null +++ b/api/statistic.go @@ -0,0 +1,117 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package api + +import ( + "net/http" + "strings" + + "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/models" + svc_utils "github.com/vmware/harbor/service/utils" + "github.com/vmware/harbor/utils/log" +) + +// StatisticAPI handles request to /api/statistics/ +type StatisticAPI struct { + BaseAPI + userID int +} + +//Prepare validates the URL and the user +func (s *StatisticAPI) Prepare() { + s.userID = s.ValidateUser() +} + +// Get total projects and repos of the user +func (s *StatisticAPI) Get() { + isAdmin, err := dao.IsAdminRole(s.userID) + if err != nil { + log.Errorf("Error occured in check admin, error: %v", err) + s.CustomAbort(http.StatusInternalServerError, "Internal error.") + } + var projectList []models.Project + if isAdmin { + projectList, err = dao.GetAllProjects("") + } else { + projectList, err = dao.GetUserRelevantProjects(s.userID, "") + } + if err != nil { + log.Errorf("Error occured in QueryProject, error: %v", err) + s.CustomAbort(http.StatusInternalServerError, "Internal error.") + } + proMap := map[string]int{} + proMap["my_project_count"] = 0 + proMap["my_repo_count"] = 0 + proMap["public_project_count"] = 0 + proMap["public_repo_count"] = 0 + var publicProjects []models.Project + publicProjects, err = dao.GetPublicProjects("") + if err != nil { + log.Errorf("Error occured in QueryPublicProject, error: %v", err) + s.CustomAbort(http.StatusInternalServerError, "Internal error.") + } + proMap["public_project_count"] = len(publicProjects) + for i := 0; i < len(publicProjects); i++ { + proMap["public_repo_count"] += getRepoCountByProject(publicProjects[i].Name) + } + if isAdmin { + proMap["total_project_count"] = len(projectList) + proMap["total_repo_count"] = getTotalRepoCount() + } + for i := 0; i < len(projectList); i++ { + if isAdmin { + projectList[i].Role = models.PROJECTADMIN + } + if projectList[i].Role == models.PROJECTADMIN || projectList[i].Role == models.DEVELOPER || + projectList[i].Role == models.GUEST { + proMap["my_project_count"]++ + proMap["my_repo_count"] += getRepoCountByProject(projectList[i].Name) + } + } + s.Data["json"] = proMap + s.ServeJSON() +} + +//getReposByProject returns repo numbers of specified project +func getRepoCountByProject(projectName string) int { + repoList, err := svc_utils.GetRepoFromCache() + if err != nil { + log.Errorf("Failed to get repo from cache, error: %v", err) + return 0 + } + var resp int + if len(projectName) > 0 { + for _, r := range repoList { + if strings.Contains(r, "/") && r[0:strings.LastIndex(r, "/")] == projectName { + resp++ + } + } + return resp + } + return 0 +} + +//getTotalRepoCount returns total repo count +func getTotalRepoCount() int { + repoList, err := svc_utils.GetRepoFromCache() + if err != nil { + log.Errorf("Failed to get repo from cache, error: %v", err) + return 0 + } + return len(repoList) + +} diff --git a/contrib/Configure_mirror.md b/contrib/Configure_mirror.md new file mode 100644 index 000000000..00badf130 --- /dev/null +++ b/contrib/Configure_mirror.md @@ -0,0 +1,30 @@ +# Configuring Harbor as a local registry mirror + +Harbor runs as a local registry by default. It can also be configured as a registry mirror, +which caches downloaded images for subsequent use. Note that under this setup, the Harbor registry only acts as a mirror server and +no longer accepts image pushing requests. Edit `Deploy/templates/registry/config.yml` before executing `./prepare`, and append a `proxy` section as follows: + +``` +proxy: + remoteurl: https://registry-1.docker.io +``` +In order to access private images on the Docker Hub, a username and a password can be supplied: + +``` +proxy: + remoteurl: https://registry-1.docker.io + username: [username] + password: [password] +``` +You will need to pass the `--registry-mirror` option to your Docker daemon on startup: + +``` +docker --registry-mirror=https:// daemon +``` +For example, if your mirror is serving on `http://reg.yourdomain.com`, you would run: + +``` +docker --registry-mirror=https://reg.yourdomain.com daemon +``` + +Refer to the [Registry as a pull through cache](https://github.com/docker/distribution/blob/master/docs/mirror.md) for detailed information. diff --git a/contrib/README.md b/contrib/README.md index b11059165..f9ff01b75 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -1 +1 @@ -The `contrib` directory contains documents, scripts, and other helpful things which are contributed by community. +The `contrib` directory contains documents, scripts, and other helpful things which are contributed by the community. diff --git a/dao/accesslog.go b/dao/accesslog.go index 908fb9280..dc18e9d66 100644 --- a/dao/accesslog.go +++ b/dao/accesslog.go @@ -20,13 +20,11 @@ import ( "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" - - "github.com/astaxie/beego/orm" ) // AddAccessLog persists the access logs func AddAccessLog(accessLog models.AccessLog) error { - o := orm.NewOrm() + o := GetOrmer() p, err := o.Raw(`insert into access_log (user_id, project_id, repo_name, repo_tag, guid, operation, op_time) values (?, ?, ?, ?, ?, ?, now())`).Prepare() @@ -43,7 +41,7 @@ func AddAccessLog(accessLog models.AccessLog) error { //GetAccessLogs gets access logs according to different conditions func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { - o := orm.NewOrm() + o := GetOrmer() sql := `select a.log_id, u.username, a.repo_name, a.repo_tag, a.operation, a.op_time from access_log a left join user u on a.user_id = u.user_id where a.project_id = ? ` @@ -106,7 +104,7 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) { // AccessLog ... func AccessLog(username, projectName, repoName, repoTag, action string) error { - o := orm.NewOrm() + o := GetOrmer() sql := "insert into access_log (user_id, project_id, repo_name, repo_tag, operation, op_time) " + "select (select user_id as user_id from user where username=?), " + "(select project_id as project_id from project where name=?), ?, ?, ?, now() " diff --git a/dao/base.go b/dao/base.go index 57530ba0d..a03db5cb3 100644 --- a/dao/base.go +++ b/dao/base.go @@ -22,6 +22,7 @@ import ( "os" "strings" + "sync" "time" "github.com/astaxie/beego/orm" @@ -98,3 +99,14 @@ func InitDB() { panic(err) } } + +var globalOrm orm.Ormer +var once sync.Once + +// GetOrmer :set ormer singleton +func GetOrmer() orm.Ormer { + once.Do(func() { + globalOrm = orm.NewOrm() + }) + return globalOrm +} diff --git a/dao/dao_test.go b/dao/dao_test.go index 1019593c6..17dceeae0 100644 --- a/dao/dao_test.go +++ b/dao/dao_test.go @@ -364,7 +364,7 @@ func TestChangeUserPasswordWithIncorrectOldPassword(t *testing.T) { } func TestQueryRelevantProjectsWhenNoProjectAdded(t *testing.T) { - projects, err := GetUserRelevantProjects(currentUser.UserID) + projects, err := SearchProjects(currentUser.UserID) if err != nil { t.Errorf("Error occurred in QueryRelevantProjects: %v", err) } @@ -583,39 +583,6 @@ func TestIsProjectPublic(t *testing.T) { } } -func TestQueryProject(t *testing.T) { - query1 := models.Project{ - UserID: 1, - } - projects, err := QueryProject(query1) - if err != nil { - t.Errorf("Error in Query Project: %v, query: %+v", err, query1) - } - if len(projects) != 2 { - t.Errorf("Expecting get 2 projects, but actual: %d, the list: %+v", len(projects), projects) - } - query2 := models.Project{ - Public: 1, - } - projects, err = QueryProject(query2) - if err != nil { - t.Errorf("Error in Query Project: %v, query: %+v", err, query2) - } - if len(projects) != 1 { - t.Errorf("Expecting get 1 project, but actual: %d, the list: %+v", len(projects), projects) - } - query3 := models.Project{ - UserID: 9, - } - projects, err = QueryProject(query3) - if err != nil { - t.Errorf("Error in Query Project: %v, query: %+v", err, query3) - } - if len(projects) != 0 { - t.Errorf("Expecting get 0 project, but actual: %d, the list: %+v", len(projects), projects) - } -} - func TestGetUserProjectRoles(t *testing.T) { r, err := GetUserProjectRoles(currentUser.UserID, currentProject.ProjectID) if err != nil { @@ -643,20 +610,20 @@ func TestProjectPermission(t *testing.T) { } func TestGetUserRelevantProjects(t *testing.T) { - projects, err := GetUserRelevantProjects(currentUser.UserID) + projects, err := GetUserRelevantProjects(currentUser.UserID, "") if err != nil { t.Errorf("Error occurred in GetUserRelevantProjects: %v", err) } - if len(projects) != 2 { - t.Errorf("Expected length of relevant projects is 2, but actual: %d, the projects: %+v", len(projects), projects) + if len(projects) != 1 { + t.Errorf("Expected length of relevant projects is 1, but actual: %d, the projects: %+v", len(projects), projects) } - if projects[1].Name != projectName { + if projects[0].Name != projectName { t.Errorf("Expected project name in the list: %s, actual: %s", projectName, projects[1].Name) } } func TestGetAllProjects(t *testing.T) { - projects, err := GetAllProjects() + projects, err := GetAllProjects("") if err != nil { t.Errorf("Error occurred in GetAllProjects: %v", err) } @@ -668,6 +635,19 @@ func TestGetAllProjects(t *testing.T) { } } +func TestGetPublicProjects(t *testing.T) { + projects, err := GetPublicProjects("") + if err != nil { + t.Errorf("Error occurred in getProjects: %v", err) + } + if len(projects) != 1 { + t.Errorf("Expected length of projects is 1, but actual: %d, the projects: %+v", len(projects), projects) + } + if projects[0].Name != "library" { + t.Errorf("Expected project name in the list: %s, actual: %s", "library", projects[0].Name) + } +} + func TestAddProjectMember(t *testing.T) { err := AddProjectMember(currentProject.ProjectID, 1, models.DEVELOPER) if err != nil { @@ -1071,3 +1051,10 @@ func TestDeleteRepJob(t *testing.T) { t.Errorf("Able to find rep job after deletion, id: %d", jobID) } } + +func TestGetOrmer(t *testing.T) { + o := GetOrmer() + if o == nil { + t.Errorf("Error get ormer.") + } +} diff --git a/dao/project.go b/dao/project.go index 6e01b070d..e444f8a26 100644 --- a/dao/project.go +++ b/dao/project.go @@ -22,7 +22,6 @@ import ( "fmt" "time" - "github.com/astaxie/beego/orm" "github.com/vmware/harbor/utils/log" ) @@ -38,7 +37,7 @@ func AddProject(project models.Project) (int64, error) { return 0, errors.New("project name contains illegal characters") } - o := orm.NewOrm() + o := GetOrmer() p, err := o.Raw("insert into project (owner_id, name, creation_time, update_time, deleted, public) values (?, ?, ?, ?, ?, ?)").Prepare() if err != nil { @@ -79,45 +78,9 @@ func IsProjectPublic(projectName string) bool { return project.Public == 1 } -// QueryProject querys the projects based on publicity and user, disregarding the names etc. -func QueryProject(query models.Project) ([]models.Project, error) { - o := orm.NewOrm() - - sql := `select distinct - p.project_id, p.owner_id, p.name,p.creation_time, p.update_time, p.public - from project p - left join project_member pm on p.project_id = pm.project_id - where p.deleted = 0 ` - - queryParam := make([]interface{}, 1) - - if query.Public == 1 { - sql += ` and p.public = ?` - queryParam = append(queryParam, query.Public) - } else if isAdmin, _ := IsAdminRole(query.UserID); isAdmin == false { - sql += ` and (pm.user_id = ?) ` - queryParam = append(queryParam, query.UserID) - } - - if query.Name != "" { - sql += " and p.name like ? " - queryParam = append(queryParam, query.Name) - } - - sql += " order by p.name " - - var r []models.Project - _, err := o.Raw(sql, queryParam).QueryRows(&r) - - if err != nil { - return nil, err - } - return r, nil -} - //ProjectExists returns whether the project exists according to its name of ID. func ProjectExists(nameOrID interface{}) (bool, error) { - o := orm.NewOrm() + o := GetOrmer() type dummy struct{} sql := `select project_id from project where deleted = 0 and ` switch nameOrID.(type) { @@ -140,7 +103,7 @@ func ProjectExists(nameOrID interface{}) (bool, error) { // GetProjectByID ... func GetProjectByID(id int64) (*models.Project, error) { - o := orm.NewOrm() + o := GetOrmer() sql := `select p.project_id, p.name, u.username as owner_name, p.owner_id, p.creation_time, p.update_time, p.public from project p left join user u on p.owner_id = u.user_id where p.deleted = 0 and p.project_id = ?` @@ -163,7 +126,7 @@ func GetProjectByID(id int64) (*models.Project, error) { // GetProjectByName ... func GetProjectByName(name string) (*models.Project, error) { - o := orm.NewOrm() + o := GetOrmer() var p []models.Project n, err := o.Raw(`select * from project where name = ? and deleted = 0`, name).QueryRows(&p) if err != nil { @@ -179,7 +142,7 @@ func GetProjectByName(name string) (*models.Project, error) { // GetPermission gets roles that the user has according to the project. func GetPermission(username, projectName string) (string, error) { - o := orm.NewOrm() + o := GetOrmer() sql := `select r.role_code from role as r inner join project_member as pm on r.role_id = pm.role @@ -202,18 +165,18 @@ func GetPermission(username, projectName string) (string, error) { // ToggleProjectPublicity toggles the publicity of the project. func ToggleProjectPublicity(projectID int64, publicity int) error { - o := orm.NewOrm() + o := GetOrmer() sql := "update project set public = ? where project_id = ?" _, err := o.Raw(sql, publicity, projectID).Exec() return err } -// GetUserRelevantProjects returns a project list, +// SearchProjects returns a project list, // which satisfies the following conditions: // 1. the project is not deleted // 2. the prject is public or the user is a member of the project -func GetUserRelevantProjects(userID int) ([]models.Project, error) { - o := orm.NewOrm() +func SearchProjects(userID int) ([]models.Project, error) { + o := GetOrmer() sql := `select distinct p.project_id, p.name, p.public from project p left join project_member pm on p.project_id = pm.project_id @@ -228,14 +191,66 @@ func GetUserRelevantProjects(userID int) ([]models.Project, error) { return projects, nil } -// GetAllProjects returns all projects which are not deleted -func GetAllProjects() ([]models.Project, error) { - o := orm.NewOrm() - sql := `select project_id, name, public +// GetUserRelevantProjects returns the projects of the user which are not deleted and name like projectName +func GetUserRelevantProjects(userID int, projectName string) ([]models.Project, error) { + o := GetOrmer() + sql := `select distinct + p.project_id, p.owner_id, p.name,p.creation_time, p.update_time, p.public, pm.role role + from project p + left join project_member pm on p.project_id = pm.project_id + where p.deleted = 0 and pm.user_id= ?` + + queryParam := make([]interface{}, 1) + queryParam = append(queryParam, userID) + if projectName != "" { + sql += " and p.name like ? " + queryParam = append(queryParam, projectName) + } + sql += " order by p.name " + var r []models.Project + _, err := o.Raw(sql, queryParam).QueryRows(&r) + if err != nil { + return nil, err + } + return r, nil +} + +//GetPublicProjects returns all public projects whose name like projectName +func GetPublicProjects(projectName string) ([]models.Project, error) { + publicProjects, err := getProjects(1, projectName) + if err != nil { + return nil, err + } + return publicProjects, nil +} + +// GetAllProjects returns all projects which are not deleted and name like projectName +func GetAllProjects(projectName string) ([]models.Project, error) { + allProjects, err := getProjects(0, projectName) + if err != nil { + return nil, err + } + return allProjects, nil +} + +func getProjects(public int, projectName string) ([]models.Project, error) { + o := GetOrmer() + sql := `select project_id, owner_id, creation_time, update_time, name, public from project where deleted = 0` + queryParam := make([]interface{}, 1) + if public == 1 { + sql += " and public = ? " + queryParam = append(queryParam, public) + } + if len(projectName) > 0 { + sql += " and name like ? " + queryParam = append(queryParam, projectName) + } + sql += " order by name " var projects []models.Project - if _, err := o.Raw(sql).QueryRows(&projects); err != nil { + log.Debugf("sql xxx", sql) + if _, err := o.Raw(sql, queryParam).QueryRows(&projects); err != nil { return nil, err } return projects, nil diff --git a/dao/projectmember.go b/dao/projectmember.go index d994afb43..52947f869 100644 --- a/dao/projectmember.go +++ b/dao/projectmember.go @@ -16,13 +16,12 @@ package dao import ( - "github.com/astaxie/beego/orm" "github.com/vmware/harbor/models" ) // AddProjectMember inserts a record to table project_member func AddProjectMember(projectID int64, userID int, role int) error { - o := orm.NewOrm() + o := GetOrmer() sql := "insert into project_member (project_id, user_id , role) values (?, ?, ?)" @@ -33,7 +32,7 @@ func AddProjectMember(projectID int64, userID int, role int) error { // UpdateProjectMember updates the record in table project_member func UpdateProjectMember(projectID int64, userID int, role int) error { - o := orm.NewOrm() + o := GetOrmer() sql := "update project_member set role = ? where project_id = ? and user_id = ?" @@ -44,7 +43,7 @@ func UpdateProjectMember(projectID int64, userID int, role int) error { // DeleteProjectMember delete the record from table project_member func DeleteProjectMember(projectID int64, userID int) error { - o := orm.NewOrm() + o := GetOrmer() sql := "delete from project_member where project_id = ? and user_id = ?" @@ -57,7 +56,7 @@ func DeleteProjectMember(projectID int64, userID int) error { // GetUserByProject gets all members of the project. func GetUserByProject(projectID int64, queryUser models.User) ([]models.User, error) { - o := orm.NewOrm() + o := GetOrmer() u := []models.User{} sql := `select u.user_id, u.username, r.name rolename, r.role_id from user u diff --git a/dao/register.go b/dao/register.go index 6f5351504..3dc388c16 100644 --- a/dao/register.go +++ b/dao/register.go @@ -22,8 +22,6 @@ import ( "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils" - - "github.com/astaxie/beego/orm" ) // Register is used for user to register, the password is encrypted before the record is inserted into database. @@ -34,7 +32,7 @@ func Register(user models.User) (int64, error) { return 0, err } - o := orm.NewOrm() + o := GetOrmer() p, err := o.Raw("insert into user (username, password, realname, email, comment, salt, sysadmin_flag, creation_time, update_time) values (?, ?, ?, ?, ?, ?, ?, ?, ?)").Prepare() if err != nil { @@ -108,7 +106,7 @@ func UserExists(user models.User, target string) (bool, error) { return false, errors.New("User name and email are blank.") } - o := orm.NewOrm() + o := GetOrmer() sql := `select user_id from user where 1=1 ` queryParam := make([]interface{}, 1) diff --git a/dao/role.go b/dao/role.go index 2adf93aaa..00281f40d 100644 --- a/dao/role.go +++ b/dao/role.go @@ -18,14 +18,13 @@ package dao import ( "fmt" - "github.com/astaxie/beego/orm" "github.com/vmware/harbor/models" ) // GetUserProjectRoles returns roles that the user has according to the project. func GetUserProjectRoles(userID int, projectID int64) ([]models.Role, error) { - o := orm.NewOrm() + o := GetOrmer() sql := `select * from role @@ -76,7 +75,7 @@ func IsAdminRole(userIDOrUsername interface{}) (bool, error) { // GetRoleByID ... func GetRoleByID(id int) (*models.Role, error) { - o := orm.NewOrm() + o := GetOrmer() sql := `select * from role diff --git a/dao/user.go b/dao/user.go index 3d94d2a76..d337a55e2 100644 --- a/dao/user.go +++ b/dao/user.go @@ -22,14 +22,13 @@ import ( "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils" - "github.com/astaxie/beego/orm" "github.com/vmware/harbor/utils/log" ) // GetUser ... func GetUser(query models.User) (*models.User, error) { - o := orm.NewOrm() + o := GetOrmer() sql := `select user_id, username, email, realname, comment, reset_uuid, salt, sysadmin_flag, creation_time, update_time @@ -66,7 +65,7 @@ func GetUser(query models.User) (*models.User, error) { // LoginByDb is used for user to login with database auth mode. func LoginByDb(auth models.AuthModel) (*models.User, error) { - o := orm.NewOrm() + o := GetOrmer() var users []models.User n, err := o.Raw(`select * from user where (username = ? or email = ?) and deleted = 0`, @@ -91,7 +90,7 @@ func LoginByDb(auth models.AuthModel) (*models.User, error) { // ListUsers lists all users according to different conditions. func ListUsers(query models.User) ([]models.User, error) { - o := orm.NewOrm() + o := GetOrmer() u := []models.User{} sql := `select user_id, username, email, realname, comment, reset_uuid, salt, sysadmin_flag, creation_time, update_time @@ -111,7 +110,7 @@ func ListUsers(query models.User) ([]models.User, error) { // ToggleUserAdminRole gives a user admin role. func ToggleUserAdminRole(u models.User) error { - o := orm.NewOrm() + o := GetOrmer() sql := `update user set sysadmin_flag =not sysadmin_flag where user_id = ?` @@ -133,7 +132,7 @@ func ChangeUserPassword(u models.User, oldPassword ...string) (err error) { return errors.New("Wrong numbers of params.") } - o := orm.NewOrm() + o := GetOrmer() var r sql.Result if len(oldPassword) == 0 { @@ -159,7 +158,7 @@ func ChangeUserPassword(u models.User, oldPassword ...string) (err error) { // ResetUserPassword ... func ResetUserPassword(u models.User) error { - o := orm.NewOrm() + o := GetOrmer() r, err := o.Raw(`update user set password=?, reset_uuid=? where reset_uuid=?`, utils.Encrypt(u.Password, u.Salt), "", u.ResetUUID).Exec() if err != nil { return err @@ -176,7 +175,7 @@ func ResetUserPassword(u models.User) error { // UpdateUserResetUUID ... func UpdateUserResetUUID(u models.User) error { - o := orm.NewOrm() + o := GetOrmer() _, err := o.Raw(`update user set reset_uuid=? where email=?`, u.ResetUUID, u.Email).Exec() return err } @@ -207,7 +206,7 @@ func CheckUserPassword(query models.User) (*models.User, error) { queryParam = append(queryParam, currentUser.Username) queryParam = append(queryParam, utils.Encrypt(query.Password, currentUser.Salt)) } - o := orm.NewOrm() + o := GetOrmer() var user []models.User n, err := o.Raw(sql, queryParam).QueryRows(&user) @@ -226,7 +225,7 @@ func CheckUserPassword(query models.User) (*models.User, error) { // DeleteUser ... func DeleteUser(userID int) error { - o := orm.NewOrm() + o := GetOrmer() _, err := o.Raw(`update user set deleted = 1 where user_id = ?`, userID).Exec() return err } diff --git a/docs/img/caicloudLogoWeb.png b/docs/img/caicloudLogoWeb.png new file mode 100644 index 000000000..1de2c884e Binary files /dev/null and b/docs/img/caicloudLogoWeb.png differ diff --git a/docs/installation_guide.md b/docs/installation_guide.md index e03c7f5c8..3edd8bbd9 100644 --- a/docs/installation_guide.md +++ b/docs/installation_guide.md @@ -4,7 +4,9 @@ Harbor can be installed in one of two ways: 1. From source code - This goes through a full build process, _and requires an Internet connection_. 2. Pre-built installation package - This can save time (no building necessary!) as well as allows for installation on a host that is _not_ connected to the Internet. -This guide describes both of these approaches +This guide describes both of these approaches. + +In addition, the deployment instructions on Kubernetes has been created by the community. Refer to [Deploy Harbor on Kubernetes](kubernetes_deployment.md) for details. ## Prerequisites for the target host Harbor is deployed as several Docker containers, and, therefore, can be deployed on any Linux distribution that supports Docker. @@ -35,7 +37,7 @@ The parameters are described below - note that at the very least, you will need * **hostname**: The target host's hostname, which is used to access the UI and the registry service. It should be the IP address or the fully qualified domain name (FQDN) of your target machine, e.g., `192.168.1.10` or `reg.yourdomain.com`. _Do NOT use `localhost` or `127.0.0.1` for the hostname - the registry service needs to be accessible by external clients!_ * **ui_url_protocol**: (**http** or **https**. Default is **http**) The protocol used to access the UI and the token/notification service. By default, this is _http_. To set up the https protocol, refer to [Configuring Harbor with HTTPS Access](configure_https.md). -* **Email settings**: These parameters are needed for Harbor to be able to send a user a "password reset" email, and are only necessary if that functionality is needed. Also, do mnote that by default SSL connectivity is _not_ enabled - if your SMTP server requires SSL, but does _not_ support STARTTLS, then you should enable SSL by setting **email_ssl = true**. +* **Email settings**: These parameters are needed for Harbor to be able to send a user a "password reset" email, and are only necessary if that functionality is needed. Also, do note that by default SSL connectivity is _not_ enabled - if your SMTP server requires SSL, but does _not_ support STARTTLS, then you should enable SSL by setting **email_ssl = true**. * email_server = smtp.mydomain.com * email_server_port = 25 * email_username = sample_admin@mydomain.com @@ -47,11 +49,33 @@ The parameters are described below - note that at the very least, you will need * **auth_mode**: The type of authentication that is used. By default it is **db_auth**, i.e. the credentials are stored in a database. For LDAP authentication, set this to **ldap_auth**. * **ldap_url**: The LDAP endpoint URL (e.g. `ldaps://ldap.mydomain.com`). _Only used when **auth_mode** is set to *ldap_auth* ._ * **ldap_basedn**: The basedn template for verifying the user's credentials against LDAP (e.g. `uid=%s,ou=people,dc=mydomain,dc=com`). _Only used when **auth_mode** is set to *ldap_auth* ._ -* **db_password**: The root password for the mySQL database used for **db_auth**. _Change this password for any production use!!_ -* **self_registration**: (**on** or **off**. Default is **on**) Enable / Disable the ability for a user to register themselves. When disabled, new users can only be created by the Admin user, only an admin user can create new users in Harbor. _NOTE: When **auth_mode** is set to **ldap_auth**, self-registration feature is **always** disabled, and this flag is ignored. +* **db_password**: The root password for the mySQL database used for **db_auth**. _Change this password for any production use!_ +* **self_registration**: (**on** or **off**. Default is **on**) Enable / Disable the ability for a user to register themselves. When disabled, new users can only be created by the Admin user, only an admin user can create new users in Harbor. _NOTE: When **auth_mode** is set to **ldap_auth**, self-registration feature is **always** disabled, and this flag is ignored._ + +#### Configuring storage backend (optional) + +By default, Harbor stores images on your local filesystem. In a production environment, you may consider +using other storage backend instead of the local filesystem, like S3, Openstack Swift, Ceph, etc. +What you need to update is the section of `storage` in the file `Deploy/templates/registry/config.yml`. +For example, if you use Openstack Swift as your storage backend, the section may look like this: + +``` +storage: + swift: + username: admin + password: ADMIN_PASS + authurl: http://keystone_addr:35357/v3 + tenant: admin + domain: default + region: regionOne + container: docker_images +``` + +_NOTE: For detailed information on storage backend of a registry, refer to [Registry Configuration Reference](https://docs.docker.com/registry/configuration/) ._ + #### Building and starting Harbor -Once **harbord.cfg** is configured, build and start Harbor as follows. Note that Note that the docker-compose process can take a while! +Once **harbord.cfg** and storage backend (optional) are configured, build and start Harbor as follows. Note that the docker-compose process can take a while. ```sh $ cd Deploy @@ -77,52 +101,9 @@ $ docker push reg.yourdomain.com/myproject/myrepo For information on how to use Harbor, please refer to [User Guide of Harbor](user_guide.md) . -#### Configuring Harbor with HTTPS Access -Harbor does not ship with any certificates, and, by default, uses HTTP to serve requests. While this makes it relatively simple to set-up and run - especially for a development or testing environment - it is **not** recommended for a production environment. To enable HTTPS, please refer to [Configuring Harbor with HTTPS Access](configure_https.md) +#### Configuring Harbor with HTTPS access +Harbor does not ship with any certificates, and, by default, uses HTTP to serve requests. While this makes it relatively simple to set up and run - especially for a development or testing environment - it is **not** recommended for a production environment. To enable HTTPS, please refer to [Configuring Harbor with HTTPS Access](configure_https.md). -#### Configuring Harbor as a local registry mirror -The Harbor runs as a local private registry by default, it can be easily configured to run as a local registry mirror, which can keep most of the redundant image fetch traffic on your local network. You just need to edit `config/registry/config.yml` after execute `./prepare`, and append a `proxy` section as follows: - -``` -proxy: - remoteurl: https://registry-1.docker.io -``` -In order to access private images on the Docker Hub, a username and password can be supplied: - -``` -proxy: - remoteurl: https://registry-1.docker.io - username: [username] - password: [password] -``` -You will need to pass the `--registry-mirror` option to your Docker daemon on startup: - -``` -docker --registry-mirror=https:// daemon -``` -For example, if your mirror is serving on `http:/reg.yourdomain.com`, you would run: - -``` -docker --registry-mirror=https://reg.yourdomain.com daemon -``` - -Refer to the [Registry as a pull through cache](https://github.com/docker/distribution/blob/master/docs/mirror.md) for detail information. - -#### Configuring storage backend - -By default, the Harbor store images on your local filesystem. In production environment, you may consider using higher available storage backend instead of the local filesystem, like S3, Openstack Swift, Ceph, etc. Fortunately, the Registry supports multiple storage backend, refer to the [Registry Configuration Reference](https://docs.docker.com/registry/configuration/) for detail information. All you need to do is update the section of `storage`, and fill in the fields according to your specied backend. For example, if you use Openstack Swift as your storage backend, the file may look like this: - -``` -storage: - swift: - username: admin - password: ADMIN_PASS - authurl: http://keystone_addr:35357/v3 - tenant: admin - domain: default - region: regionOne - container: docker_images -``` ## Installation from a pre-built package @@ -135,7 +116,7 @@ $ cd harbor Next, configure Harbor as described earlier in [Configuring Harbor](#configuring-harbor). -Finally, run the **prepare** script to generate config files, and use docker compose to build / start Harbor. +Finally, run the **prepare** script to generate config files, and use docker compose to build and start Harbor. ``` @@ -151,23 +132,25 @@ $ sudo docker-compose up -d ``` ### Deploying Harbor on a host which does not have Internet access -*docker-compose up* pulls the base images from Docker Hub and builds new images for the containers, which, necessarily, requires internet access. To deploy Harbor on a host that is not connected to the Internet +*docker-compose up* pulls the base images from Docker Hub and builds new images for the containers, which, necessarily, requires Internet access. To deploy Harbor on a host that is not connected to the Internet: + 1. Prepare Harbor on a machine that has access to the Internet. 2. Export the images as tgz files 3. Transfer them to the target host. 4. Load the tgz file into Docker's local image repo on the host. -THese steps are detailed below +These steps are detailed below: #### Building and saving images for offline installation -On a machine that is connected to the Internet, +On a machine that is connected to the Internet, + 1. Extract the files from the pre-built installation package. 2. Then, run `docker-compose build` to build the images. 3. Use the script `save_image.sh` to export these images as tar files. Note that the tar files will be stored in the `images/` directory. 4. Package everything in the directory `harbor/` into a tgz file 5. Transfer this tgz file to the target machine. -The commands, in detail, are as follows +The commands, in detail, are as follows: ``` $ cd harbor @@ -188,8 +171,8 @@ $ cd ../ $ tar -cvzf harbor_offline-0.1.1.tgz harbor ``` -The file `harbor_offline-0.1.1.tgz` contains the images and other files required to start Harbor. You can use tools such as `rsync` or `scp` to transfer the this file to the target host. -On the target host, execute the following commands to start Harbor. _Note that before running the **prepare** script, you **must** update **harbor.cfg** to reflect the right configuration of the target machine!!_ (Refer to Section [Configuring Harbor](#configuring-harbor) +The file `harbor_offline-0.1.1.tgz` contains the images and other files required to start Harbor. You can use tools such as `rsync` or `scp` to transfer this file to the target host. +On the target host, execute the following commands to start Harbor. _Note that before running the **prepare** script, you **must** update **harbor.cfg** to reflect the right configuration of the target machine!_ (Refer to Section [Configuring Harbor](#configuring-harbor)). ``` $ tar -xzvf harbor_offline-0.1.1.tgz @@ -219,7 +202,7 @@ $ sudo docker-compose up -d ``` ### Managing Harbor's lifecycle -You can use docker-compose to manage the container lifecycle of the containers. A few useful commands are listed below: +You can use docker-compose to manage the lifecycle of the containers. A few useful commands are listed below: *Build and start Harbor:* ``` @@ -239,7 +222,7 @@ Stopping harbor_registry_1 ... done Stopping harbor_mysql_1 ... done Stopping harbor_log_1 ... done ``` -*Restart Harbor after stopping* +*Restart Harbor after stopping:* ``` $ sudo docker-compose start Starting harbor_log_1 @@ -247,8 +230,8 @@ Starting harbor_mysql_1 Starting harbor_registry_1 Starting harbor_ui_1 Starting harbor_proxy_1 -```` -*Remove Harbor's containers while keeping the image data and Harbor's database files on the file system: * +``` +*Remove Harbor's containers while keeping the image data and Harbor's database files on the file system:* ``` $ sudo docker-compose rm Going to remove harbor_proxy_1, harbor_ui_1, harbor_registry_1, harbor_mysql_1, harbor_log_1 @@ -265,14 +248,14 @@ $ rm -r /data/database $ rm -r /data/registry ``` -Please check the [Docker Compose command-line reference](https://docs.docker.com/compose/reference/) for more on docker-compose +Please check the [Docker Compose command-line reference](https://docs.docker.com/compose/reference/) for more on docker-compose. ### Persistent data and log files -By default, registry data is persisted in the target host's `/data/` of directory. This data remains unchanged even when Harbor's containers are removed and/or recreated. -In addition, Harbor users `rsyslog` to collect the logs of each container. By default, these log files are stored in the directory `/var/log/harbor/` on the target host. +By default, registry data is persisted in the target host's `/data/` directory. This data remains unchanged even when Harbor's containers are removed and/or recreated. +In addition, Harbor uses `rsyslog` to collect the logs of each container. By default, these log files are stored in the directory `/var/log/harbor/` on the target host. ##Troubleshooting -1.When setting up Harbor behind an nginx proxy or elastic load balancing, look for the line below, in `Deploy/config/nginx/nginx.conf` and remove it from the sections: `location /`, `location /v2/` and `location /service/`. +1.When setting up Harbor behind an nginx proxy or elastic load balancing, look for the line below, in `Deploy/config/nginx/nginx.conf` and remove it from the sections if the proxy already has similar settings: `location /`, `location /v2/` and `location /service/`. ``` proxy_set_header X-Forwarded-Proto $scheme; ``` diff --git a/docs/kubernetes_deployment.md b/docs/kubernetes_deployment.md index 109c2768e..0184e539f 100644 --- a/docs/kubernetes_deployment.md +++ b/docs/kubernetes_deployment.md @@ -1,6 +1,6 @@ -## Deploy harbor on kubernetes. -For now, it's a little tricky to start harbor on kubernetes because - 1. Registry uses https, so we need cert or workaround to avoid errors like this: +## Deploying Harbor on Kubernetes +To deploy Harbor on Kubernetes, it requires some additional steps because + 1. When Harbor registry uses https, so we need cert or workaround to avoid errors like this: ``` Error response from daemon: invalid registry endpoint https://{HOST}/v0/: unable to ping registry endpoint https://{HOST}/v0/ v2 ping attempt failed with error: Get https://{HOST}/v2/: EOF @@ -19,12 +19,12 @@ For now, it's a little tricky to start harbor on kubernetes because sudo service docker restart ``` - 2. The registry config file need to know the IP (or DNS name) of the registry, but on kubernetes, you won't know the IP before the service is created. There are several workarounds to solve this problem for now: + 2. The registry config file needs to have the IP (or DNS name) of the registry, but on Kubernetes, you don't know the IP before the service is created. There are several workarounds to solve this problem for now: - Use DNS name and link th DNS name with the IP after the service is created. - Rebuild the registry image with the service IP after the service is created and use ```kubectl rolling-update``` to update to the new image. -To start harbor on kubernetes, you first need to build the docker images. The docker images for deploying Harbor on Kubernetes depends on the docker images to deploy Harbor with docker-compose. So the first step is to build docker images with docker-compose. Before actually building the images, you need to first adjust the [configuration](https://github.com/vmware/harbor/blob/master/Deploy/harbor.cfg): +To start Harbor on Kubernetes, you first need to build the docker images. The docker images for deploying Harbor on Kubernetes depends on the docker images to deploy Harbor with docker-compose. So the first step is to build docker images with docker-compose. Before actually building the images, you need to first adjust the [configuration](https://github.com/vmware/harbor/blob/master/Deploy/harbor.cfg): - Change the [hostname](https://github.com/vmware/harbor/blob/master/Deploy/harbor.cfg#L5) to ```localhost``` - Adjust the [email settings](https://github.com/vmware/harbor/blob/master/Deploy/harbor.cfg#L11) according to your needs. diff --git a/docs/prepare-swagger.sh b/docs/prepare-swagger.sh index dee9b2e9b..742d07fdb 100755 --- a/docs/prepare-swagger.sh +++ b/docs/prepare-swagger.sh @@ -9,10 +9,10 @@ wget https://github.com/swagger-api/swagger-ui/archive/v2.1.4.tar.gz -O swagger. echo "Untarring Swagger UI package to the static file path..." tar -C ../static/vendors -zxf swagger.tar.gz swagger-ui-2.1.4/dist echo "Executing some processes..." -sed -i 's/http:\/\/petstore\.swagger\.io\/v2\/swagger\.json/'$SCHEME':\/\/'$SERVER_IP'\/static\/resources\/yaml\/swagger\.yaml/g' \ - ../static/vendors/swagger-ui-2.1.4/dist/index.html +sed -i.bak 's/http:\/\/petstore\.swagger\.io\/v2\/swagger\.json/'$SCHEME':\/\/'$SERVER_IP'\/static\/resources\/yaml\/swagger\.yaml/g' \ +../static/vendors/swagger-ui-2.1.4/dist/index.html mkdir -p ../static/resources/yaml cp swagger.yaml ../static/resources/yaml -sed -i 's/host: localhost/host: '$SERVER_IP'/g' ../static/resources/yaml/swagger.yaml -sed -i 's/ \- http$/ \- '$SCHEME'/g' ../static/resources/yaml/swagger.yaml +sed -i.bak 's/host: localhost/host: '$SERVER_IP'/g' ../static/resources/yaml/swagger.yaml +sed -i.bak 's/ \- http$/ \- '$SCHEME'/g' ../static/resources/yaml/swagger.yaml echo "Finish preparation for the Swagger UI." diff --git a/job/config/config.go b/job/config/config.go index 8fb90c360..400256720 100644 --- a/job/config/config.go +++ b/job/config/config.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "os" "strconv" @@ -11,6 +12,7 @@ const defaultMaxWorkers int = 10 var maxJobWorkers int var localRegURL string +var logDir string func init() { maxWorkersEnv := os.Getenv("MAX_JOB_WORKERS") @@ -26,8 +28,27 @@ func init() { localRegURL = "http://registry:5000/" } + logDir = os.Getenv("LOG_DIR") + if len(logDir) == 0 { + logDir = "/var/log" + } + + f, err := os.Open(logDir) + defer f.Close() + if err != nil { + panic(err) + } + finfo, err := f.Stat() + if err != nil { + panic(err) + } + if !finfo.IsDir() { + panic(fmt.Sprintf("%s is not a direcotry", logDir)) + } + log.Debugf("config: maxJobWorkers: %d", maxJobWorkers) log.Debugf("config: localRegURL: %s", localRegURL) + log.Debugf("config: logDir: %s", logDir) } func MaxJobWorkers() int { @@ -37,3 +58,7 @@ func MaxJobWorkers() int { func LocalRegURL() string { return localRegURL } + +func LogDir() string { + return logDir +} diff --git a/job/replication/statehandlers.go b/job/replication/statehandlers.go index 6d789dcc5..9a2b140eb 100644 --- a/job/replication/statehandlers.go +++ b/job/replication/statehandlers.go @@ -26,8 +26,8 @@ import ( "github.com/docker/distribution" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" - "github.com/vmware/harbor/job/utils" "github.com/vmware/harbor/models" + "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry" "github.com/vmware/harbor/utils/registry/auth" ) @@ -64,13 +64,13 @@ type BaseHandler struct { blobsExistence map[string]bool //key: digest of blob, value: existence - logger *utils.Logger + logger *log.Logger } // InitBaseHandler initializes a BaseHandler: creating clients for source and destination registry, // listing tags of the repository if parameter tags is nil. func InitBaseHandler(repository, srcURL, srcSecretKey, - dstURL, dstUsr, dstPwd string, tags []string, logger *utils.Logger) (*BaseHandler, error) { + dstURL, dstUsr, dstPwd string, tags []string, logger *log.Logger) (*BaseHandler, error) { logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s", repository, tags, srcURL, dstURL, dstUsr) diff --git a/job/statehandlers.go b/job/statehandlers.go index 1d63b6065..10c72c6b5 100644 --- a/job/statehandlers.go +++ b/job/statehandlers.go @@ -4,7 +4,6 @@ import ( "time" "github.com/vmware/harbor/dao" - "github.com/vmware/harbor/job/utils" "github.com/vmware/harbor/models" "github.com/vmware/harbor/utils/log" ) @@ -51,7 +50,7 @@ func (su StatusUpdater) Enter() (string, error) { type ImgPuller struct { DummyHandler img string - logger utils.Logger + logger *log.Logger } func (ip ImgPuller) Enter() (string, error) { @@ -64,7 +63,7 @@ func (ip ImgPuller) Enter() (string, error) { type ImgPusher struct { DummyHandler targetURL string - logger utils.Logger + logger *log.Logger } func (ip ImgPusher) Enter() (string, error) { diff --git a/job/statemachine.go b/job/statemachine.go index 8d2e72702..521108514 100644 --- a/job/statemachine.go +++ b/job/statemachine.go @@ -31,7 +31,7 @@ type JobSM struct { Transitions map[string]map[string]struct{} Handlers map[string]StateHandler desiredState string - Logger utils.Logger + Logger *log.Logger Parms *RepJobParm lock *sync.Mutex } @@ -200,7 +200,7 @@ func (sm *JobSM) Reset(jid int64) error { sm.Parms.TargetUsername = target.Username sm.Parms.TargetPassword = target.Password //init states handlers - sm.Logger = utils.Logger{sm.JobID} + sm.Logger = utils.NewLogger(sm.JobID) sm.CurrentState = models.JobPending sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning}) @@ -226,7 +226,7 @@ func (sm *JobSM) Reset(jid int64) error { func addImgOutTransition(sm *JobSM) error { base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, "", sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword, - nil, &sm.Logger) + nil, sm.Logger) if err != nil { return err } diff --git a/job/utils/logger.go b/job/utils/logger.go index 0641f916a..2905e3f50 100644 --- a/job/utils/logger.go +++ b/job/utils/logger.go @@ -3,37 +3,23 @@ package utils import ( "fmt" - "github.com/vmware/harbor/dao" + "github.com/vmware/harbor/job/config" "github.com/vmware/harbor/utils/log" + "os" + "path/filepath" ) -const ( - INFO = "info" - WARN = "warning" - ERR = "error" -) - -type Logger struct { - ID int64 -} - -func (l *Logger) Infof(format string, v ...interface{}) { - err := dao.AddJobLog(l.ID, INFO, fmt.Sprintf(format, v...)) +func NewLogger(jobID int64) *log.Logger { + logFile := GetJobLogPath(jobID) + f, err := os.OpenFile(logFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0660) if err != nil { - log.Warningf("Failed to add job log, id: %d, error: %v", l.ID, err) + log.Errorf("Failed to open log file %s, the log of job %d will be printed to standard output", logFile, jobID) + f = os.Stdout } + return log.New(f, log.NewTextFormatter(), log.InfoLevel) } -func (l *Logger) Warningf(format string, v ...interface{}) { - err := dao.AddJobLog(l.ID, WARN, fmt.Sprintf(format, v...)) - if err != nil { - log.Warningf("Failed to add job log, id: %d, error: %v", l.ID, err) - } -} - -func (l *Logger) Errorf(format string, v ...interface{}) { - err := dao.AddJobLog(l.ID, ERR, fmt.Sprintf(format, v...)) - if err != nil { - log.Warningf("Failed to add job log, id: %d, error: %v", l.ID, err) - } +func GetJobLogPath(jobID int64) string { + fn := fmt.Sprintf("job_%d.log", jobID) + return filepath.Join(config.LogDir(), fn) } diff --git a/jobservice/router.go b/jobservice/router.go index c173020b4..6a87bd995 100644 --- a/jobservice/router.go +++ b/jobservice/router.go @@ -7,6 +7,7 @@ import ( ) func initRouters() { - beego.Router("/api/jobs/replication", &api.ReplicationJob{}) - beego.Router("/api/jobs/replication/actions", &api.ReplicationJob{}, "post:HandleAction") + beego.Router("/api/replicationJobs", &api.ReplicationJob{}) + beego.Router("/api/replicationJobs/:id/log", &api.ReplicationJob{}, "get:GetLog") + beego.Router("/api/replicationJobs/actions", &api.ReplicationJob{}, "post:HandleAction") } diff --git a/jobservice/newtest.json b/jobservice/sync_policy.json similarity index 100% rename from jobservice/newtest.json rename to jobservice/sync_policy.json diff --git a/jobservice/sync_repo.json b/jobservice/sync_repo.json new file mode 100644 index 000000000..550c7ddc4 --- /dev/null +++ b/jobservice/sync_repo.json @@ -0,0 +1 @@ +{"policy_id": 1, "repository":"library/ubuntu"} diff --git a/migration/changelog.md b/migration/changelog.md new file mode 100644 index 000000000..ab14c17d5 --- /dev/null +++ b/migration/changelog.md @@ -0,0 +1,19 @@ +# What's New in Harbor Database Schema +Changelog for harbor database schema + +## 0.1.0 + +## 0.1.1 + + - create table `project_member` + - create table `schema_version` + - drop table `user_project_role` + - drop table `project_role` + - add column `creation_time` to table `user` + - add column `sysadmin_flag` to table `user` + - add column `update_time` to table `user` + - add column `role_mask` to table `role` + - add column `update_time` to table `project` + - delete data `AMDRWS` from table `role` + - delete data `A` from table `access` + diff --git a/migration/migration_harbor/versions/0_1_1.py b/migration/migration_harbor/versions/0_1_1.py index 746446f39..ecec2cfb3 100644 --- a/migration/migration_harbor/versions/0_1_1.py +++ b/migration/migration_harbor/versions/0_1_1.py @@ -83,7 +83,7 @@ def upgrade(): session.delete(role) session.query(Role).update({Role.role_id: Role.role_id - 1}) - #delete M from table access + #delete A from table access acc = session.query(Access).filter_by(access_id=1).first() session.delete(acc) session.query(Access).update({Access.access_id: Access.access_id - 1}) diff --git a/models/project.go b/models/project.go index e240c609f..2ebd5fc32 100644 --- a/models/project.go +++ b/models/project.go @@ -34,4 +34,6 @@ type Project struct { Togglable bool UpdateTime time.Time `orm:"update_time" json:"update_time"` + Role int `json:"role_id"` + RepoCount int `json:"repo_count"` } diff --git a/ui/router.go b/ui/router.go index e1f35da25..052a942a8 100644 --- a/ui/router.go +++ b/ui/router.go @@ -54,6 +54,7 @@ func initRouters() { beego.Router("/api/search", &api.SearchAPI{}) beego.Router("/api/projects/:pid/members/?:mid", &api.ProjectMemberAPI{}) beego.Router("/api/projects/?:id", &api.ProjectAPI{}) + beego.Router("/api/statistics", &api.StatisticAPI{}) beego.Router("/api/projects/:id/logs/filter", &api.ProjectAPI{}, "post:FilterAccessLog") beego.Router("/api/users", &api.UserAPI{}) beego.Router("/api/users/?:id", &api.UserAPI{}) diff --git a/utils/log/logger.go b/utils/log/logger.go index c713aa1b0..c9a6c5072 100644 --- a/utils/log/logger.go +++ b/utils/log/logger.go @@ -27,7 +27,7 @@ import ( var logger = New(os.Stdout, NewTextFormatter(), WarningLevel) func init() { - logger.callDepth = 3 + logger.callDepth = 4 // TODO add item in configuaration file lvl := os.Getenv("LOG_LEVEL") @@ -52,6 +52,7 @@ type Logger struct { fmtter Formatter lvl Level callDepth int + skipLine bool mu sync.Mutex } @@ -61,7 +62,7 @@ func New(out io.Writer, fmtter Formatter, lvl Level) *Logger { out: out, fmtter: fmtter, lvl: lvl, - callDepth: 2, + callDepth: 3, } } @@ -121,8 +122,7 @@ func (l *Logger) output(record *Record) (err error) { // Debug ... func (l *Logger) Debug(v ...interface{}) { if l.lvl <= DebugLevel { - line := line(l.callDepth) - record := NewRecord(time.Now(), fmt.Sprint(v...), line, DebugLevel) + record := NewRecord(time.Now(), fmt.Sprint(v...), l.getLine(), DebugLevel) l.output(record) } } @@ -130,8 +130,7 @@ func (l *Logger) Debug(v ...interface{}) { // Debugf ... func (l *Logger) Debugf(format string, v ...interface{}) { if l.lvl <= DebugLevel { - line := line(l.callDepth) - record := NewRecord(time.Now(), fmt.Sprintf(format, v...), line, DebugLevel) + record := NewRecord(time.Now(), fmt.Sprintf(format, v...), l.getLine(), DebugLevel) l.output(record) } } @@ -171,8 +170,7 @@ func (l *Logger) Warningf(format string, v ...interface{}) { // Error ... func (l *Logger) Error(v ...interface{}) { if l.lvl <= ErrorLevel { - line := line(l.callDepth) - record := NewRecord(time.Now(), fmt.Sprint(v...), line, ErrorLevel) + record := NewRecord(time.Now(), fmt.Sprint(v...), l.getLine(), ErrorLevel) l.output(record) } } @@ -180,8 +178,7 @@ func (l *Logger) Error(v ...interface{}) { // Errorf ... func (l *Logger) Errorf(format string, v ...interface{}) { if l.lvl <= ErrorLevel { - line := line(l.callDepth) - record := NewRecord(time.Now(), fmt.Sprintf(format, v...), line, ErrorLevel) + record := NewRecord(time.Now(), fmt.Sprintf(format, v...), l.getLine(), ErrorLevel) l.output(record) } } @@ -189,8 +186,7 @@ func (l *Logger) Errorf(format string, v ...interface{}) { // Fatal ... func (l *Logger) Fatal(v ...interface{}) { if l.lvl <= FatalLevel { - line := line(l.callDepth) - record := NewRecord(time.Now(), fmt.Sprint(v...), line, FatalLevel) + record := NewRecord(time.Now(), fmt.Sprint(v...), l.getLine(), FatalLevel) l.output(record) } os.Exit(1) @@ -199,13 +195,19 @@ func (l *Logger) Fatal(v ...interface{}) { // Fatalf ... func (l *Logger) Fatalf(format string, v ...interface{}) { if l.lvl <= FatalLevel { - line := line(l.callDepth) - record := NewRecord(time.Now(), fmt.Sprintf(format, v...), line, FatalLevel) + record := NewRecord(time.Now(), fmt.Sprintf(format, v...), l.getLine(), FatalLevel) l.output(record) } os.Exit(1) } +func (l *Logger) getLine() string { + if l.skipLine { + return "" + } + return line(l.callDepth) +} + // Debug ... func Debug(v ...interface{}) { logger.Debug(v...)