Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-08-20 19:12:54 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #210 from reasonerjt/job-service

Browse Source 
job service rework
This commit is contained in:
Daniel Jiang 2016-05-10 19:50:38 +08:00
commit 61bcfa2e10
47 changed files with 2457 additions and 728 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -5,3 +5,4 @@ Deploy/config/ui/app.conf
Deploy/config/db/env
Deploy/harbor.cfg
ui/ui
*.pyc

4
.travis.yml
View File

@ -1,14 +1,14 @@
language: go
go:
- 1.5.3
- 1.6.2
go_import_path: github.com/vmware/harbor
service:
- mysql
env: GO15VENDOREXPERIMENT=1 DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD=
env: DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD=
install:
- sudo apt-get update && sudo apt-get install -y libldap2-dev

52
Deploy/db/registry.sql
View File

@ -103,19 +103,41 @@ create table access_log (
FOREIGN KEY (project_id) REFERENCES project (project_id)
);
create table job (
job_id int NOT NULL AUTO_INCREMENT,
job_type varchar(64) NOT NULL,
status varchar(64) NOT NULL,
options text,
parms text,
create table replication_policy (
id int NOT NULL AUTO_INCREMENT,
name varchar(256),
project_id int NOT NULL,
target_id int NOT NULL,
enabled tinyint(1) NOT NULL DEFAULT 1,
description text,
cron_str varchar(256),
triggered_by varchar(64),
creation_time timestamp,
update_time timestamp,
PRIMARY KEY (job_id)
);
start_time timestamp,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
PRIMARY KEY (id)
);
create table replication_target (
id int NOT NULL AUTO_INCREMENT,
name varchar(64),
url varchar(64),
username varchar(40),
password varchar(40),
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
PRIMARY KEY (id)
);
create table replication_job (
id int NOT NULL AUTO_INCREMENT,
status varchar(64) NOT NULL,
policy_id int NOT NULL,
repository varchar(256) NOT NULL,
operation varchar(64) NOT NULL,
creation_time timestamp default CURRENT_TIMESTAMP,
update_time timestamp default CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP,
PRIMARY KEY (id)
);
create table job_log (
log_id int NOT NULL AUTO_INCREMENT,
@ -125,7 +147,7 @@ create table job_log (
creation_time timestamp,
update_time timestamp,
PRIMARY KEY (log_id),
FOREIGN KEY (job_id) REFERENCES job (job_id)
FOREIGN KEY (job_id) REFERENCES replication_job (id)
);
create table properties (
@ -136,3 +158,9 @@ create table properties (
insert into properties (k, v) values
('schema_version', '0.1.1');
CREATE TABLE IF NOT EXISTS `alembic_version` (
`version_num` varchar(32) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
insert into alembic_version values ('0.1.1');

2
Deploy/prepare
View File

@ -116,7 +116,7 @@ FNULL = open(os.devnull, 'w')
from functools import wraps
def stat_decorator(func):
#@wraps(func)
@wraps(func)
def check_wrapper(*args, **kwargs):
stat = func(*args, **kwargs)
message = "Generated configuration file: %s" % kwargs['path'] \

2
Deploy/templates/ui/env
View File

@ -12,4 +12,4 @@ LDAP_URL=$ldap_url
LDAP_BASE_DN=$ldap_basedn
SELF_REGISTRATION=$self_registration
LOG_LEVEL=debug
GODEBUG=netdns=cgo
GODEBUG=netdns=cgo

3
Dockerfile.ui
View File

@ -1,4 +1,4 @@
FROM golang:1.5.1
FROM golang:1.6.2
MAINTAINER jiangd@vmware.com
@ -11,7 +11,6 @@ COPY . /go/src/github.com/vmware/harbor
COPY ./vendor/golang.org /go/src/golang.org
WORKDIR /go/src/github.com/vmware/harbor/ui
ENV GO15VENDOREXPERIMENT 1
RUN go get -d github.com/docker/distribution \
&& go get -d github.com/docker/libtrust \
&& go get -d github.com/go-sql-driver/mysql \

4
api/job.go
View File

@ -1,5 +1,6 @@
package api
/*
import (
"encoding/json"
"fmt"
@ -15,6 +16,7 @@ type JobAPI struct {
BaseAPI
}
func (ja *JobAPI) Post() {
var je models.JobEntry
ja.DecodeJSONReq(&je)
@ -82,4 +84,4 @@ func (ja *JobAPI) Get() {
ja.Data["json"] = jobs
}
ja.ServeJSON()
}
}*/

106
api/replication.go Normal file
View File

@ -0,0 +1,106 @@
package api
import (
"encoding/json"
"fmt"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/job"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"strconv"
"strings"
)
type ReplicationJob struct {
BaseAPI
}
type ReplicationReq struct {
PolicyID int64 `json:"policy_id"`
}
func (rj *ReplicationJob) Post() {
var data ReplicationReq
rj.DecodeJSONReq(&data)
log.Debugf("data: %+v", data)
p, err := dao.GetRepPolicy(data.PolicyID)
if err != nil {
log.Errorf("Failed to get policy, error: %v", err)
rj.RenderError(http.StatusInternalServerError, fmt.Sprintf("Failed to get policy, id: %d", data.PolicyID))
return
}
repoList, err := getRepoList(p.ProjectID)
if err != nil {
log.Errorf("Failed to get repository list, project id: %d, error: %v", p.ProjectID, err)
rj.RenderError(http.StatusInternalServerError, err.Error())
return
}
log.Debugf("repo list: %v", repoList)
for _, repo := range repoList {
j := models.RepJob{
Repository: repo,
PolicyID: data.PolicyID,
Operation: models.RepOpTransfer,
}
log.Debugf("Creating job for repo: %s, policy: %d", repo, data.PolicyID)
id, err := dao.AddRepJob(j)
if err != nil {
log.Errorf("Failed to insert job record, error: %v", err)
rj.RenderError(http.StatusInternalServerError, err.Error())
return
}
log.Debugf("Send job to scheduler, job id: %d", id)
job.Schedule(id)
}
}
// calls the api from UI to get repo list
func getRepoList(projectID int64) ([]string, error) {
uiURL := os.Getenv("UI_URL")
if len(uiURL) == 0 {
uiURL = "ui"
}
if !strings.HasSuffix(uiURL, "/") {
uiURL += "/"
}
//TODO:Use secret key instead
uiUser := os.Getenv("UI_USR")
if len(uiUser) == 0 {
uiUser = "admin"
}
uiPwd := os.Getenv("UI_PWD")
if len(uiPwd) == 0 {
uiPwd = "Harbor12345"
}
client := &http.Client{}
req, err := http.NewRequest("GET", uiURL+"api/repositories?project_id="+strconv.Itoa(int(projectID)), nil)
if err != nil {
log.Errorf("Error when creating request: %v")
return nil, err
}
req.SetBasicAuth(uiUser, uiPwd)
resp, err := client.Do(req)
if err != nil {
log.Errorf("Error when calling UI api to get repositories, error: %v", err)
return nil, err
}
if resp.StatusCode != http.StatusOK {
log.Errorf("Unexpected status code: %d", resp.StatusCode)
dump, _ := httputil.DumpResponse(resp, true)
log.Debugf("response: %q", dump)
return nil, fmt.Errorf("Unexpected status code when getting repository list: %d", resp.StatusCode)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Errorf("Failed to read the response body, error: %v")
return nil, err
}
var repoList []string
err = json.Unmarshal(body, &repoList)
return repoList, err
}

124
api/repository.go
View File

@ -23,6 +23,7 @@ import (
"strings"
"time"
"github.com/docker/distribution/manifest/schema1"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
svc_utils "github.com/vmware/harbor/service/utils"
@ -40,61 +41,20 @@ type RepositoryAPI struct {
BaseAPI
userID int
username string
registry *registry.Registry
}
// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission.
func (ra *RepositoryAPI) Prepare() {
userID, ok := ra.GetSession("userId").(int)
if !ok {
ra.userID = dao.NonExistUserID
} else {
ra.userID = userID
userID = dao.NonExistUserID
}
ra.userID = userID
username, ok := ra.GetSession("username").(string)
if !ok {
log.Warning("failed to get username from session")
ra.username = ""
} else {
if ok {
ra.username = username
}
var client *http.Client
//no session, initialize a standard auth handler
if ra.userID == dao.NonExistUserID && len(ra.username) == 0 {
username, password, _ := ra.Ctx.Request.BasicAuth()
credential := auth.NewBasicAuthCredential(username, password)
client = registry.NewClientStandardAuthHandlerEmbeded(credential)
log.Debug("initializing standard auth handler")
} else {
// session works, initialize a username auth handler
username := ra.username
if len(username) == 0 {
user, err := dao.GetUser(models.User{
UserID: ra.userID,
})
if err != nil {
log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err)
return
}
username = user.Username
}
client = registry.NewClientUsernameAuthHandlerEmbeded(username)
log.Debug("initializing username auth handler: %s", username)
}
endpoint := os.Getenv("REGISTRY_URL")
r, err := registry.New(endpoint, client)
if err != nil {
log.Fatalf("error occurred while initializing auth handler for repository API: %v", err)
}
ra.registry = r
}
// Get ...
@ -156,10 +116,16 @@ func (ra *RepositoryAPI) Delete() {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags := []string{}
tag := ra.GetString("tag")
if len(tag) == 0 {
tagList, err := ra.registry.ListTag(repoName)
tagList, err := rc.ListTag()
if err != nil {
e, ok := errors.ParseError(err)
if ok {
@ -169,16 +135,14 @@ func (ra *RepositoryAPI) Delete() {
log.Error(err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
}
tags = append(tags, tagList...)
} else {
tags = append(tags, tag)
}
for _, t := range tags {
if err := ra.registry.DeleteTag(repoName, t); err != nil {
if err := rc.DeleteTag(t); err != nil {
e, ok := errors.ParseError(err)
if ok {
ra.CustomAbort(e.StatusCode, e.Message)
@ -206,15 +170,23 @@ type tag struct {
// GetTags handles GET /api/repositories/tags
func (ra *RepositoryAPI) GetTags() {
var tags []string
repoName := ra.GetString("repo_name")
tags, err := ra.registry.ListTag(repoName)
if len(repoName) == 0 {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags := []string{}
ts, err := rc.ListTag()
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
@ -222,6 +194,8 @@ func (ra *RepositoryAPI) GetTags() {
}
}
tags = append(tags, ts...)
ra.Data["json"] = tags
ra.ServeJSON()
}
@ -231,13 +205,23 @@ func (ra *RepositoryAPI) GetManifests() {
repoName := ra.GetString("repo_name")
tag := ra.GetString("tag")
if len(repoName) == 0 || len(tag) == 0 {
ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
item := models.RepoItem{}
_, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1)
mediaTypes := []string{schema1.MediaTypeManifest}
_, _, payload, err := rc.PullManifest(tag, mediaTypes)
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
@ -264,3 +248,31 @@ func (ra *RepositoryAPI) GetManifests() {
ra.Data["json"] = item
ra.ServeJSON()
}
func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) {
endpoint := os.Getenv("REGISTRY_URL")
//no session, use basic auth
if ra.userID == dao.NonExistUserID {
username, password, _ := ra.Ctx.Request.BasicAuth()
credential := auth.NewBasicAuthCredential(username, password)
return registry.NewRepositoryWithCredential(repoName, endpoint, credential)
}
//session exists, use username
if len(ra.username) == 0 {
u := models.User{
UserID: ra.userID,
}
user, err := dao.GetUser(u)
if err != nil {
return nil, err
}
ra.username = user.Username
}
return registry.NewRepositoryWithUsername(repoName, endpoint, ra.username)
}

4
api/user.go
View File

@ -187,7 +187,9 @@ func (ua *UserAPI) Delete() {
// ChangePassword handles PUT to /api/users/{}/password
func (ua *UserAPI) ChangePassword() {
if !(ua.AuthMode == "db_auth") {
ldapAdminUser := (ua.AuthMode == "ldap_auth" && ua.userID == 1 && ua.userID == ua.currentUserID)
if !(ua.AuthMode == "db_auth" || ldapAdminUser) {
ua.CustomAbort(http.StatusForbidden, "")
}

8
controllers/base.go
View File

@ -41,6 +41,7 @@ type BaseController struct {
beego.Controller
i18n.Locale
SelfRegistration bool
IsLdapAdminUser bool
IsAdmin bool
AuthMode string
}
@ -115,7 +116,11 @@ func (b *BaseController) Prepare() {
if sessionUserID != nil {
b.Data["Username"] = b.GetSession("username")
b.Data["UserId"] = sessionUserID.(int)
if (sessionUserID == 1 && b.AuthMode == "ldap_auth") {
b.IsLdapAdminUser = true
}
var err error
b.IsAdmin, err = dao.IsAdminRole(sessionUserID.(int))
if err != nil {
@ -126,6 +131,7 @@ func (b *BaseController) Prepare() {
b.Data["IsAdmin"] = b.IsAdmin
b.Data["SelfRegistration"] = b.SelfRegistration
b.Data["IsLdapAdminUser"] = b.IsLdapAdminUser
}

8
dao/accesslog.go
View File

@ -62,6 +62,14 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) {
sql += ` and u.username like ? `
queryParam = append(queryParam, accessLog.Username)
}
if accessLog.RepoName != "" {
sql += ` and a.repo_name = ? `
queryParam = append(queryParam, accessLog.RepoName)
}
if accessLog.RepoTag != "" {
sql += ` and a.repo_tag = ? `
queryParam = append(queryParam, accessLog.RepoTag)
}
if accessLog.Keywords != "" {
sql += ` and a.operation in ( `
keywordList := strings.Split(accessLog.Keywords, "/")

292
dao/dao_test.go
View File

@ -20,11 +20,9 @@ import (
"testing"
"time"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/models"
"github.com/astaxie/beego/orm"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
)
func execUpdate(o orm.Ormer, sql string, params interface{}) error {
@ -102,6 +100,8 @@ func clearUp(username string) {
const username string = "Tester01"
const projectName string = "test_project"
const repoTag string = "test1.1"
const repoTag2 string = "test1.2"
const SysAdmin int = 1
const projectAdmin int = 2
const developer int = 3
@ -419,6 +419,66 @@ func TestGetAccessLog(t *testing.T) {
}
}
func TestAddAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
accessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
RepoName: currentProject.Name + "/",
RepoTag: repoTag,
GUID: "N/A",
Operation: "create",
OpTime: time.Now(),
}
err = AddAccessLog(accessLog)
if err != nil {
t.Errorf("Error occurred in AddAccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
if len(accessLogList) != 1 {
t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList))
}
if accessLogList[0].RepoName != projectName+"/" {
t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName)
}
if accessLogList[0].RepoTag != repoTag {
t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag, accessLogList[0].RepoTag)
}
}
func TestAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
accessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
RepoName: currentProject.Name + "/",
RepoTag: repoTag2,
Operation: "create",
}
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/", repoTag2, "create")
if err != nil {
t.Errorf("Error occurred in AccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
if len(accessLogList) != 1 {
t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList))
}
if accessLogList[0].RepoName != projectName+"/" {
t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName)
}
if accessLogList[0].RepoTag != repoTag2 {
t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag2, accessLogList[0].RepoTag)
}
}
func TestProjectExists(t *testing.T) {
var exists bool
var err error
@ -660,3 +720,227 @@ func TestDeleteUser(t *testing.T) {
t.Errorf("user is not nil after deletion, user: %+v", user)
}
}
var targetID, policyID, jobID int64
func TestAddRepTarget(t *testing.T) {
target := models.RepTarget{
URL: "127.0.0.1:5000",
Username: "admin",
Password: "admin",
}
//_, err := AddRepTarget(target)
id, err := AddRepTarget(target)
t.Logf("added target, id: %d", id)
if err != nil {
t.Errorf("Error occurred in AddRepTarget: %v", err)
} else {
targetID = id
}
id2 := id + 99
tgt, err := GetRepTarget(id2)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id2)
}
if tgt != nil {
t.Errorf("There should not be a target with id: %d", id2)
}
tgt, err = GetRepTarget(id)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, id)
}
if tgt == nil {
t.Errorf("Unable to find a target with id: %d", id)
}
if tgt.URL != "127.0.0.1:5000" {
t.Errorf("Unexpected url in target: %s, expected 127.0.0.1:5000", tgt.URL)
}
if tgt.Username != "admin" {
t.Errorf("Unexpected username in target: %s, expected admin", tgt.Username)
}
}
func TestAddRepPolicy(t *testing.T) {
policy := models.RepPolicy{
ProjectID: 1,
Enabled: 1,
TargetID: targetID,
Description: "whatever",
Name: "mypolicy",
}
id, err := AddRepPolicy(policy)
t.Logf("added policy, id: %d", id)
if err != nil {
t.Errorf("Error occurred in AddRepPolicy: %v", err)
} else {
policyID = id
}
p, err := GetRepPolicy(id)
if err != nil {
t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id)
}
if p == nil {
t.Errorf("Unable to find a policy with id: %d", id)
}
if p.Name != "mypolicy" || p.TargetID != targetID || p.Enabled != 1 || p.Description != "whatever" {
t.Errorf("The data does not match, expected: Name: mypolicy, TargetID: %d, Enabled: 1, Description: whatever;\n result: Name: %s, TargetID: %d, Enabled: %d, Description: %s",
targetID, p.Name, p.TargetID, p.Enabled, p.Description)
}
var tm time.Time = time.Now().AddDate(0, 0, -1)
if !p.StartTime.After(tm) {
t.Errorf("Unexpected start_time: %v", p.StartTime)
}
}
func TestDisableRepPolicy(t *testing.T) {
err := DisableRepPolicy(policyID)
if err != nil {
t.Errorf("Failed to disable policy, id: %d", policyID)
}
p, err := GetRepPolicy(policyID)
if err != nil {
t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID)
}
if p == nil {
t.Errorf("Unable to find a policy with id: %d", policyID)
}
if p.Enabled == 1 {
t.Errorf("The Enabled value of replication policy is still 1 after disabled, id: %d", policyID)
}
}
func TestEnableRepPolicy(t *testing.T) {
err := EnableRepPolicy(policyID)
if err != nil {
t.Errorf("Failed to disable policy, id: %d", policyID)
}
p, err := GetRepPolicy(policyID)
if err != nil {
t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, policyID)
}
if p == nil {
t.Errorf("Unable to find a policy with id: %d", policyID)
}
if p.Enabled == 0 {
t.Errorf("The Enabled value of replication policy is still 0 after disabled, id: %d", policyID)
}
}
func TestAddRepPolicy2(t *testing.T) {
policy2 := models.RepPolicy{
ProjectID: 3,
Enabled: 0,
TargetID: 3,
Description: "whatever",
Name: "mypolicy",
}
id, err := AddRepPolicy(policy2)
t.Logf("added policy, id: %d", id)
if err != nil {
t.Errorf("Error occurred in AddRepPolicy: %v", err)
}
p, err := GetRepPolicy(id)
if err != nil {
t.Errorf("Error occurred in GetPolicy: %v, id: %d", err, id)
}
if p == nil {
t.Errorf("Unable to find a policy with id: %d", id)
}
var tm time.Time
if p.StartTime.After(tm) {
t.Errorf("Unexpected start_time: %v", p.StartTime)
}
}
func TestAddRepJob(t *testing.T) {
job := models.RepJob{
Repository: "library/ubuntu",
PolicyID: policyID,
Operation: "transfer",
}
id, err := AddRepJob(job)
if err != nil {
t.Errorf("Error occurred in AddRepJob: %v", err)
} else {
jobID = id
}
j, err := GetRepJob(id)
if err != nil {
t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, id)
}
if j == nil {
t.Errorf("Unable to find a job with id: %d", id)
}
if j.Status != models.JobPending || j.Repository != "library/ubuntu" || j.PolicyID != policyID || j.Operation != "transfer" {
t.Errorf("Expected data of job, id: %d, Status: %s, Repository: library/ubuntu, PolicyID: %d, Operation: transfer, "+
"but in returned data:, Status: %s, Repository: %s, Operation: %s, PolicyID: %d", id, models.JobPending, policyID, j.Status, j.Repository, j.Operation, j.PolicyID)
}
}
func TestUpdateRepJobStatus(t *testing.T) {
err := UpdateRepJobStatus(jobID, models.JobFinished)
if err != nil {
t.Errorf("Error occured in UpdateRepJobStatus, error: %v, id: %d", err, jobID)
return
}
j, err := GetRepJob(jobID)
if err != nil {
t.Errorf("Error occurred in GetRepJob: %v, id: %d", err, jobID)
}
if j == nil {
t.Errorf("Unable to find a job with id: %d", jobID)
}
if j.Status != models.JobFinished {
t.Errorf("Job's status: %s, expected: %s, id: %d", j.Status, models.JobFinished, jobID)
}
}
func TestDeleteRepTarget(t *testing.T) {
err := DeleteRepTarget(targetID)
if err != nil {
t.Errorf("Error occured in DeleteRepTarget: %v, id: %d", err, targetID)
return
}
t.Logf("deleted target, id: %d", targetID)
tgt, err := GetRepTarget(targetID)
if err != nil {
t.Errorf("Error occurred in GetTarget: %v, id: %d", err, targetID)
}
if tgt != nil {
t.Errorf("Able to find target after deletion, id: %d", targetID)
}
}
func TestDeleteRepPolicy(t *testing.T) {
err := DeleteRepPolicy(policyID)
if err != nil {
t.Errorf("Error occured in DeleteRepPolicy: %v, id: %d", err, policyID)
return
}
t.Logf("delete rep policy, id: %d", policyID)
p, err := GetRepPolicy(policyID)
if err != nil {
t.Errorf("Error occured in GetRepPolicy:%v", err)
}
if p != nil {
t.Errorf("Able to find rep policy after deletion, id: %d", policyID)
}
}
func TestDeleteRepJob(t *testing.T) {
err := DeleteRepJob(jobID)
if err != nil {
t.Errorf("Error occured in DeleteRepJob: %v, id: %d", err, jobID)
return
}
t.Logf("deleted rep job, id: %d", jobID)
j, err := GetRepJob(jobID)
if err != nil {
t.Errorf("Error occured in GetRepJob:%v", err)
}
if j != nil {
t.Errorf("Able to find rep job after deletion, id: %d", jobID)
}
}

113
dao/replication_job.go Normal file
View File

@ -0,0 +1,113 @@
package dao
import (
"fmt"
"github.com/astaxie/beego/orm"
"github.com/vmware/harbor/models"
)
func AddRepTarget(target models.RepTarget) (int64, error) {
o := orm.NewOrm()
return o.Insert(&target)
}
func GetRepTarget(id int64) (*models.RepTarget, error) {
o := orm.NewOrm()
t := models.RepTarget{ID: id}
err := o.Read(&t)
if err == orm.ErrNoRows {
return nil, nil
}
return &t, err
}
func DeleteRepTarget(id int64) error {
o := orm.NewOrm()
_, err := o.Delete(&models.RepTarget{ID: id})
return err
}
func AddRepPolicy(policy models.RepPolicy) (int64, error) {
o := orm.NewOrm()
sqlTpl := `insert into replication_policy (name, project_id, target_id, enabled, description, cron_str, start_time, creation_time, update_time ) values (?, ?, ?, ?, ?, ?, %s, NOW(), NOW())`
var sql string
if policy.Enabled == 1 {
sql = fmt.Sprintf(sqlTpl, "NOW()")
} else {
sql = fmt.Sprintf(sqlTpl, "NULL")
}
p, err := o.Raw(sql).Prepare()
if err != nil {
return 0, err
}
r, err := p.Exec(policy.Name, policy.ProjectID, policy.TargetID, policy.Enabled, policy.Description, policy.CronStr)
if err != nil {
return 0, err
}
id, err := r.LastInsertId()
return id, err
}
func GetRepPolicy(id int64) (*models.RepPolicy, error) {
o := orm.NewOrm()
p := models.RepPolicy{ID: id}
err := o.Read(&p)
if err == orm.ErrNoRows {
return nil, nil
}
return &p, err
}
func DeleteRepPolicy(id int64) error {
o := orm.NewOrm()
_, err := o.Delete(&models.RepPolicy{ID: id})
return err
}
func updateRepPolicyEnablement(id int64, enabled int) error {
o := orm.NewOrm()
p := models.RepPolicy{
ID: id,
Enabled: enabled}
num, err := o.Update(&p, "Enabled")
if num == 0 {
err = fmt.Errorf("Failed to update replication policy with id: %d", id)
}
return err
}
func EnableRepPolicy(id int64) error {
return updateRepPolicyEnablement(id, 1)
}
func DisableRepPolicy(id int64) error {
return updateRepPolicyEnablement(id, 0)
}
func AddRepJob(job models.RepJob) (int64, error) {
o := orm.NewOrm()
if len(job.Status) == 0 {
job.Status = models.JobPending
}
return o.Insert(&job)
}
func GetRepJob(id int64) (*models.RepJob, error) {
o := orm.NewOrm()
j := models.RepJob{ID: id}
err := o.Read(&j)
if err == orm.ErrNoRows {
return nil, nil
}
return &j, err
}
func DeleteRepJob(id int64) error {
o := orm.NewOrm()
_, err := o.Delete(&models.RepJob{ID: id})
return err
}
func UpdateRepJobStatus(id int64, status string) error {
o := orm.NewOrm()
j := models.RepJob{
ID: id,
Status: status,
}
num, err := o.Update(&j, "Status")
if num == 0 {
err = fmt.Errorf("Failed to update replication job with id: %d", id)
}
return err
}

64
docs/swagger.yaml
View File

@ -109,7 +109,7 @@ paths:
tags:
- Products
responses:
200:
201:
description: Project created successfully.
400:
description: Unsatisfied with constraints of the project creation.
@ -383,6 +383,28 @@ paths:
description: User does not have permission of admin role.
500:
description: Unexpected internal errors.
post:
summary: Creates a new user account.
description: |
This endpoint is to create a user if the user does not already exist.
parameters:
- name: user
in: body
description: New created user.
required: true
schema:
$ref: '#/definitions/User'
tags:
- Products
responses:
201:
description: User created successfully.
400:
description: Unsatisfied with constraints of the user creation.
403:
description: User registration can only be used by admin role user when self-registration is off.
500:
description: Unexpected internal errors.
/users/{user_id}:
put:
summary: Update a registered user to change to be an administrator of Harbor.
@ -438,6 +460,37 @@ paths:
description: User ID does not exist.
500:
description: Unexpected internal errors.
/users/{user_id}/password:
put:
summary: Change the password on a user that already exists.
description: |
This endpoint is for user to update password. Users with the admin role can change any user's password. Guest users can change only their own password.
parameters:
- name: user_id
in: path
type: integer
format: int32
required: true
description: Registered user ID.
- name: password
in: body
description: Password to be updated.
required: true
schema:
$ref: '#/definitions/Password'
tags:
- Products
responses:
200:
description: Updated password successfully.
400:
description: Invalid user ID; Old password is blank; New password is blank.
401:
description: Old password is not correct.
403:
description: Guests can only change their own account.
500:
description: Unexpected internal errors.
/repositories:
get:
summary: Get repositories accompany with relevant project and repo name.
@ -640,6 +693,15 @@ definitions:
deleted:
type: integer
format: int32
Password:
type: object
properties:
old_password:
type: string
description: The user's existing password.
new_password:
type: string
description: New password for marking as to be updated.
AccessLog:
type: object
properties:

2
job/imgout/runner.go
View File

@ -1,5 +1,6 @@
package imgout
/*
import (
"encoding/json"
//"github.com/vmware/harbor/dao"
@ -72,3 +73,4 @@ func (r *Runner) init(je models.JobEntry) error {
r.AddTransition("push-img", job.JobFinished, job.StatusUpdater{job.DummyHandler{JobID: r.JobID}, job.JobFinished})
return nil
}
*/

41
job/scheduler.go
View File

@ -1,32 +1,31 @@
package job
import (
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"os"
"strconv"
)
var lock chan bool
func Schedule(jobID int64) {
//TODO: introduce jobqueue to better control concurrent job numbers
go HandleRepJob(jobID)
}
const defaultMaxJobs int64 = 10
func init() {
maxJobsEnv := os.Getenv("MAX_CONCURRENT_JOB")
maxJobs, err := strconv.ParseInt(maxJobsEnv, 10, 32)
func HandleRepJob(id int64) {
sm := &JobSM{JobID: id}
err := sm.Init()
if err != nil {
log.Warningf("Failed to parse max job setting, error: %v, the default value: %d will be used", err, defaultMaxJobs)
maxJobs = defaultMaxJobs
log.Errorf("Failed to initialize statemachine, error: %v")
err2 := dao.UpdateRepJobStatus(id, models.JobError)
if err2 != nil {
log.Errorf("Failed to update job status to ERROR, error:%v", err2)
}
return
}
if sm.Parms.Enabled == 0 {
log.Debugf("The policy of job:%d is disabled, will cancel the job")
_ = dao.UpdateRepJobStatus(id, models.JobCanceled)
} else {
sm.Start(models.JobRunning)
}
lock = make(chan bool, maxJobs)
}
func Schedule(job models.JobEntry) {
log.Infof("job: %d will be scheduled", job.ID)
//TODO: add support for cron string when needed.
go func() {
lock <- true
defer func() { <-lock }()
log.Infof("running job: %d", job.ID)
run(job)
}()
}

73
job/statehandlers.go Normal file
View File

@ -0,0 +1,73 @@
package job
import (
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"time"
)
// StateHandler handles transition, it associates with each state, will be called when
// SM enters and exits a state during a transition.
type StateHandler interface {
// Enter returns the next state, if it returns empty string the SM will hold the current state or
// or decide the next state.
Enter() (string, error)
//Exit should be idempotent
Exit() error
}
type DummyHandler struct {
JobID int64
}
func (dh DummyHandler) Enter() (string, error) {
return "", nil
}
func (dh DummyHandler) Exit() error {
return nil
}
type StatusUpdater struct {
DummyHandler
State string
}
func (su StatusUpdater) Enter() (string, error) {
err := dao.UpdateRepJobStatus(su.JobID, su.State)
if err != nil {
log.Warningf("Failed to update state of job: %d, state: %s, error: %v", su.JobID, su.State, err)
}
var next string = models.JobContinue
if su.State == models.JobStopped || su.State == models.JobError || su.State == models.JobFinished {
next = ""
}
return next, err
}
type ImgPuller struct {
DummyHandler
img string
logger Logger
}
func (ip ImgPuller) Enter() (string, error) {
ip.logger.Infof("I'm pretending to pull img:%s, then sleep 30s", ip.img)
time.Sleep(30 * time.Second)
ip.logger.Infof("wake up from sleep....")
return "push-img", nil
}
type ImgPusher struct {
DummyHandler
targetURL string
logger Logger
}
func (ip ImgPusher) Enter() (string, error) {
ip.logger.Infof("I'm pretending to push img to:%s, then sleep 30s", ip.targetURL)
time.Sleep(30 * time.Second)
ip.logger.Infof("wake up from sleep....")
return models.JobContinue, nil
}

129
job/statemachine.go
View File

@ -3,59 +3,22 @@ package job
import (
"fmt"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
"github.com/vmware/harbor/utils/log"
"os"
"sync"
)
// StateHandler handles transition, it associates with each state, will be called when
// SM enters and exits a state during a transition.
type StateHandler interface {
// Enter returns the next state, if it returns empty string the SM will hold the current state or
// or decide the next state.
Enter() (string, error)
//Exit should be idempotent
Exit() error
type RepJobParm struct {
LocalRegURL string
TargetURL string
TargetUsername string
TargetPassword string
Repository string
Enabled int
Operation string
}
type DummyHandler struct {
JobID int64
}
func (dh DummyHandler) Enter() (string, error) {
return "", nil
}
func (dh DummyHandler) Exit() error {
return nil
}
type StatusUpdater struct {
DummyHandler
State string
}
func (su StatusUpdater) Enter() (string, error) {
err := dao.UpdateJobStatus(su.JobID, su.State)
if err != nil {
log.Warningf("Failed to update state of job: %d, state: %s, error: %v", su.JobID, su.State, err)
}
var next string = JobContinue
if su.State == JobStopped || su.State == JobError || su.State == JobFinished {
next = ""
}
return next, err
}
const (
JobPending string = "pending"
JobRunning string = "running"
JobError string = "error"
JobStopped string = "stopped"
JobFinished string = "finished"
// statemachine will move to next possible state based on trasition table
JobContinue string = "_continue"
)
type JobSM struct {
JobID int64
CurrentState string
@ -64,8 +27,10 @@ type JobSM struct {
ForcedStates map[string]struct{}
Transitions map[string]map[string]struct{}
Handlers map[string]StateHandler
lock *sync.Mutex
desiredState string
Logger Logger
Parms *RepJobParm
lock *sync.Mutex
}
// EnsterState transit the statemachine from the current state to the state in parameter.
@ -87,7 +52,7 @@ func (sm *JobSM) EnterState(s string) (string, error) {
log.Debugf("No handler found for state:%s, skip", sm.CurrentState)
}
enterHandler, ok := sm.Handlers[s]
var next string = JobContinue
var next string = models.JobContinue
var err error
if ok {
if next, err = enterHandler.Enter(); err != nil {
@ -115,14 +80,14 @@ func (sm *JobSM) Start(s string) {
sm.setDesiredState("")
continue
}
if n == JobContinue && len(sm.Transitions[sm.CurrentState]) == 1 {
if n == models.JobContinue && len(sm.Transitions[sm.CurrentState]) == 1 {
for n = range sm.Transitions[sm.CurrentState] {
break
}
log.Debugf("Continue to state: %s", n)
continue
}
if n == JobContinue && len(sm.Transitions[sm.CurrentState]) != 1 {
if n == models.JobContinue && len(sm.Transitions[sm.CurrentState]) != 1 {
log.Errorf("Next state is continue but there are %d possible next states in transition table", len(sm.Transitions[sm.CurrentState]))
err = fmt.Errorf("Unable to continue")
break
@ -132,7 +97,7 @@ func (sm *JobSM) Start(s string) {
}
if err != nil {
log.Warningf("The statemachin will enter error state due to error: %v", err)
sm.EnterState(JobError)
sm.EnterState(models.JobError)
}
}
@ -154,7 +119,7 @@ func (sm *JobSM) RemoveTransition(from string, to string) {
}
func (sm *JobSM) Stop() {
sm.setDesiredState(JobStopped)
sm.setDesiredState(models.JobStopped)
}
func (sm *JobSM) getDesiredState() string {
@ -169,12 +134,60 @@ func (sm *JobSM) setDesiredState(s string) {
sm.desiredState = s
}
func (sm *JobSM) InitJobSM() {
func (sm *JobSM) Init() error {
//init parms
regURL := os.Getenv("LOCAL_REGISTRY_URL")
if len(regURL) == 0 {
regURL = "http://registry:5000/"
}
job, err := dao.GetRepJob(sm.JobID)
if err != nil {
return fmt.Errorf("Failed to get job, error: %v", err)
}
if job == nil {
return fmt.Errorf("The job doesn't exist in DB, job id: %d", sm.JobID)
}
policy, err := dao.GetRepPolicy(job.PolicyID)
if err != nil {
return fmt.Errorf("Failed to get policy, error: %v", err)
}
if policy == nil {
return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID)
}
sm.Parms = &RepJobParm{
LocalRegURL: regURL,
Repository: job.Repository,
Enabled: policy.Enabled,
Operation: job.Operation,
}
if policy.Enabled == 0 {
//handler will cancel this job
return nil
}
target, err := dao.GetRepTarget(policy.TargetID)
if err != nil {
return fmt.Errorf("Failed to get target, error: %v", err)
}
if target == nil {
return fmt.Errorf("The target doesn't exist in DB, target id: %d", policy.TargetID)
}
sm.Parms.TargetURL = target.URL
sm.Parms.TargetUsername = target.Username
sm.Parms.TargetPassword = target.Password
//init states handlers
sm.lock = &sync.Mutex{}
sm.Handlers = make(map[string]StateHandler)
sm.Transitions = make(map[string]map[string]struct{})
sm.CurrentState = JobPending
sm.AddTransition(JobPending, JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, JobRunning})
sm.Handlers[JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, JobError}
sm.Handlers[JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, JobStopped}
sm.Logger = Logger{sm.JobID}
sm.CurrentState = models.JobPending
sm.AddTransition(models.JobPending, models.JobRunning, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobRunning})
sm.Handlers[models.JobError] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobError}
sm.Handlers[models.JobStopped] = StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobStopped}
if sm.Parms.Operation == models.RepOpTransfer {
sm.AddTransition(models.JobRunning, "pull-img", ImgPuller{DummyHandler: DummyHandler{JobID: sm.JobID}, img: sm.Parms.Repository, logger: sm.Logger})
//only handle on target for now
sm.AddTransition("pull-img", "push-img", ImgPusher{DummyHandler: DummyHandler{JobID: sm.JobID}, targetURL: sm.Parms.TargetURL, logger: sm.Logger})
sm.AddTransition("push-img", models.JobFinished, StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished})
}
return nil
}

3
jobservice/my_start.sh
View File

@ -3,5 +3,8 @@ export MYSQL_PORT=3306
export MYSQL_USR=root
export MYSQL_PWD=root123
export LOG_LEVEL=debug
export UI_URL=http://127.0.0.1/
export UI_USR=admin
export UI_PWD=Harbor12345
./jobservice

1
jobservice/newtest.json Normal file
View File

@ -0,0 +1 @@
{"policy_id": 1}

2
jobservice/populate.sql Normal file
View File

@ -0,0 +1,2 @@
insert into replication_target (name, url, username, password) values ('test', '192.168.0.2:5000', 'testuser', 'passw0rd');
insert into replication_policy (name, project_id, target_id, enabled, start_time) value ('test_policy', 1, 1, 1, NOW());

2
jobservice/router.go
View File

@ -7,5 +7,5 @@ import (
)
func initRouters() {
beego.Router("/api/jobs/?:id", &api.JobAPI{})
beego.Router("/api/jobs/replication", &api.ReplicationJob{})
}

23
migration/Dockerfile Normal file
View File

@ -0,0 +1,23 @@
FROM mysql:5.6
MAINTAINER bhe@vmware.com
RUN sed -i -e 's/us.archive.ubuntu.com/archive.ubuntu.com/g' /etc/apt/sources.list
RUN apt-get update
RUN apt-get install -y curl python python-pip git python-mysqldb
RUN pip install alembic
RUN mkdir -p /harbor-migration
WORKDIR /harbor-migration
COPY ./ ./
COPY ./migration.cfg ./
RUN ./prepare.sh
ENTRYPOINT ["./run.sh"]

51
migration/README.md Normal file
View File

@ -0,0 +1,51 @@
# migration
Migration is a module for migrating database schema between different version of project [harbor](https://github.com/vmware/harbor)
**WARNING!!** You must backup your data before migrating
###installation
- step 1: modify migration.cfg
- step 2: build image from dockerfile
```
cd harbor-migration
docker build -t your-image-name .
```
###migration operation
- show instruction of harbor-migration
```docker run your-image-name help```
- create backup file in `/path/to/backup`
```
docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name backup
```
- restore from backup file in `/path/to/backup`
```
docker run -ti -v /data/database:/var/lib/mysql -v /path/to/backup:/harbor-migration/backup your-image-name restore
```
- perform database schema upgrade
```docker run -ti -v /data/database:/var/lib/mysql your-image-name up head```
- perform database schema downgrade(downgrade has been disabled)
```docker run -v /data/database:/var/lib/mysql your-image-name down base```
###migration step
- step 1: stop and remove harbor service
```
docker-compose stop && docker-compose rm -f
```
- step 2: perform migration operation
- step 3: rebuild newest harbor images and restart service
```
docker-compose build && docker-compose up -d
```

4
migration/alembic.sql Normal file
View File

@ -0,0 +1,4 @@
use `registry`;
CREATE TABLE IF NOT EXISTS `alembic_version` (
`version_num` varchar(32) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

68
migration/alembic.tpl Normal file
View File

@ -0,0 +1,68 @@
echo "
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = migration_harbor
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; this defaults
# to migration_harbor/versions. When using multiple version
# directories, initial revisions must be specified with --version-path
# version_locations = %(here)s/bar %(here)s/bat migration_harbor/versions
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = mysql://$db_username:$db_password@localhost:$db_port/$db_name
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S"

4
migration/migration.cfg Normal file
View File

@ -0,0 +1,4 @@
db_username="root"
db_password="root123"
db_port="3306"
db_name="registry"

85
migration/migration_harbor/env.py Normal file
View File

@ -0,0 +1,85 @@
# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import with_statement
from alembic import context
from sqlalchemy import engine_from_config, pool
from logging.config import fileConfig
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = None
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=target_metadata, literal_binds=True)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section),
prefix='sqlalchemy.',
poolclass=pool.NullPool)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
migration/migration_harbor/script.py.mako Normal file
View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

128
migration/migration_harbor/versions/0_1_1.py Normal file
View File

@ -0,0 +1,128 @@
# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""0.1.0 to 0.1.1
Revision ID: 0.1.1
Revises:
Create Date: 2016-04-18 18:32:14.101897
"""
# revision identifiers, used by Alembic.
revision = '0.1.1'
down_revision = None
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import mysql
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, relationship
from datetime import datetime
Session = sessionmaker()
Base = declarative_base()
class Properties(Base):
__tablename__ = 'properties'
k = sa.Column(sa.String(64), primary_key = True)
v = sa.Column(sa.String(128), nullable = False)
class ProjectMember(Base):
__tablename__ = 'project_member'
project_id = sa.Column(sa.Integer(), primary_key = True)
user_id = sa.Column(sa.Integer(), primary_key = True)
role = sa.Column(sa.Integer(), nullable = False)
creation_time = sa.Column(sa.DateTime(), nullable = True)
update_time = sa.Column(sa.DateTime(), nullable = True)
sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'], ),
sa.ForeignKeyConstraint(['role'], [u'role.role_id'], ),
sa.ForeignKeyConstraint(['user_id'], [u'user.user_id'], ),
class UserProjectRole(Base):
__tablename__ = 'user_project_role'
upr_id = sa.Column(sa.Integer(), primary_key = True)
user_id = sa.Column(sa.Integer(), sa.ForeignKey('user.user_id'))
pr_id = sa.Column(sa.Integer(), sa.ForeignKey('project_role.pr_id'))
project_role = relationship("ProjectRole")
class ProjectRole(Base):
__tablename__ = 'project_role'
pr_id = sa.Column(sa.Integer(), primary_key = True)
project_id = sa.Column(sa.Integer(), nullable = False)
role_id = sa.Column(sa.Integer(), nullable = False)
sa.ForeignKeyConstraint(['role_id'], [u'role.role_id'])
sa.ForeignKeyConstraint(['project_id'], [u'project.project_id'])
class Access(Base):
__tablename__ = 'access'
access_id = sa.Column(sa.Integer(), primary_key = True)
access_code = sa.Column(sa.String(1))
comment = sa.Column(sa.String(30))
def upgrade():
"""
update schema&data
"""
bind = op.get_bind()
session = Session(bind=bind)
#delete M from table access
acc = session.query(Access).filter_by(access_id=1).first()
session.delete(acc)
#create table property
Properties.__table__.create(bind)
session.add(Properties(k='schema_version', v='0.1.1'))
#create table project_member
ProjectMember.__table__.create(bind)
#fill data
join_result = session.query(UserProjectRole).join(UserProjectRole.project_role).all()
for result in join_result:
session.add(ProjectMember(project_id=result.project_role.project_id, \
user_id=result.user_id, role=result.project_role.role_id, \
creation_time=datetime.now(), update_time=datetime.now()))
#drop user_project_role table before drop project_role
#because foreign key constraint
op.drop_table('user_project_role')
op.drop_table('project_role')
#add column to table project
op.add_column('project', sa.Column('update_time', sa.DateTime(), nullable=True))
#add column to table role
op.add_column('role', sa.Column('role_mask', sa.Integer(), server_default=sa.text(u"'0'"), nullable=False))
#add column to table user
op.add_column('user', sa.Column('creation_time', sa.DateTime(), nullable=True))
op.add_column('user', sa.Column('sysadmin_flag', sa.Integer(), nullable=True))
op.add_column('user', sa.Column('update_time', sa.DateTime(), nullable=True))
session.commit()
def downgrade():
"""
Downgrade has been disabled.
"""
pass

3
migration/prepare.sh Executable file
View File

@ -0,0 +1,3 @@
#!/bin/bash
source ./migration.cfg
source ./alembic.tpl > ./alembic.ini

94
migration/run.sh Executable file
View File

@ -0,0 +1,94 @@
#!/bin/bash
source ./migration.cfg
WAITTIME=60
DBCNF="-hlocalhost -u${db_username}"
#prevent shell to print insecure message
export MYSQL_PWD="${db_password}"
if [[ $1 = "help" || $1 = "h" || $# = 0 ]]; then
echo "Usage:"
echo "backup perform database backup"
echo "restore perform database restore"
echo "up, upgrade perform database schema upgrade"
echo "h, help usage help"
exit 0
fi
if [[ $1 = "up" || $1 = "upgrade" ]]; then
echo "Please backup before upgrade."
read -p "Enter y to continue updating or n to abort:" ans
case $ans in
[Yy]* )
;;
[Nn]* )
exit 0
;;
* ) echo "illegal answer: $ans. Upgrade abort!!"
exit 1
;;
esac
fi
echo 'Trying to start mysql server...'
DBRUN=0
nohup mysqld 2>&1 > ./nohup.log&
for i in $(seq 1 $WAITTIME); do
echo "$(/usr/sbin/service mysql status)"
if [[ "$(/usr/sbin/service mysql status)" =~ "not running" ]]; then
sleep 1
else
DBRUN=1
break
fi
done
if [[ $DBRUN -eq 0 ]]; then
echo "timeout. Can't run mysql server."
exit 1
fi
key="$1"
case $key in
up|upgrade)
VERSION="$2"
if [[ -z $VERSION ]]; then
VERSION="head"
echo "Version is not specified. Default version is head."
fi
echo "Performing upgrade ${VERSION}..."
if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \
where table_schema='registry' and table_name='alembic_version';") -eq 0 ]]; then
echo "table alembic_version does not exist. Trying to initial alembic_version."
mysql $DBCNF < ./alembic.sql
#compatible with version 0.1.0 and 0.1.1
if [[ $(mysql $DBCNF -N -s -e "select count(*) from information_schema.tables \
where table_schema='registry' and table_name='properties'") -eq 0 ]]; then
echo "table properties does not exist. The version of registry is 0.1.0"
else
echo "The version of registry is 0.1.1"
mysql $DBCNF -e "insert into registry.alembic_version values ('0.1.1')"
fi
fi
alembic -c ./alembic.ini upgrade ${VERSION}
echo "Upgrade performed."
;;
backup)
echo "Performing backup..."
mysqldump $DBCNF --add-drop-database --databases registry > ./backup/registry.sql
echo "Backup performed."
;;
restore)
echo "Performing restore..."
mysql $DBCNF < ./backup/registry.sql
echo "Restore performed."
;;
*)
echo "unknown option"
exit 0
;;
esac

11
models/base.go Normal file
View File

@ -0,0 +1,11 @@
package models
import (
"github.com/astaxie/beego/orm"
)
func init() {
orm.RegisterModel(new(RepTarget),
new(RepPolicy),
new(RepJob))
}

65
models/replication_job.go Normal file
View File

@ -0,0 +1,65 @@
package models
import (
"time"
)
const (
JobPending string = "pending"
JobRunning string = "running"
JobError string = "error"
JobStopped string = "stopped"
JobFinished string = "finished"
JobCanceled string = "canceled"
// statemachine will move to next possible state based on trasition table
JobContinue string = "_continue"
RepOpTransfer string = "transfer"
RepOpDelete string = "delete"
)
type RepPolicy struct {
ID int64 `orm:"column(id)" json:"id"`
ProjectID int64 `orm:"column(project_id)" json:"project_id"`
TargetID int64 `orm:"column(target_id)" json:"target_id"`
Name string `orm:"column(name)" json:"name"`
Target RepTarget `orm:"-" json:"target"`
Enabled int `orm:"column(enabled)" json:"enabled"`
Description string `orm:"column(description)" json:"description"`
CronStr string `orm:"column(cron_str)" json:"cron_str"`
StartTime time.Time `orm:"column(start_time)" json:"start_time"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
type RepJob struct {
ID int64 `orm:"column(id)" json:"id"`
Status string `orm:"column(status)" json:"status"`
Repository string `orm:"column(repository)" json:"repository"`
PolicyID int64 `orm:"column(policy_id)" json:"policy_id"`
Operation string `orm:"column(operation)" json:"operation"`
Policy RepPolicy `orm:"-" json:"policy"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
type RepTarget struct {
ID int64 `orm:"column(id)" json:"id"`
URL string `orm:"column(url)" json:"url"`
Name string `orm:"column(name)" json:"name"`
Username string `orm:"column(username)" json:"username"`
Password string `orm:"column(password)" json:"password"`
CreationTime time.Time `orm:"column(creation_time);auto_now_add" json:"creation_time"`
UpdateTime time.Time `orm:"column(update_time);auto_now" json:"update_time"`
}
func (rt *RepTarget) TableName() string {
return "replication_target"
}
func (rj *RepJob) TableName() string {
return "replication_job"
}
func (rp *RepPolicy) TableName() string {
return "replication_policy"
}

31
service/token/authutils.go
View File

@ -39,6 +39,7 @@ const (
// GetResourceActions ...
func GetResourceActions(scopes []string) []*token.ResourceActions {
log.Debugf("scopes: %+v", scopes)
var res []*token.ResourceActions
for _, s := range scopes {
if s == "" {
@ -59,6 +60,7 @@ func GetResourceActions(scopes []string) []*token.ResourceActions {
func FilterAccess(username string, authenticated bool, a *token.ResourceActions) {
if a.Type == "registry" && a.Name == "catalog" {
log.Infof("current access, type: %s, name:%s, actions:%v \n", a.Type, a.Name, a.Actions)
return
}
@ -108,7 +110,7 @@ func FilterAccess(username string, authenticated bool, a *token.ResourceActions)
}
// GenTokenForUI is for the UI process to call, so it won't establish a https connection from UI to proxy.
func GenTokenForUI(username string, service string, scopes []string) (string, error) {
func GenTokenForUI(username string, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
access := GetResourceActions(scopes)
for _, a := range access {
FilterAccess(username, true, a)
@ -117,22 +119,22 @@ func GenTokenForUI(username string, service string, scopes []string) (string, er
}
// MakeToken makes a valid jwt token based on parms.
func MakeToken(username, service string, access []*token.ResourceActions) (string, error) {
func MakeToken(username, service string, access []*token.ResourceActions) (token string, expiresIn int, issuedAt *time.Time, err error) {
pk, err := libtrust.LoadKeyFile(privateKey)
if err != nil {
return "", err
return "", 0, nil, err
}
tk, err := makeTokenCore(issuer, username, service, expiration, access, pk)
tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk)
if err != nil {
return "", err
return "", 0, nil, err
}
rs := fmt.Sprintf("%s.%s", tk.Raw, base64UrlEncode(tk.Signature))
return rs, nil
return rs, expiresIn, issuedAt, nil
}
//make token core
func makeTokenCore(issuer, subject, audience string, expiration int,
access []*token.ResourceActions, signingKey libtrust.PrivateKey) (*token.Token, error) {
access []*token.ResourceActions, signingKey libtrust.PrivateKey) (t *token.Token, expiresIn int, issuedAt *time.Time, err error) {
joseHeader := &token.Header{
Type: "JWT",
@ -142,10 +144,12 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
jwtID, err := randString(16)
if err != nil {
return nil, fmt.Errorf("Error to generate jwt id: %s", err)
return nil, 0, nil, fmt.Errorf("Error to generate jwt id: %s", err)
}
now := time.Now()
now := time.Now().UTC()
issuedAt = &now
expiresIn = expiration * 60
claimSet := &token.ClaimSet{
Issuer: issuer,
@ -161,10 +165,10 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
var joseHeaderBytes, claimSetBytes []byte
if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil {
return nil, fmt.Errorf("unable to marshal jose header: %s", err)
return nil, 0, nil, fmt.Errorf("unable to marshal jose header: %s", err)
}
if claimSetBytes, err = json.Marshal(claimSet); err != nil {
return nil, fmt.Errorf("unable to marshal claim set: %s", err)
return nil, 0, nil, fmt.Errorf("unable to marshal claim set: %s", err)
}
encodedJoseHeader := base64UrlEncode(joseHeaderBytes)
@ -173,12 +177,13 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
var signatureBytes []byte
if signatureBytes, _, err = signingKey.Sign(strings.NewReader(payload), crypto.SHA256); err != nil {
return nil, fmt.Errorf("unable to sign jwt payload: %s", err)
return nil, 0, nil, fmt.Errorf("unable to sign jwt payload: %s", err)
}
signature := base64UrlEncode(signatureBytes)
tokenString := fmt.Sprintf("%s.%s", payload, signature)
return token.NewToken(tokenString)
t, err = token.NewToken(tokenString)
return
}
func randString(length int) (string, error) {

8
service/token/token.go
View File

@ -17,6 +17,7 @@ package token
import (
"net/http"
"time"
"github.com/vmware/harbor/auth"
"github.com/vmware/harbor/models"
@ -43,7 +44,6 @@ func (h *Handler) Get() {
authenticated := authenticate(username, password)
service := h.GetString("service")
scopes := h.GetStrings("scope")
log.Debugf("scopes: %+v", scopes)
if len(scopes) == 0 && !authenticated {
log.Info("login request with invalid credentials")
@ -59,14 +59,16 @@ func (h *Handler) Get() {
func (h *Handler) serveToken(username, service string, access []*token.ResourceActions) {
writer := h.Ctx.ResponseWriter
//create token
rawToken, err := MakeToken(username, service, access)
rawToken, expiresIn, issuedAt, err := MakeToken(username, service, access)
if err != nil {
log.Errorf("Failed to make token, error: %v", err)
writer.WriteHeader(http.StatusInternalServerError)
return
}
tk := make(map[string]string)
tk := make(map[string]interface{})
tk["token"] = rawToken
tk["expires_in"] = expiresIn
tk["issued_at"] = issuedAt.Format(time.RFC3339)
h.Data["json"] = tk
h.ServeJSON()
}

45
service/utils/cache.go
View File

@ -25,10 +25,14 @@ import (
"github.com/astaxie/beego/cache"
)
// Cache is the global cache in system.
var Cache cache.Cache
var registryClient *registry.Registry
var (
// Cache is the global cache in system.
Cache cache.Cache
endpoint string
username string
registryClient *registry.Registry
repositoryClients map[string]*registry.Repository
)
const catalogKey string = "catalog"
@ -39,17 +43,25 @@ func init() {
log.Errorf("Failed to initialize cache, error:%v", err)
}
endpoint := os.Getenv("REGISTRY_URL")
client := registry.NewClientUsernameAuthHandlerEmbeded("admin")
registryClient, err = registry.New(endpoint, client)
if err != nil {
log.Fatalf("error occurred while initializing authentication handler used by cache: %v", err)
}
endpoint = os.Getenv("REGISTRY_URL")
username = "admin"
repositoryClients = make(map[string]*registry.Repository, 10)
}
// RefreshCatalogCache calls registry's API to get repository list and write it to cache.
func RefreshCatalogCache() error {
log.Debug("refreshing catalog cache...")
if registryClient == nil {
var err error
registryClient, err = registry.NewRegistryWithUsername(endpoint, username)
if err != nil {
log.Errorf("error occurred while initializing registry client used by cache: %v", err)
return err
}
}
var err error
rs, err := registryClient.Catalog()
if err != nil {
return err
@ -58,10 +70,19 @@ func RefreshCatalogCache() error {
repos := []string{}
for _, repo := range rs {
tags, err := registryClient.ListTag(repo)
rc, ok := repositoryClients[repo]
if !ok {
rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username)
if err != nil {
log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err)
continue
}
repositoryClients[repo] = rc
}
tags, err := rc.ListTag()
if err != nil {
log.Errorf("error occurred while list tag for %s: %v", repo, err)
return err
continue
}
if len(tags) != 0 {

60
utils/registry/auth/authorizer.go Normal file
View File

@ -0,0 +1,60 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"net/http"
au "github.com/docker/distribution/registry/client/auth"
)
// Handler authorizes requests according to the schema
type Handler interface {
// Scheme : basic, bearer
Scheme() string
//AuthorizeRequest adds basic auth or token auth to the header of request
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// RequestAuthorizer holds a handler list, which will authorize request.
// Implements interface RequestModifier
type RequestAuthorizer struct {
handlers []Handler
challenges []au.Challenge
}
// NewRequestAuthorizer ...
func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *RequestAuthorizer {
return &RequestAuthorizer{
handlers: handlers,
challenges: challenges,
}
}
// ModifyRequest adds authorization to the request
func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error {
for _, handler := range r.handlers {
for _, challenge := range r.challenges {
if handler.Scheme() == challenge.Scheme {
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return err
}
}
}
}
return nil
}

44
utils/registry/auth/credential.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"net/http"
)
// Credential ...
type Credential interface {
// AddAuthorization adds authorization information to request
AddAuthorization(req *http.Request)
}
// Implements interface Credential
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}

197
utils/registry/auth/handler.go
View File

@ -1,197 +0,0 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
token_util "github.com/vmware/harbor/service/token"
"github.com/vmware/harbor/utils/log"
registry_errors "github.com/vmware/harbor/utils/registry/errors"
)
const (
// credential type
basicAuth string = "basic_auth"
secretKey string = "secret_key"
)
// Handler authorizes the request when encounters a 401 error
type Handler interface {
// Schema : basic, bearer
Schema() string
//AuthorizeRequest adds basic auth or token auth to the header of request
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// Credential ...
type Credential interface {
// AddAuthorization adds authorization information to request
AddAuthorization(req *http.Request)
}
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}
type token struct {
Token string `json:"token"`
}
type standardTokenHandler struct {
client *http.Client
credential Credential
}
// NewStandardTokenHandler returns a standard token handler. The handler will request a token
// from token server whose URL is specified in the "WWW-authentication" header and add it to
// the origin request
// TODO deal with https
func NewStandardTokenHandler(credential Credential) Handler {
return &standardTokenHandler{
client: &http.Client{
Transport: http.DefaultTransport,
},
credential: credential,
}
}
// Schema implements the corresponding method in interface AuthHandler
func (t *standardTokenHandler) Schema() string {
return "bearer"
}
// AuthorizeRequest implements the corresponding method in interface AuthHandler
func (t *standardTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
realm, ok := params["realm"]
if !ok {
return errors.New("no realm")
}
service := params["service"]
scope := params["scope"]
u, err := url.Parse(realm)
if err != nil {
return err
}
q := u.Query()
q.Add("service", service)
for _, s := range strings.Split(scope, " ") {
q.Add("scope", s)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
}
t.credential.AddAuthorization(r)
resp, err := t.client.Do(r)
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return registry_errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
decoder := json.NewDecoder(resp.Body)
tk := &token{}
if err = decoder.Decode(tk); err != nil {
return err
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk.Token))
log.Debugf("standardTokenHandler generated token successfully | %s %s", req.Method, req.URL)
return nil
}
type usernameTokenHandler struct {
username string
}
// NewUsernameTokenHandler returns a handler which will generate
// a token according the user's privileges
func NewUsernameTokenHandler(username string) Handler {
return &usernameTokenHandler{
username: username,
}
}
// Schema implements the corresponding method in interface AuthHandler
func (u *usernameTokenHandler) Schema() string {
return "bearer"
}
// AuthorizeRequest implements the corresponding method in interface AuthHandler
func (u *usernameTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
service := params["service"]
scopes := []string{}
scope := params["scope"]
if len(scope) != 0 {
scopes = strings.Split(scope, " ")
}
token, err := token_util.GenTokenForUI(u.username, service, scopes)
if err != nil {
return err
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
log.Debugf("usernameTokenHandler generated token successfully | %s %s", req.Method, req.URL)
return nil
}

230
utils/registry/auth/tokenhandler.go Normal file
View File

@ -0,0 +1,230 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
token_util "github.com/vmware/harbor/service/token"
"github.com/vmware/harbor/utils/log"
registry_errors "github.com/vmware/harbor/utils/registry/errors"
)
type scope struct {
Type string
Name string
Actions []string
}
func (s *scope) string() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
type tokenGenerator func(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error)
// Implements interface Handler
type tokenHandler struct {
scope *scope
tg tokenGenerator
cache string // cached token
expiresIn int // The duration in seconds since the token was issued that it will remain valid
issuedAt *time.Time // The RFC3339-serialized UTC standard time at which a given token was issued
}
// Scheme returns the scheme that the handler can handle
func (t *tokenHandler) Scheme() string {
return "bearer"
}
// AuthorizeRequest will add authorization header which contains a token before the request is sent
func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
var scopes []*scope
var token string
hasFrom := false
from := req.URL.Query().Get("from")
if len(from) != 0 {
s := &scope{
Type: "repository",
Name: from,
Actions: []string{"pull"},
}
scopes = append(scopes, s)
// do not cache the token if "from" appears
hasFrom = true
}
scopes = append(scopes, t.scope)
expired := true
if t.expiresIn != 0 && t.issuedAt != nil {
expired = t.issuedAt.Add(time.Duration(t.expiresIn) * time.Second).Before(time.Now().UTC())
}
if expired || hasFrom {
scopeStrs := []string{}
for _, scope := range scopes {
scopeStrs = append(scopeStrs, scope.string())
}
to, expiresIn, issuedAt, err := t.tg(params["realm"], params["service"], scopeStrs)
if err != nil {
return err
}
token = to
if !hasFrom {
t.cache = token
t.expiresIn = expiresIn
t.issuedAt = issuedAt
log.Debug("add token to cache")
}
} else {
token = t.cache
log.Debug("get token from cache")
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
log.Debugf("add token to request: %s %s", req.Method, req.URL.String())
return nil
}
// Implements interface Handler
type standardTokenHandler struct {
tokenHandler
client *http.Client
credential Credential
}
// NewStandardTokenHandler returns a standard token handler. The handler will request a token
// from token server and add it to the origin request
// TODO deal with https
func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, scopeActions ...string) Handler {
handler := &standardTokenHandler{
client: &http.Client{
Transport: http.DefaultTransport,
},
credential: credential,
}
handler.scope = &scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
}
handler.tg = handler.generateToken
return handler
}
func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
u, err := url.Parse(realm)
if err != nil {
return
}
q := u.Query()
q.Add("service", service)
for _, scope := range scopes {
q.Add("scope", scope)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return
}
s.credential.AddAuthorization(r)
resp, err := s.client.Do(r)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode != http.StatusOK {
err = registry_errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
tk := struct {
Token string `json:"token"`
ExpiresIn int `json:"expires_in"`
IssuedAt string `json:"issued_at"`
}{}
if err = json.Unmarshal(b, &tk); err != nil {
return
}
token = tk.Token
expiresIn = tk.ExpiresIn
t, err := time.Parse(time.RFC3339, tk.IssuedAt)
if err != nil {
log.Errorf("error occurred while parsing issued_at: %v", err)
err = nil
} else {
issuedAt = &t
}
log.Debug("get token from token server")
return
}
// Implements interface Handler
type usernameTokenHandler struct {
tokenHandler
username string
}
// NewUsernameTokenHandler returns a handler which will generate a token according to
// the user's privileges
func NewUsernameTokenHandler(username string, scopeType, scopeName string, scopeActions ...string) Handler {
handler := &usernameTokenHandler{
username: username,
}
handler.scope = &scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
}
handler.tg = handler.generateToken
return handler
}
func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
token, expiresIn, issuedAt, err = token_util.GenTokenForUI(u.username, service, scopes)
log.Debug("get token by calling GenTokenForUI directly")
return
}

116
utils/registry/httpclient.go
View File

@ -1,116 +0,0 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"net/http"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
)
// NewClient returns a http.Client according to the handlers provided
func NewClient(handlers []auth.Handler) *http.Client {
transport := NewAuthTransport(http.DefaultTransport, handlers)
return &http.Client{
Transport: transport,
}
}
// NewClientStandardAuthHandlerEmbeded return a http.Client which will authorize the request
// according to the credential provided and send it again when encounters a 401 error
func NewClientStandardAuthHandlerEmbeded(credential auth.Credential) *http.Client {
handlers := []auth.Handler{}
tokenHandler := auth.NewStandardTokenHandler(credential)
handlers = append(handlers, tokenHandler)
return NewClient(handlers)
}
// NewClientUsernameAuthHandlerEmbeded return a http.Client which will authorize the request
// according to the user's privileges and send it again when encounters a 401 error
func NewClientUsernameAuthHandlerEmbeded(username string) *http.Client {
handlers := []auth.Handler{}
tokenHandler := auth.NewUsernameTokenHandler(username)
handlers = append(handlers, tokenHandler)
return NewClient(handlers)
}
type authTransport struct {
transport http.RoundTripper
handlers []auth.Handler
}
// NewAuthTransport wraps the AuthHandlers to be http.RounTripper
func NewAuthTransport(transport http.RoundTripper, handlers []auth.Handler) http.RoundTripper {
return &authTransport{
transport: transport,
handlers: handlers,
}
}
// RoundTrip ...
func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) {
originResp, originErr := a.transport.RoundTrip(req)
if originErr != nil {
return originResp, originErr
}
log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL)
if originResp.StatusCode != http.StatusUnauthorized {
return originResp, nil
}
challenges := auth.ParseChallengeFromResponse(originResp)
reqChanged := false
for _, challenge := range challenges {
scheme := challenge.Scheme
for _, handler := range a.handlers {
if scheme != handler.Schema() {
log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema())
continue
}
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return nil, err
}
reqChanged = true
}
}
if !reqChanged {
log.Warning("no handler match scheme")
return originResp, nil
}
resp, err := a.transport.RoundTrip(req)
if err == nil {
log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL)
}
return resp, err
}

286
utils/registry/registry.go
View File

@ -21,57 +21,68 @@ import (
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
"github.com/vmware/harbor/utils/registry/errors"
)
// Registry holds information of a registry entiry
// Registry holds information of a registry entity
type Registry struct {
Endpoint *url.URL
client *http.Client
ub *uRLBuilder
}
type uRLBuilder struct {
root *url.URL
}
// NewRegistry returns an instance of registry
func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
endpoint = strings.TrimRight(endpoint, "/")
var (
// ManifestVersion1 : schema 1
ManifestVersion1 = manifest.Versioned{
SchemaVersion: 1,
MediaType: schema1.MediaTypeManifest,
}
// ManifestVersion2 : schema 2
ManifestVersion2 = manifest.Versioned{
SchemaVersion: 2,
MediaType: schema2.MediaTypeManifest,
}
)
// New returns an instance of Registry
func New(endpoint string, client *http.Client) (*Registry, error) {
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
return &Registry{
registry := &Registry{
Endpoint: u,
client: client,
ub: &uRLBuilder{
root: u,
},
}, nil
}
log.Debugf("initialized a registry client: %s", endpoint)
return registry, nil
}
// NewRegistryWithUsername returns a Registry instance which will authorize the request
// according to the privileges of user
func NewRegistryWithUsername(endpoint, username string) (*Registry, error) {
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
client, err := newClient(endpoint, username, nil, "registry", "catalog", "*")
if err != nil {
return nil, err
}
registry := &Registry{
Endpoint: u,
client: client,
}
log.Debugf("initialized a registry client with username: %s %s", endpoint, username)
return registry, nil
}
// Catalog ...
func (r *Registry) Catalog() ([]string, error) {
repos := []string{}
req, err := http.NewRequest("GET", r.ub.buildCatalogURL(), nil)
req, err := http.NewRequest("GET", buildCatalogURL(r.Endpoint.String()), nil)
if err != nil {
return repos, err
}
@ -108,209 +119,34 @@ func (r *Registry) Catalog() ([]string, error) {
}
}
// ListTag ...
func (r *Registry) ListTag(name string) ([]string, error) {
tags := []string{}
req, err := http.NewRequest("GET", r.ub.buildTagListURL(name), nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return tags, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = tagsResp.Tags
return tags, nil
}
return tags, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
func buildCatalogURL(endpoint string) string {
return fmt.Sprintf("%s/v2/_catalog", endpoint)
}
// ManifestExist ...
func (r *Registry) ManifestExist(name, reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", r.ub.buildManifestURL(name, reference), nil)
func newClient(endpoint, username string, credential auth.Credential,
scopeType, scopeName string, scopeActions ...string) (*http.Client, error) {
endpoint = strings.TrimRight(endpoint, "/")
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return
return nil, err
}
// request Schema 2 manifest, if the registry does not support it,
// Schema 1 manifest will be returned
req.Header.Set(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
return
var handlers []auth.Handler
var handler auth.Handler
if credential != nil {
handler = auth.NewStandardTokenHandler(credential, scopeType, scopeName, scopeActions...)
} else {
handler = auth.NewUsernameTokenHandler(username, scopeType, scopeName, scopeActions...)
}
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
handlers = append(handlers, handler)
if resp.StatusCode == http.StatusNotFound {
return
}
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Registry) PullManifest(name, reference string, version manifest.Versioned) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", r.ub.buildManifestURL(name, reference), nil)
if err != nil {
return
}
// if the registry does not support schema 2, schema 1 manifest will be returned
req.Header.Set(http.CanonicalHeaderKey("Accept"), version.MediaType)
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Registry) DeleteManifest(name, digest string) error {
req, err := http.NewRequest("DELETE", r.ub.buildManifestURL(name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// DeleteTag ...
func (r *Registry) DeleteTag(name, tag string) error {
digest, exist, err := r.ManifestExist(name, tag)
if err != nil {
return err
}
if !exist {
return errors.Error{
StatusCode: http.StatusNotFound,
}
}
return r.DeleteManifest(name, digest)
}
// DeleteBlob ...
func (r *Registry) DeleteBlob(name, digest string) error {
req, err := http.NewRequest("DELETE", r.ub.buildBlobURL(name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
func (u *uRLBuilder) buildCatalogURL() string {
return fmt.Sprintf("%s/v2/_catalog", u.root.String())
}
func (u *uRLBuilder) buildTagListURL(name string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", u.root.String(), name)
}
func (u *uRLBuilder) buildManifestURL(name, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", u.root.String(), name, reference)
}
func (u *uRLBuilder) buildBlobURL(name, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", u.root.String(), name, reference)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
return &http.Client{
Transport: transport,
}, nil
}

505
utils/registry/repository.go Normal file
View File

@ -0,0 +1,505 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
"github.com/vmware/harbor/utils/registry/errors"
)
// Repository holds information of a repository entity
type Repository struct {
Name string
Endpoint *url.URL
client *http.Client
}
// TODO add agent to header of request, notifications need it
// NewRepository returns an instance of Repository
func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
return repository, nil
}
// NewRepositoryWithCredential returns a Repository instance which will authorize the request
// according to the credenttial
func NewRepositoryWithCredential(name, endpoint string, credential auth.Credential) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
client, err := newClient(endpoint, "", credential, "repository", name, "pull", "push")
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
log.Debugf("initialized a repository client with credential: %s %s", endpoint, name)
return repository, nil
}
// NewRepositoryWithUsername returns a Repository instance which will authorize the request
// according to the privileges of user
func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
client, err := newClient(endpoint, username, nil, "repository", name, "pull", "push")
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
log.Debugf("initialized a repository client with username: %s %s %s", endpoint, name, username)
return repository, nil
}
// ListTag ...
func (r *Repository) ListTag() ([]string, error) {
tags := []string{}
req, err := http.NewRequest("GET", buildTagListURL(r.Endpoint.String(), r.Name), nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return tags, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = tagsResp.Tags
return tags, nil
}
return tags, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// ManifestExist ...
func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest)
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
if resp.StatusCode == http.StatusNotFound {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
for _, mediaType := range acceptMediaTypes {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PushManifest ...
func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) {
req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference),
bytes.NewReader(payload))
if err != nil {
return
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusCreated {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Repository) DeleteManifest(digest string) error {
req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// DeleteTag ...
func (r *Repository) DeleteTag(tag string) error {
digest, exist, err := r.ManifestExist(tag)
if err != nil {
return err
}
if !exist {
return errors.Error{
StatusCode: http.StatusNotFound,
}
}
return r.DeleteManifest(digest)
}
// BlobExist ...
func (r *Repository) BlobExist(digest string) (bool, error) {
req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return false, err
}
resp, err := r.client.Do(req)
if err != nil {
return false, err
}
if resp.StatusCode == http.StatusOK {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
return false, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// PullBlob ...
func (r *Repository) PullBlob(digest string) (size int64, data []byte, err error) {
req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return
}
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
size, err = strconv.ParseInt(contengLength, 10, 64)
if err != nil {
return
}
data = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) {
req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil)
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusAccepted {
location = resp.Header.Get(http.CanonicalHeaderKey("Location"))
uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID"))
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data []byte) error {
req, err := http.NewRequest("PUT", buildMonolithicBlobUploadURL(location, digest), bytes.NewReader(data))
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusCreated {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// PushBlob ...
func (r *Repository) PushBlob(digest string, size int64, data []byte) error {
exist, err := r.BlobExist(digest)
if err != nil {
return err
}
if exist {
log.Infof("blob already exists, skip pushing: %s %s", r.Name, digest)
return nil
}
location, _, err := r.initiateBlobUpload(r.Name)
if err != nil {
return err
}
return r.monolithicBlobUpload(location, digest, size, data)
}
// DeleteBlob ...
func (r *Repository) DeleteBlob(digest string) error {
req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}
func buildTagListURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName)
}
func buildManifestURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference)
}
func buildBlobURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference)
}
func buildInitiateBlobUploadURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName)
}
func buildMonolithicBlobUploadURL(location, digest string) string {
return fmt.Sprintf("%s&digest=%s", location, digest)
}

59
utils/registry/transport.go Normal file
View File

@ -0,0 +1,59 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"net/http"
"github.com/vmware/harbor/utils/log"
)
// RequestModifier modifies request
type RequestModifier interface {
ModifyRequest(*http.Request) error
}
// Transport holds information about base transport and modifiers
type Transport struct {
transport http.RoundTripper
modifiers []RequestModifier
}
// NewTransport ...
func NewTransport(transport http.RoundTripper, modifiers []RequestModifier) *Transport {
return &Transport{
transport: transport,
modifiers: modifiers,
}
}
// RoundTrip ...
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
for _, modifier := range t.modifiers {
if err := modifier.ModifyRequest(req); err != nil {
return nil, err
}
}
resp, err := t.transport.RoundTrip(req)
if err != nil {
return nil, err
}
log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL.String())
return resp, err
}

4
views/segment/header-content.tpl
View File

@ -57,6 +57,10 @@
<li><a id="aChangePassword" href="/changePassword" target="_blank"><span class="glyphicon glyphicon-pencil"></span>&nbsp;&nbsp;{{i18n .Lang "change_password"}}</a></li>
<li role="separator" class="divider"></li>
{{ end }}
{{ if eq .IsLdapAdminUser true }}
<li><a id="aChangePassword" href="/changePassword" target="_blank"><span class="glyphicon glyphicon-pencil"></span>&nbsp;&nbsp;{{i18n .Lang "change_password"}}</a></li>
<li role="separator" class="divider"></li>
{{ end }}
{{ if eq .AuthMode "db_auth" }}
{{ if eq .IsAdmin true }}
<li><a id="aAddUser" href="/addUser" target="_blank"><span class="glyphicon glyphicon-plus"></span>&nbsp;&nbsp;{{i18n .Lang "add_user"}}</a></li>