Merge remote-tracking branch 'upstream/master'

Conflicts:
	api/repository.go
This commit is contained in:
Wenkai Yin 2016-05-05 15:27:37 +08:00
commit 5fdfa0bdbd
14 changed files with 281 additions and 149 deletions

View File

@ -1,14 +1,14 @@
language: go
go:
- 1.5.3
- 1.6.2
go_import_path: github.com/vmware/harbor
service:
- mysql
env: GO15VENDOREXPERIMENT=1 DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD=
env: DB_HOST=127.0.0.1 DB_PORT=3306 DB_USR=root DB_PWD=
install:
- sudo apt-get update && sudo apt-get install -y libldap2-dev

View File

@ -1,4 +1,4 @@
FROM golang:1.5.1
FROM golang:1.6.2
MAINTAINER jiangd@vmware.com
@ -11,7 +11,6 @@ COPY . /go/src/github.com/vmware/harbor
COPY ./vendor/golang.org /go/src/golang.org
WORKDIR /go/src/github.com/vmware/harbor/ui
ENV GO15VENDOREXPERIMENT 1
RUN go get -d github.com/docker/distribution \
&& go get -d github.com/docker/libtrust \
&& go get -d github.com/go-sql-driver/mysql \

View File

@ -181,8 +181,9 @@ func (ra *RepositoryAPI) GetTags() {
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
repoName := ra.GetString("repo_name")
tags, err := ra.registry.ListTag(repoName)
tags := []string{}
ts, err := rc.ListTag()
if err != nil {
e, ok := errors.ParseError(err)
if ok {

View File

@ -62,6 +62,14 @@ func GetAccessLogs(accessLog models.AccessLog) ([]models.AccessLog, error) {
sql += ` and u.username like ? `
queryParam = append(queryParam, accessLog.Username)
}
if accessLog.RepoName != "" {
sql += ` and a.repo_name = ? `
queryParam = append(queryParam, accessLog.RepoName)
}
if accessLog.RepoTag != "" {
sql += ` and a.repo_tag = ? `
queryParam = append(queryParam, accessLog.RepoTag)
}
if accessLog.Keywords != "" {
sql += ` and a.operation in ( `
keywordList := strings.Split(accessLog.Keywords, "/")

View File

@ -102,6 +102,8 @@ func clearUp(username string) {
const username string = "Tester01"
const projectName string = "test_project"
const repoTag string = "test1.1"
const repoTag2 string = "test1.2"
const SysAdmin int = 1
const projectAdmin int = 2
const developer int = 3
@ -419,6 +421,66 @@ func TestGetAccessLog(t *testing.T) {
}
}
func TestAddAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
accessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
RepoName: currentProject.Name + "/",
RepoTag: repoTag,
GUID: "N/A",
Operation: "create",
OpTime: time.Now(),
}
err = AddAccessLog(accessLog)
if err != nil {
t.Errorf("Error occurred in AddAccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
if len(accessLogList) != 1 {
t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList))
}
if accessLogList[0].RepoName != projectName+"/" {
t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName)
}
if accessLogList[0].RepoTag != repoTag {
t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag, accessLogList[0].RepoTag)
}
}
func TestAccessLog(t *testing.T) {
var err error
var accessLogList []models.AccessLog
accessLog := models.AccessLog{
UserID: currentUser.UserID,
ProjectID: currentProject.ProjectID,
RepoName: currentProject.Name + "/",
RepoTag: repoTag2,
Operation: "create",
}
err = AccessLog(currentUser.Username, currentProject.Name, currentProject.Name+"/", repoTag2, "create")
if err != nil {
t.Errorf("Error occurred in AccessLog: %v", err)
}
accessLogList, err = GetAccessLogs(accessLog)
if err != nil {
t.Errorf("Error occurred in GetAccessLog: %v", err)
}
if len(accessLogList) != 1 {
t.Errorf("The length of accesslog list should be 1, actual: %d", len(accessLogList))
}
if accessLogList[0].RepoName != projectName+"/" {
t.Errorf("The project name does not match, expected: %s, actual: %s", projectName+"/", accessLogList[0].RepoName)
}
if accessLogList[0].RepoTag != repoTag2 {
t.Errorf("The repo tag does not match, expected: %s, actual: %s", repoTag2, accessLogList[0].RepoTag)
}
}
func TestProjectExists(t *testing.T) {
var exists bool
var err error

View File

@ -109,7 +109,7 @@ paths:
tags:
- Products
responses:
200:
201:
description: Project created successfully.
400:
description: Unsatisfied with constraints of the project creation.
@ -383,6 +383,28 @@ paths:
description: User does not have permission of admin role.
500:
description: Unexpected internal errors.
post:
summary: Creates a new user account.
description: |
This endpoint is to create a user if the user does not already exist.
parameters:
- name: user
in: body
description: New created user.
required: true
schema:
$ref: '#/definitions/User'
tags:
- Products
responses:
201:
description: User created successfully.
400:
description: Unsatisfied with constraints of the user creation.
403:
description: User registration can only be used by admin role user when self-registration is off.
500:
description: Unexpected internal errors.
/users/{user_id}:
put:
summary: Update a registered user to change to be an administrator of Harbor.
@ -438,6 +460,37 @@ paths:
description: User ID does not exist.
500:
description: Unexpected internal errors.
/users/{user_id}/password:
put:
summary: Change the password on a user that already exists.
description: |
This endpoint is for user to update password. Users with the admin role can change any user's password. Guest users can change only their own password.
parameters:
- name: user_id
in: path
type: integer
format: int32
required: true
description: Registered user ID.
- name: password
in: body
description: Password to be updated.
required: true
schema:
$ref: '#/definitions/Password'
tags:
- Products
responses:
200:
description: Updated password successfully.
400:
description: Invalid user ID; Old password is blank; New password is blank.
401:
description: Old password is not correct.
403:
description: Guests can only change their own account.
500:
description: Unexpected internal errors.
/repositories:
get:
summary: Get repositories accompany with relevant project and repo name.
@ -640,6 +693,15 @@ definitions:
deleted:
type: integer
format: int32
Password:
type: object
properties:
old_password:
type: string
description: The user's existing password.
new_password:
type: string
description: New password for marking as to be updated.
AccessLog:
type: object
properties:

View File

@ -39,6 +39,7 @@ const (
// GetResourceActions ...
func GetResourceActions(scopes []string) []*token.ResourceActions {
log.Debugf("scopes: %+v", scopes)
var res []*token.ResourceActions
for _, s := range scopes {
if s == "" {
@ -59,6 +60,7 @@ func GetResourceActions(scopes []string) []*token.ResourceActions {
func FilterAccess(username string, authenticated bool, a *token.ResourceActions) {
if a.Type == "registry" && a.Name == "catalog" {
log.Infof("current access, type: %s, name:%s, actions:%v \n", a.Type, a.Name, a.Actions)
return
}
@ -108,7 +110,7 @@ func FilterAccess(username string, authenticated bool, a *token.ResourceActions)
}
// GenTokenForUI is for the UI process to call, so it won't establish a https connection from UI to proxy.
func GenTokenForUI(username string, service string, scopes []string) (string, error) {
func GenTokenForUI(username string, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
access := GetResourceActions(scopes)
for _, a := range access {
FilterAccess(username, true, a)
@ -117,22 +119,22 @@ func GenTokenForUI(username string, service string, scopes []string) (string, er
}
// MakeToken makes a valid jwt token based on parms.
func MakeToken(username, service string, access []*token.ResourceActions) (string, error) {
func MakeToken(username, service string, access []*token.ResourceActions) (token string, expiresIn int, issuedAt *time.Time, err error) {
pk, err := libtrust.LoadKeyFile(privateKey)
if err != nil {
return "", err
return "", 0, nil, err
}
tk, err := makeTokenCore(issuer, username, service, expiration, access, pk)
tk, expiresIn, issuedAt, err := makeTokenCore(issuer, username, service, expiration, access, pk)
if err != nil {
return "", err
return "", 0, nil, err
}
rs := fmt.Sprintf("%s.%s", tk.Raw, base64UrlEncode(tk.Signature))
return rs, nil
return rs, expiresIn, issuedAt, nil
}
//make token core
func makeTokenCore(issuer, subject, audience string, expiration int,
access []*token.ResourceActions, signingKey libtrust.PrivateKey) (*token.Token, error) {
access []*token.ResourceActions, signingKey libtrust.PrivateKey) (t *token.Token, expiresIn int, issuedAt *time.Time, err error) {
joseHeader := &token.Header{
Type: "JWT",
@ -142,10 +144,12 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
jwtID, err := randString(16)
if err != nil {
return nil, fmt.Errorf("Error to generate jwt id: %s", err)
return nil, 0, nil, fmt.Errorf("Error to generate jwt id: %s", err)
}
now := time.Now()
now := time.Now().UTC()
issuedAt = &now
expiresIn = expiration * 60
claimSet := &token.ClaimSet{
Issuer: issuer,
@ -161,10 +165,10 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
var joseHeaderBytes, claimSetBytes []byte
if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil {
return nil, fmt.Errorf("unable to marshal jose header: %s", err)
return nil, 0, nil, fmt.Errorf("unable to marshal jose header: %s", err)
}
if claimSetBytes, err = json.Marshal(claimSet); err != nil {
return nil, fmt.Errorf("unable to marshal claim set: %s", err)
return nil, 0, nil, fmt.Errorf("unable to marshal claim set: %s", err)
}
encodedJoseHeader := base64UrlEncode(joseHeaderBytes)
@ -173,12 +177,13 @@ func makeTokenCore(issuer, subject, audience string, expiration int,
var signatureBytes []byte
if signatureBytes, _, err = signingKey.Sign(strings.NewReader(payload), crypto.SHA256); err != nil {
return nil, fmt.Errorf("unable to sign jwt payload: %s", err)
return nil, 0, nil, fmt.Errorf("unable to sign jwt payload: %s", err)
}
signature := base64UrlEncode(signatureBytes)
tokenString := fmt.Sprintf("%s.%s", payload, signature)
return token.NewToken(tokenString)
t, err = token.NewToken(tokenString)
return
}
func randString(length int) (string, error) {

View File

@ -17,6 +17,7 @@ package token
import (
"net/http"
"time"
"github.com/vmware/harbor/auth"
"github.com/vmware/harbor/models"
@ -43,7 +44,6 @@ func (h *Handler) Get() {
authenticated := authenticate(username, password)
service := h.GetString("service")
scopes := h.GetStrings("scope")
log.Debugf("scopes: %+v", scopes)
if len(scopes) == 0 && !authenticated {
log.Info("login request with invalid credentials")
@ -59,14 +59,16 @@ func (h *Handler) Get() {
func (h *Handler) serveToken(username, service string, access []*token.ResourceActions) {
writer := h.Ctx.ResponseWriter
//create token
rawToken, err := MakeToken(username, service, access)
rawToken, expiresIn, issuedAt, err := MakeToken(username, service, access)
if err != nil {
log.Errorf("Failed to make token, error: %v", err)
writer.WriteHeader(http.StatusInternalServerError)
return
}
tk := make(map[string]string)
tk := make(map[string]interface{})
tk["token"] = rawToken
tk["expires_in"] = expiresIn
tk["issued_at"] = issuedAt.Format(time.RFC3339)
h.Data["json"] = tk
h.ServeJSON()
}

View File

@ -75,14 +75,14 @@ func RefreshCatalogCache() error {
rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username)
if err != nil {
log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err)
return err
continue
}
repositoryClients[repo] = rc
}
tags, err := rc.ListTag()
if err != nil {
log.Errorf("error occurred while list tag for %s: %v", repo, err)
return err
continue
}
if len(tags) != 0 {

View File

@ -23,13 +23,14 @@ import (
// Handler authorizes requests according to the schema
type Handler interface {
// Schema : basic, bearer
Schema() string
// Scheme : basic, bearer
Scheme() string
//AuthorizeRequest adds basic auth or token auth to the header of request
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// RequestAuthorizer holds a handler list, which will authorize request
// RequestAuthorizer holds a handler list, which will authorize request.
// Implements interface RequestModifier
type RequestAuthorizer struct {
handlers []Handler
challenges []au.Challenge
@ -47,7 +48,7 @@ func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *Reques
func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error {
for _, handler := range r.handlers {
for _, challenge := range r.challenges {
if handler.Schema() == challenge.Scheme {
if handler.Scheme() == challenge.Scheme {
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return err
}

View File

@ -25,6 +25,7 @@ type Credential interface {
AddAuthorization(req *http.Request)
}
// Implements interface Credential
type basicAuthCredential struct {
username string
password string

View File

@ -21,6 +21,7 @@ import (
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"time"
@ -39,61 +40,68 @@ func (s *scope) string() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
type token struct {
token string
expiresIn time.Time
}
type tokenGenerator func(realm, service string, scopes []string) (*token, error)
type tokenGenerator func(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error)
// Implements interface Handler
type tokenHandler struct {
scope *scope
cache map[string]*token
tg tokenGenerator
cache string // cached token
expiresIn int // The duration in seconds since the token was issued that it will remain valid
issuedAt *time.Time // The RFC3339-serialized UTC standard time at which a given token was issued
}
// Schema returns the schema that the handler can handle
func (t *tokenHandler) Schema() string {
// Scheme returns the scheme that the handler can handle
func (t *tokenHandler) Scheme() string {
return "bearer"
}
// AuthorizeRequest will add authorization header which contains a token before the request is sent
func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
var token string
var scopes []*scope
var token string
// TODO handle additional scope: xxx.xxx.xxx?from=repo
hasFrom := false
from := req.URL.Query().Get("from")
if len(from) != 0 {
s := &scope{
Type: "repository",
Name: from,
Actions: []string{"pull"},
}
scopes = append(scopes, s)
// do not cache the token if "from" appears
hasFrom = true
}
scopes = append(scopes, t.scope)
key := cacheKey(scopes)
value, ok := t.cache[key]
var expired bool
expired := true
if ok {
expired = value.expiresIn.Before(time.Now())
}
if ok && !expired {
token = value.token
log.Debugf("get token from cache: %s", key)
} else {
if ok && expired {
delete(t.cache, key)
log.Debugf("token is expired, remove from cache: %s", key)
if t.expiresIn != 0 && t.issuedAt != nil {
expired = t.issuedAt.Add(time.Duration(t.expiresIn) * time.Second).Before(time.Now().UTC())
}
if expired || hasFrom {
scopeStrs := []string{}
for _, scope := range scopes {
scopeStrs = append(scopeStrs, scope.string())
}
tk, err := t.tg(params["realm"], params["service"], scopeStrs)
to, expiresIn, issuedAt, err := t.tg(params["realm"], params["service"], scopeStrs)
if err != nil {
return err
}
token = tk.token
t.cache[key] = tk
log.Debugf("add token to cache: %s", key)
token = to
if !hasFrom {
t.cache = token
t.expiresIn = expiresIn
t.issuedAt = issuedAt
log.Debug("add token to cache")
}
} else {
token = t.cache
log.Debug("get token from cache")
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
@ -102,17 +110,7 @@ func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]str
return nil
}
// cacheKey returns a string which can identify the scope array and can be used as the key in cache map
func cacheKey(scopes []*scope) string {
key := ""
for _, scope := range scopes {
key = key + scope.string() + "|"
}
key = strings.TrimRight(key, "|")
return key
}
// Implements interface Handler
type standardTokenHandler struct {
tokenHandler
client *http.Client
@ -135,16 +133,15 @@ func NewStandardTokenHandler(credential Credential, scopeType, scopeName string,
Name: scopeName,
Actions: scopeActions,
}
handler.cache = make(map[string]*token, 1)
handler.tg = handler.generateToken
return handler
}
func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) {
func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
u, err := url.Parse(realm)
if err != nil {
return nil, err
return
}
q := u.Query()
q.Add("service", service)
@ -154,46 +151,61 @@ func (s *standardTokenHandler) generateToken(realm, service string, scopes []str
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
return
}
s.credential.AddAuthorization(r)
resp, err := s.client.Do(r)
if err != nil {
return nil, err
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
return
}
if resp.StatusCode != http.StatusOK {
return nil, registry_errors.Error{
err = registry_errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
tk := struct {
Token string `json:"token"`
ExpiresIn string `json:"expires_in"`
IssuedAt string `json:"issued_at"`
}{}
if err = json.Unmarshal(b, &tk); err != nil {
return nil, err
return
}
t := &token{
token: tk.Token,
// TODO handle the expires time
expiresIn: time.Now().Add(5 * time.Minute),
token = tk.Token
expiresIn, err = strconv.Atoi(tk.ExpiresIn)
if err != nil {
expiresIn = 0
log.Errorf("error occurred while converting expires_in: %v", err)
err = nil
} else {
t, err := time.Parse(time.RFC3339, tk.IssuedAt)
if err != nil {
log.Errorf("error occurred while parsing issued_at: %v", err)
err = nil
} else {
issuedAt = &t
}
}
log.Debug("get token from token server")
return t, nil
return
}
// Implements interface Handler
type usernameTokenHandler struct {
tokenHandler
username string
@ -211,26 +223,14 @@ func NewUsernameTokenHandler(username string, scopeType, scopeName string, scope
Name: scopeName,
Actions: scopeActions,
}
handler.cache = make(map[string]*token, 1)
handler.tg = handler.generateToken
return handler
}
func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) {
tk, err := token_util.GenTokenForUI(u.username, service, scopes)
if err != nil {
return nil, err
}
t := &token{
token: tk,
// TODO handle the expires time
expiresIn: time.Now().Add(5 * time.Minute),
}
func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
token, expiresIn, issuedAt, err = token_util.GenTokenForUI(u.username, service, scopes)
log.Debug("get token by calling GenTokenForUI directly")
return t, nil
return
}

View File

@ -63,27 +63,18 @@ func NewRegistryWithUsername(endpoint, username string) (*Registry, error) {
return nil, err
}
resp, err := http.Get(buildPingURL(endpoint))
client, err := newClient(endpoint, username, nil, "registry", "catalog", "*")
if err != nil {
return nil, err
}
var handlers []auth.Handler
handler := auth.NewUsernameTokenHandler(username, "registry", "catalog", "*")
handlers = append(handlers, handler)
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
registry := &Registry{
Endpoint: u,
client: &http.Client{
Transport: transport,
},
client: client,
}
log.Debugf("initialized a registry client with username: %s %s", endpoint, username)
return registry, nil
}
@ -131,3 +122,31 @@ func (r *Registry) Catalog() ([]string, error) {
func buildCatalogURL(endpoint string) string {
return fmt.Sprintf("%s/v2/_catalog", endpoint)
}
func newClient(endpoint, username string, credential auth.Credential,
scopeType, scopeName string, scopeActions ...string) (*http.Client, error) {
endpoint = strings.TrimRight(endpoint, "/")
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return nil, err
}
var handlers []auth.Handler
var handler auth.Handler
if credential != nil {
handler = auth.NewStandardTokenHandler(credential, scopeType, scopeName, scopeActions...)
} else {
handler = auth.NewUsernameTokenHandler(username, scopeType, scopeName, scopeActions...)
}
handlers = append(handlers, handler)
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
return &http.Client{
Transport: transport,
}, nil
}

View File

@ -71,26 +71,12 @@ func NewRepositoryWithCredential(name, endpoint string, credential auth.Credenti
return nil, err
}
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return nil, err
}
var handlers []auth.Handler
handler := auth.NewStandardTokenHandler(credential, "repository", name, "pull", "push")
handlers = append(handlers, handler)
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
client, err := newClient(endpoint, "", credential, "repository", name, "pull", "push")
repository := &Repository{
Name: name,
Endpoint: u,
client: &http.Client{
Transport: transport,
},
client: client,
}
log.Debugf("initialized a repository client with credential: %s %s", endpoint, name)
@ -109,29 +95,15 @@ func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, er
return nil, err
}
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return nil, err
}
var handlers []auth.Handler
handler := auth.NewUsernameTokenHandler(username, "repository", name, "pull", "push")
handlers = append(handlers, handler)
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
client, err := newClient(endpoint, username, nil, "repository", name, "pull", "push")
repository := &Repository{
Name: name,
Endpoint: u,
client: &http.Client{
Transport: transport,
},
client: client,
}
log.Debugf("initialized a repository client with username: %s %s", endpoint, name)
log.Debugf("initialized a repository client with username: %s %s", endpoint, name, username)
return repository, nil
}