Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-07 11:10:16 +01:00
Code Issues Projects Releases Wiki Activity

Merge branch 'registry_api_util_update'

Browse Source
This commit is contained in:
Wenkai Yin 2016-04-28 10:55:31 +08:00
commit 5cc51a753d
10 changed files with 1079 additions and 613 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
api/repository.go
View File

@ -23,6 +23,7 @@ import (
"strings"
"time"
"github.com/docker/distribution/manifest/schema1"
"github.com/vmware/harbor/dao"
"github.com/vmware/harbor/models"
svc_utils "github.com/vmware/harbor/service/utils"
@ -40,61 +41,20 @@ type RepositoryAPI struct {
BaseAPI
userID int
username string
registry *registry.Registry
}
// Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission.
func (ra *RepositoryAPI) Prepare() {
userID, ok := ra.GetSession("userId").(int)
if !ok {
ra.userID = dao.NonExistUserID
} else {
ra.userID = userID
userID = dao.NonExistUserID
}
ra.userID = userID
username, ok := ra.GetSession("username").(string)
if !ok {
log.Warning("failed to get username from session")
ra.username = ""
} else {
if ok {
ra.username = username
}
var client *http.Client
//no session, initialize a standard auth handler
if ra.userID == dao.NonExistUserID && len(ra.username) == 0 {
username, password, _ := ra.Ctx.Request.BasicAuth()
credential := auth.NewBasicAuthCredential(username, password)
client = registry.NewClientStandardAuthHandlerEmbeded(credential)
log.Debug("initializing standard auth handler")
} else {
// session works, initialize a username auth handler
username := ra.username
if len(username) == 0 {
user, err := dao.GetUser(models.User{
UserID: ra.userID,
})
if err != nil {
log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err)
return
}
username = user.Username
}
client = registry.NewClientUsernameAuthHandlerEmbeded(username)
log.Debug("initializing username auth handler: %s", username)
}
endpoint := os.Getenv("REGISTRY_URL")
r, err := registry.New(endpoint, client)
if err != nil {
log.Fatalf("error occurred while initializing auth handler for repository API: %v", err)
}
ra.registry = r
}
// Get ...
@ -156,10 +116,16 @@ func (ra *RepositoryAPI) Delete() {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags := []string{}
tag := ra.GetString("tag")
if len(tag) == 0 {
tagList, err := ra.registry.ListTag(repoName)
tagList, err := rc.ListTag()
if err != nil {
e, ok := errors.ParseError(err)
if ok {
@ -169,16 +135,14 @@ func (ra *RepositoryAPI) Delete() {
log.Error(err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
}
tags = append(tags, tagList...)
} else {
tags = append(tags, tag)
}
for _, t := range tags {
if err := ra.registry.DeleteTag(repoName, t); err != nil {
if err := rc.DeleteTag(t); err != nil {
e, ok := errors.ParseError(err)
if ok {
ra.CustomAbort(e.StatusCode, e.Message)
@ -218,16 +182,23 @@ type manifest struct {
// GetTags handles GET /api/repositories/tags
func (ra *RepositoryAPI) GetTags() {
var tags []string
repoName := ra.GetString("repo_name")
if len(repoName) == 0 {
ra.CustomAbort(http.StatusBadRequest, "repo_name is nil")
}
tags, err := ra.registry.ListTag(repoName)
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags := []string{}
ts, err := rc.ListTag()
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
@ -235,6 +206,8 @@ func (ra *RepositoryAPI) GetTags() {
}
}
tags = append(tags, ts...)
ra.Data["json"] = tags
ra.ServeJSON()
}
@ -244,13 +217,23 @@ func (ra *RepositoryAPI) GetManifests() {
repoName := ra.GetString("repo_name")
tag := ra.GetString("tag")
if len(repoName) == 0 || len(tag) == 0 {
ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil")
}
rc, err := ra.initializeRepositoryClient(repoName)
if err != nil {
log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err)
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
item := models.RepoItem{}
_, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1)
mediaTypes := []string{schema1.MediaTypeManifest}
_, _, payload, err := rc.PullManifest(tag, mediaTypes)
if err != nil {
e, ok := errors.ParseError(err)
if ok {
log.Info(e)
ra.CustomAbort(e.StatusCode, e.Message)
} else {
log.Error(err)
@ -278,3 +261,31 @@ func (ra *RepositoryAPI) GetManifests() {
ra.Data["json"] = item
ra.ServeJSON()
}
func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) {
endpoint := os.Getenv("REGISTRY_URL")
//no session, use basic auth
if ra.userID == dao.NonExistUserID {
username, password, _ := ra.Ctx.Request.BasicAuth()
credential := auth.NewBasicAuthCredential(username, password)
return registry.NewRepositoryWithCredential(repoName, endpoint, credential)
}
//session exists, use username
if len(ra.username) == 0 {
u := models.User{
UserID: ra.userID,
}
user, err := dao.GetUser(u)
if err != nil {
return nil, err
}
ra.username = user.Username
}
return registry.NewRepositoryWithUsername(repoName, endpoint, ra.username)
}

43
service/utils/cache.go
View File

@ -25,10 +25,14 @@ import (
"github.com/astaxie/beego/cache"
)
// Cache is the global cache in system.
var Cache cache.Cache
var registryClient *registry.Registry
var (
// Cache is the global cache in system.
Cache cache.Cache
endpoint string
username string
registryClient *registry.Registry
repositoryClients map[string]*registry.Repository
)
const catalogKey string = "catalog"
@ -39,17 +43,25 @@ func init() {
log.Errorf("Failed to initialize cache, error:%v", err)
}
endpoint := os.Getenv("REGISTRY_URL")
client := registry.NewClientUsernameAuthHandlerEmbeded("admin")
registryClient, err = registry.New(endpoint, client)
if err != nil {
log.Fatalf("error occurred while initializing authentication handler used by cache: %v", err)
}
endpoint = os.Getenv("REGISTRY_URL")
username = "admin"
repositoryClients = make(map[string]*registry.Repository, 10)
}
// RefreshCatalogCache calls registry's API to get repository list and write it to cache.
func RefreshCatalogCache() error {
log.Debug("refreshing catalog cache...")
if registryClient == nil {
var err error
registryClient, err = registry.NewRegistryWithUsername(endpoint, username)
if err != nil {
log.Errorf("error occurred while initializing registry client used by cache: %v", err)
return err
}
}
var err error
rs, err := registryClient.Catalog()
if err != nil {
return err
@ -58,7 +70,16 @@ func RefreshCatalogCache() error {
repos := []string{}
for _, repo := range rs {
tags, err := registryClient.ListTag(repo)
rc, ok := repositoryClients[repo]
if !ok {
rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username)
if err != nil {
log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err)
return err
}
repositoryClients[repo] = rc
}
tags, err := rc.ListTag()
if err != nil {
log.Errorf("error occurred while list tag for %s: %v", repo, err)
return err

59
utils/registry/auth/authorizer.go Normal file
View File

@ -0,0 +1,59 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"net/http"
au "github.com/docker/distribution/registry/client/auth"
)
// Handler authorizes requests according to the schema
type Handler interface {
// Schema : basic, bearer
Schema() string
//AuthorizeRequest adds basic auth or token auth to the header of request
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// RequestAuthorizer holds a handler list, which will authorize request
type RequestAuthorizer struct {
handlers []Handler
challenges []au.Challenge
}
// NewRequestAuthorizer ...
func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *RequestAuthorizer {
return &RequestAuthorizer{
handlers: handlers,
challenges: challenges,
}
}
// ModifyRequest adds authorization to the request
func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error {
for _, handler := range r.handlers {
for _, challenge := range r.challenges {
if handler.Schema() == challenge.Scheme {
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return err
}
}
}
}
return nil
}

43
utils/registry/auth/credential.go Normal file
View File

@ -0,0 +1,43 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"net/http"
)
// Credential ...
type Credential interface {
// AddAuthorization adds authorization information to request
AddAuthorization(req *http.Request)
}
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}

197
utils/registry/auth/handler.go
View File

@ -1,197 +0,0 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
token_util "github.com/vmware/harbor/service/token"
"github.com/vmware/harbor/utils/log"
registry_errors "github.com/vmware/harbor/utils/registry/errors"
)
const (
// credential type
basicAuth string = "basic_auth"
secretKey string = "secret_key"
)
// Handler authorizes the request when encounters a 401 error
type Handler interface {
// Schema : basic, bearer
Schema() string
//AuthorizeRequest adds basic auth or token auth to the header of request
AuthorizeRequest(req *http.Request, params map[string]string) error
}
// Credential ...
type Credential interface {
// AddAuthorization adds authorization information to request
AddAuthorization(req *http.Request)
}
type basicAuthCredential struct {
username string
password string
}
// NewBasicAuthCredential ...
func NewBasicAuthCredential(username, password string) Credential {
return &basicAuthCredential{
username: username,
password: password,
}
}
func (b *basicAuthCredential) AddAuthorization(req *http.Request) {
req.SetBasicAuth(b.username, b.password)
}
type token struct {
Token string `json:"token"`
}
type standardTokenHandler struct {
client *http.Client
credential Credential
}
// NewStandardTokenHandler returns a standard token handler. The handler will request a token
// from token server whose URL is specified in the "WWW-authentication" header and add it to
// the origin request
// TODO deal with https
func NewStandardTokenHandler(credential Credential) Handler {
return &standardTokenHandler{
client: &http.Client{
Transport: http.DefaultTransport,
},
credential: credential,
}
}
// Schema implements the corresponding method in interface AuthHandler
func (t *standardTokenHandler) Schema() string {
return "bearer"
}
// AuthorizeRequest implements the corresponding method in interface AuthHandler
func (t *standardTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
realm, ok := params["realm"]
if !ok {
return errors.New("no realm")
}
service := params["service"]
scope := params["scope"]
u, err := url.Parse(realm)
if err != nil {
return err
}
q := u.Query()
q.Add("service", service)
for _, s := range strings.Split(scope, " ") {
q.Add("scope", s)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return err
}
t.credential.AddAuthorization(r)
resp, err := t.client.Do(r)
if err != nil {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return registry_errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
decoder := json.NewDecoder(resp.Body)
tk := &token{}
if err = decoder.Decode(tk); err != nil {
return err
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk.Token))
log.Debugf("standardTokenHandler generated token successfully | %s %s", req.Method, req.URL)
return nil
}
type usernameTokenHandler struct {
username string
}
// NewUsernameTokenHandler returns a handler which will generate
// a token according the user's privileges
func NewUsernameTokenHandler(username string) Handler {
return &usernameTokenHandler{
username: username,
}
}
// Schema implements the corresponding method in interface AuthHandler
func (u *usernameTokenHandler) Schema() string {
return "bearer"
}
// AuthorizeRequest implements the corresponding method in interface AuthHandler
func (u *usernameTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
service := params["service"]
scopes := []string{}
scope := params["scope"]
if len(scope) != 0 {
scopes = strings.Split(scope, " ")
}
token, err := token_util.GenTokenForUI(u.username, service, scopes)
if err != nil {
return err
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
log.Debugf("usernameTokenHandler generated token successfully | %s %s", req.Method, req.URL)
return nil
}

236
utils/registry/auth/tokenhandler.go Normal file
View File

@ -0,0 +1,236 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
token_util "github.com/vmware/harbor/service/token"
"github.com/vmware/harbor/utils/log"
registry_errors "github.com/vmware/harbor/utils/registry/errors"
)
type scope struct {
Type string
Name string
Actions []string
}
func (s *scope) string() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
type token struct {
token string
expiresIn time.Time
}
type tokenGenerator func(realm, service string, scopes []string) (*token, error)
type tokenHandler struct {
scope *scope
cache map[string]*token
tg tokenGenerator
}
// Schema returns the schema that the handler can handle
func (t *tokenHandler) Schema() string {
return "bearer"
}
// AuthorizeRequest will add authorization header which contains a token before the request is sent
func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
var token string
var scopes []*scope
// TODO handle additional scope: xxx.xxx.xxx?from=repo
scopes = append(scopes, t.scope)
key := cacheKey(scopes)
value, ok := t.cache[key]
var expired bool
if ok {
expired = value.expiresIn.Before(time.Now())
}
if ok && !expired {
token = value.token
log.Debugf("get token from cache: %s", key)
} else {
if ok && expired {
delete(t.cache, key)
log.Debugf("token is expired, remove from cache: %s", key)
}
scopeStrs := []string{}
for _, scope := range scopes {
scopeStrs = append(scopeStrs, scope.string())
}
tk, err := t.tg(params["realm"], params["service"], scopeStrs)
if err != nil {
return err
}
token = tk.token
t.cache[key] = tk
log.Debugf("add token to cache: %s", key)
}
req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
log.Debugf("add token to request: %s %s", req.Method, req.URL.String())
return nil
}
// cacheKey returns a string which can identify the scope array and can be used as the key in cache map
func cacheKey(scopes []*scope) string {
key := ""
for _, scope := range scopes {
key = key + scope.string() + "|"
}
key = strings.TrimRight(key, "|")
return key
}
type standardTokenHandler struct {
tokenHandler
client *http.Client
credential Credential
}
// NewStandardTokenHandler returns a standard token handler. The handler will request a token
// from token server and add it to the origin request
// TODO deal with https
func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, scopeActions ...string) Handler {
handler := &standardTokenHandler{
client: &http.Client{
Transport: http.DefaultTransport,
},
credential: credential,
}
handler.scope = &scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
}
handler.cache = make(map[string]*token, 1)
handler.tg = handler.generateToken
return handler
}
func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) {
u, err := url.Parse(realm)
if err != nil {
return nil, err
}
q := u.Query()
q.Add("service", service)
for _, scope := range scopes {
q.Add("scope", scope)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return nil, err
}
s.credential.AddAuthorization(r)
resp, err := s.client.Do(r)
if err != nil {
return nil, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, registry_errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
tk := struct {
Token string `json:"token"`
}{}
if err = json.Unmarshal(b, &tk); err != nil {
return nil, err
}
t := &token{
token: tk.Token,
// TODO handle the expires time
expiresIn: time.Now().Add(5 * time.Minute),
}
log.Debug("get token from token server")
return t, nil
}
type usernameTokenHandler struct {
tokenHandler
username string
}
// NewUsernameTokenHandler returns a handler which will generate a token according to
// the user's privileges
func NewUsernameTokenHandler(username string, scopeType, scopeName string, scopeActions ...string) Handler {
handler := &usernameTokenHandler{
username: username,
}
handler.scope = &scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
}
handler.cache = make(map[string]*token, 1)
handler.tg = handler.generateToken
return handler
}
func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) {
tk, err := token_util.GenTokenForUI(u.username, service, scopes)
if err != nil {
return nil, err
}
t := &token{
token: tk,
// TODO handle the expires time
expiresIn: time.Now().Add(5 * time.Minute),
}
log.Debug("get token by calling GenTokenForUI directly")
return t, nil
}

116
utils/registry/httpclient.go
View File

@ -1,116 +0,0 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"net/http"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
)
// NewClient returns a http.Client according to the handlers provided
func NewClient(handlers []auth.Handler) *http.Client {
transport := NewAuthTransport(http.DefaultTransport, handlers)
return &http.Client{
Transport: transport,
}
}
// NewClientStandardAuthHandlerEmbeded return a http.Client which will authorize the request
// according to the credential provided and send it again when encounters a 401 error
func NewClientStandardAuthHandlerEmbeded(credential auth.Credential) *http.Client {
handlers := []auth.Handler{}
tokenHandler := auth.NewStandardTokenHandler(credential)
handlers = append(handlers, tokenHandler)
return NewClient(handlers)
}
// NewClientUsernameAuthHandlerEmbeded return a http.Client which will authorize the request
// according to the user's privileges and send it again when encounters a 401 error
func NewClientUsernameAuthHandlerEmbeded(username string) *http.Client {
handlers := []auth.Handler{}
tokenHandler := auth.NewUsernameTokenHandler(username)
handlers = append(handlers, tokenHandler)
return NewClient(handlers)
}
type authTransport struct {
transport http.RoundTripper
handlers []auth.Handler
}
// NewAuthTransport wraps the AuthHandlers to be http.RounTripper
func NewAuthTransport(transport http.RoundTripper, handlers []auth.Handler) http.RoundTripper {
return &authTransport{
transport: transport,
handlers: handlers,
}
}
// RoundTrip ...
func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) {
originResp, originErr := a.transport.RoundTrip(req)
if originErr != nil {
return originResp, originErr
}
log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL)
if originResp.StatusCode != http.StatusUnauthorized {
return originResp, nil
}
challenges := auth.ParseChallengeFromResponse(originResp)
reqChanged := false
for _, challenge := range challenges {
scheme := challenge.Scheme
for _, handler := range a.handlers {
if scheme != handler.Schema() {
log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema())
continue
}
if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
return nil, err
}
reqChanged = true
}
}
if !reqChanged {
log.Warning("no handler match scheme")
return originResp, nil
}
resp, err := a.transport.RoundTrip(req)
if err == nil {
log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL)
}
return resp, err
}

283
utils/registry/registry.go
View File

@ -21,57 +21,77 @@ import (
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
"github.com/vmware/harbor/utils/registry/errors"
)
// Registry holds information of a registry entiry
// Registry holds information of a registry entity
type Registry struct {
Endpoint *url.URL
client *http.Client
ub *uRLBuilder
}
type uRLBuilder struct {
root *url.URL
}
// NewRegistry returns an instance of registry
func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
endpoint = strings.TrimRight(endpoint, "/")
var (
// ManifestVersion1 : schema 1
ManifestVersion1 = manifest.Versioned{
SchemaVersion: 1,
MediaType: schema1.MediaTypeManifest,
}
// ManifestVersion2 : schema 2
ManifestVersion2 = manifest.Versioned{
SchemaVersion: 2,
MediaType: schema2.MediaTypeManifest,
}
)
// New returns an instance of Registry
func New(endpoint string, client *http.Client) (*Registry, error) {
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
return &Registry{
registry := &Registry{
Endpoint: u,
client: client,
ub: &uRLBuilder{
root: u,
}
log.Debugf("initialized a registry client: %s", endpoint)
return registry, nil
}
// NewRegistryWithUsername returns a Registry instance which will authorize the request
// according to the privileges of user
func NewRegistryWithUsername(endpoint, username string) (*Registry, error) {
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return nil, err
}
var handlers []auth.Handler
handler := auth.NewUsernameTokenHandler(username, "registry", "catalog", "*")
handlers = append(handlers, handler)
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
registry := &Registry{
Endpoint: u,
client: &http.Client{
Transport: transport,
},
}, nil
}
return registry, nil
}
// Catalog ...
func (r *Registry) Catalog() ([]string, error) {
repos := []string{}
req, err := http.NewRequest("GET", r.ub.buildCatalogURL(), nil)
req, err := http.NewRequest("GET", buildCatalogURL(r.Endpoint.String()), nil)
if err != nil {
return repos, err
}
@ -108,209 +128,6 @@ func (r *Registry) Catalog() ([]string, error) {
}
}
// ListTag ...
func (r *Registry) ListTag(name string) ([]string, error) {
tags := []string{}
req, err := http.NewRequest("GET", r.ub.buildTagListURL(name), nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return tags, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = tagsResp.Tags
return tags, nil
}
return tags, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// ManifestExist ...
func (r *Registry) ManifestExist(name, reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", r.ub.buildManifestURL(name, reference), nil)
if err != nil {
return
}
// request Schema 2 manifest, if the registry does not support it,
// Schema 1 manifest will be returned
req.Header.Set(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
if resp.StatusCode == http.StatusNotFound {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Registry) PullManifest(name, reference string, version manifest.Versioned) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", r.ub.buildManifestURL(name, reference), nil)
if err != nil {
return
}
// if the registry does not support schema 2, schema 1 manifest will be returned
req.Header.Set(http.CanonicalHeaderKey("Accept"), version.MediaType)
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Registry) DeleteManifest(name, digest string) error {
req, err := http.NewRequest("DELETE", r.ub.buildManifestURL(name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// DeleteTag ...
func (r *Registry) DeleteTag(name, tag string) error {
digest, exist, err := r.ManifestExist(name, tag)
if err != nil {
return err
}
if !exist {
return errors.Error{
StatusCode: http.StatusNotFound,
}
}
return r.DeleteManifest(name, digest)
}
// DeleteBlob ...
func (r *Registry) DeleteBlob(name, digest string) error {
req, err := http.NewRequest("DELETE", r.ub.buildBlobURL(name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
func (u *uRLBuilder) buildCatalogURL() string {
return fmt.Sprintf("%s/v2/_catalog", u.root.String())
}
func (u *uRLBuilder) buildTagListURL(name string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", u.root.String(), name)
}
func (u *uRLBuilder) buildManifestURL(name, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", u.root.String(), name, reference)
}
func (u *uRLBuilder) buildBlobURL(name, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", u.root.String(), name, reference)
func buildCatalogURL(endpoint string) string {
return fmt.Sprintf("%s/v2/_catalog", endpoint)
}

533
utils/registry/repository.go Normal file
View File

@ -0,0 +1,533 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/utils/log"
"github.com/vmware/harbor/utils/registry/auth"
"github.com/vmware/harbor/utils/registry/errors"
)
// Repository holds information of a repository entity
type Repository struct {
Name string
Endpoint *url.URL
client *http.Client
}
// TODO add agent to header of request, notifications need it
// NewRepository returns an instance of Repository
func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
repository := &Repository{
Name: name,
Endpoint: u,
client: client,
}
return repository, nil
}
// NewRepositoryWithCredential returns a Repository instance which will authorize the request
// according to the credenttial
func NewRepositoryWithCredential(name, endpoint string, credential auth.Credential) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return nil, err
}
var handlers []auth.Handler
handler := auth.NewStandardTokenHandler(credential, "repository", name, "pull", "push")
handlers = append(handlers, handler)
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
repository := &Repository{
Name: name,
Endpoint: u,
client: &http.Client{
Transport: transport,
},
}
log.Debugf("initialized a repository client with credential: %s %s", endpoint, name)
return repository, nil
}
// NewRepositoryWithUsername returns a Repository instance which will authorize the request
// according to the privileges of user
func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, error) {
name = strings.TrimSpace(name)
endpoint = strings.TrimRight(endpoint, "/")
u, err := url.Parse(endpoint)
if err != nil {
return nil, err
}
resp, err := http.Get(buildPingURL(endpoint))
if err != nil {
return nil, err
}
var handlers []auth.Handler
handler := auth.NewUsernameTokenHandler(username, "repository", name, "pull", "push")
handlers = append(handlers, handler)
challenges := auth.ParseChallengeFromResponse(resp)
authorizer := auth.NewRequestAuthorizer(handlers, challenges)
transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer})
repository := &Repository{
Name: name,
Endpoint: u,
client: &http.Client{
Transport: transport,
},
}
log.Debugf("initialized a repository client with username: %s %s", endpoint, name)
return repository, nil
}
// ListTag ...
func (r *Repository) ListTag() ([]string, error) {
tags := []string{}
req, err := http.NewRequest("GET", buildTagListURL(r.Endpoint.String(), r.Name), nil)
if err != nil {
return tags, err
}
resp, err := r.client.Do(req)
if err != nil {
return tags, err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return tags, err
}
if resp.StatusCode == http.StatusOK {
tagsResp := struct {
Tags []string `json:"tags"`
}{}
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
tags = tagsResp.Tags
return tags, nil
}
return tags, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// ManifestExist ...
func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) {
req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest)
req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest)
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
exist = true
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
if resp.StatusCode == http.StatusNotFound {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PullManifest ...
func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) {
req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil)
if err != nil {
return
}
for _, mediaType := range acceptMediaTypes {
req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType)
}
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type"))
payload = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// PushManifest ...
func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) {
req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference),
bytes.NewReader(payload))
if err != nil {
return
}
req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType)
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusCreated {
digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
// DeleteManifest ...
func (r *Repository) DeleteManifest(digest string) error {
req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// DeleteTag ...
func (r *Repository) DeleteTag(tag string) error {
digest, exist, err := r.ManifestExist(tag)
if err != nil {
return err
}
if !exist {
return errors.Error{
StatusCode: http.StatusNotFound,
}
}
return r.DeleteManifest(digest)
}
// BlobExist ...
func (r *Repository) BlobExist(digest string) (bool, error) {
req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return false, err
}
resp, err := r.client.Do(req)
if err != nil {
return false, err
}
if resp.StatusCode == http.StatusOK {
return true, nil
}
if resp.StatusCode == http.StatusNotFound {
return false, nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return false, err
}
return false, errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// PullBlob ...
func (r *Repository) PullBlob(digest string) (size int64, data []byte, err error) {
req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return
}
resp, err := r.client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode == http.StatusOK {
contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length"))
size, err = strconv.ParseInt(contengLength, 10, 64)
if err != nil {
return
}
data = b
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) {
req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil)
req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0")
resp, err := r.client.Do(req)
if err != nil {
return
}
if resp.StatusCode == http.StatusAccepted {
location = resp.Header.Get(http.CanonicalHeaderKey("Location"))
uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID"))
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
err = errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
return
}
func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data []byte) error {
req, err := http.NewRequest("PUT", buildMonolithicBlobUploadURL(location, digest), bytes.NewReader(data))
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusCreated {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
// PushBlob ...
func (r *Repository) PushBlob(digest string, size int64, data []byte) error {
exist, err := r.BlobExist(digest)
if err != nil {
return err
}
if exist {
log.Infof("blob already exists, skip pushing: %s %s", r.Name, digest)
return nil
}
location, _, err := r.initiateBlobUpload(r.Name)
if err != nil {
return err
}
return r.monolithicBlobUpload(location, digest, size, data)
}
// DeleteBlob ...
func (r *Repository) DeleteBlob(digest string) error {
req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil)
if err != nil {
return err
}
resp, err := r.client.Do(req)
if err != nil {
return err
}
if resp.StatusCode == http.StatusAccepted {
return nil
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return errors.Error{
StatusCode: resp.StatusCode,
Message: string(b),
}
}
func buildPingURL(endpoint string) string {
return fmt.Sprintf("%s/v2/", endpoint)
}
func buildTagListURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName)
}
func buildManifestURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference)
}
func buildBlobURL(endpoint, repoName, reference string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference)
}
func buildInitiateBlobUploadURL(endpoint, repoName string) string {
return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName)
}
func buildMonolithicBlobUploadURL(location, digest string) string {
return fmt.Sprintf("%s&digest=%s", location, digest)
}

59
utils/registry/transport.go Normal file
View File

@ -0,0 +1,59 @@
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package registry
import (
"net/http"
"github.com/vmware/harbor/utils/log"
)
// RequestModifier modifies request
type RequestModifier interface {
ModifyRequest(*http.Request) error
}
// Transport holds information about base transport and modifiers
type Transport struct {
transport http.RoundTripper
modifiers []RequestModifier
}
// NewTransport ...
func NewTransport(transport http.RoundTripper, modifiers []RequestModifier) *Transport {
return &Transport{
transport: transport,
modifiers: modifiers,
}
}
// RoundTrip ...
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
for _, modifier := range t.modifiers {
if err := modifier.ModifyRequest(req); err != nil {
return nil, err
}
}
resp, err := t.transport.RoundTrip(req)
if err != nil {
return nil, err
}
log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL.String())
return resp, err
}