From 9cbfb5fe2a1a4c2f53540e60391469f8a6dc1044 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 20 Apr 2016 13:06:19 +0800 Subject: [PATCH 1/6] modify comment --- utils/registry/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/registry/registry.go b/utils/registry/registry.go index e390c4c0d..f17fbdeea 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -28,7 +28,7 @@ import ( "github.com/vmware/harbor/utils/registry/errors" ) -// Registry holds information of a registry entiry +// Registry holds information of a registry entity type Registry struct { Endpoint *url.URL client *http.Client From e7f48783f2ebf3e5ba5288edf3ef839b94bf202a Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Thu, 21 Apr 2016 14:55:15 +0800 Subject: [PATCH 2/6] add methods about blob --- utils/registry/auth/handler.go | 5 +- utils/registry/registry.go | 214 ++++++++++++++++++++++++++++++--- 2 files changed, 198 insertions(+), 21 deletions(-) diff --git a/utils/registry/auth/handler.go b/utils/registry/auth/handler.go index 61550178d..845aa7fb6 100644 --- a/utils/registry/auth/handler.go +++ b/utils/registry/auth/handler.go @@ -143,10 +143,9 @@ func (t *standardTokenHandler) AuthorizeRequest(req *http.Request, params map[st } } - decoder := json.NewDecoder(resp.Body) - tk := &token{} - if err = decoder.Decode(tk); err != nil { + + if err = json.Unmarshal(b, tk); err != nil { return err } diff --git a/utils/registry/registry.go b/utils/registry/registry.go index f17fbdeea..5c7dc271c 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -16,15 +16,16 @@ package registry import ( + "bytes" "encoding/json" "fmt" "io/ioutil" "net/http" "net/url" + "strconv" - "github.com/docker/distribution/manifest" - "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry/errors" ) @@ -39,19 +40,6 @@ type uRLBuilder struct { root *url.URL } -var ( - // ManifestVersion1 : schema 1 - ManifestVersion1 = manifest.Versioned{ - SchemaVersion: 1, - MediaType: schema1.MediaTypeManifest, - } - // ManifestVersion2 : schema 2 - ManifestVersion2 = manifest.Versioned{ - SchemaVersion: 2, - MediaType: schema2.MediaTypeManifest, - } -) - // New returns an instance of Registry func New(endpoint string, client *http.Client) (*Registry, error) { u, err := url.Parse(endpoint) @@ -190,14 +178,15 @@ func (r *Registry) ManifestExist(name, reference string) (digest string, exist b } // PullManifest ... -func (r *Registry) PullManifest(name, reference string, version manifest.Versioned) (digest, mediaType string, payload []byte, err error) { +func (r *Registry) PullManifest(name, reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) { req, err := http.NewRequest("GET", r.ub.buildManifestURL(name, reference), nil) if err != nil { return } - // if the registry does not support schema 2, schema 1 manifest will be returned - req.Header.Set(http.CanonicalHeaderKey("Accept"), version.MediaType) + for _, mediaType := range acceptMediaTypes { + req.Header.Set(http.CanonicalHeaderKey("Accept"), mediaType) + } resp, err := r.client.Do(req) if err != nil { @@ -225,6 +214,40 @@ func (r *Registry) PullManifest(name, reference string, version manifest.Version return } +// PushManifest ... +func (r *Registry) PushManifest(name, reference, mediaType string, payload []byte) (digest string, err error) { + req, err := http.NewRequest("PUT", r.ub.buildManifestURL(name, reference), + bytes.NewReader(payload)) + if err != nil { + return + } + req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType) + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusCreated { + digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + // DeleteManifest ... func (r *Registry) DeleteManifest(name, digest string) error { req, err := http.NewRequest("DELETE", r.ub.buildManifestURL(name, digest), nil) @@ -270,6 +293,153 @@ func (r *Registry) DeleteTag(name, tag string) error { return r.DeleteManifest(name, digest) } +// BlobExist ... +func (r *Registry) BlobExist(name, digest string) (bool, error) { + req, err := http.NewRequest("HEAD", r.ub.buildBlobURL(name, digest), nil) + if err != nil { + return false, err + } + + resp, err := r.client.Do(req) + if err != nil { + return false, err + } + + if resp.StatusCode == http.StatusOK { + return true, nil + } + + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, err + } + + return false, errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// PullBlob ... +func (r *Registry) PullBlob(name, digest string) (size int64, data []byte, err error) { + req, err := http.NewRequest("GET", r.ub.buildBlobURL(name, digest), nil) + if err != nil { + return + } + + resp, err := r.client.Do(req) + if err != nil { + return + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + if resp.StatusCode == http.StatusOK { + contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length")) + size, err = strconv.ParseInt(contengLength, 10, 64) + if err != nil { + return + } + data = b + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +func (r *Registry) initiateBlobUpload(name string) (location, uploadUUID string, err error) { + req, err := http.NewRequest("POST", r.ub.buildInitiateBlobUploadURL(name), nil) + req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0") + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusAccepted { + location = resp.Header.Get(http.CanonicalHeaderKey("Location")) + uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID")) + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +func (r *Registry) monolithicBlobUpload(location, digest string, size int64, data []byte) error { + req, err := http.NewRequest("PUT", r.ub.buildMonolithicBlobUploadURL(location, digest), bytes.NewReader(data)) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusCreated { + return nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// PushBlob ... +func (r *Registry) PushBlob(name, digest string, size int64, data []byte) error { + exist, err := r.BlobExist(name, digest) + if err != nil { + return err + } + + if exist { + log.Infof("blob already exists, skip pushing: %s %s", name, digest) + return nil + } + + location, _, err := r.initiateBlobUpload(name) + if err != nil { + return err + } + + return r.monolithicBlobUpload(location, digest, size, data) +} + // DeleteBlob ... func (r *Registry) DeleteBlob(name, digest string) error { req, err := http.NewRequest("DELETE", r.ub.buildBlobURL(name, digest), nil) @@ -314,3 +484,11 @@ func (u *uRLBuilder) buildManifestURL(name, reference string) string { func (u *uRLBuilder) buildBlobURL(name, reference string) string { return fmt.Sprintf("%s/v2/%s/blobs/%s", u.root.String(), name, reference) } + +func (u *uRLBuilder) buildInitiateBlobUploadURL(name string) string { + return fmt.Sprintf("%s/v2/%s/blobs/uploads/", u.root.String(), name) +} + +func (u *uRLBuilder) buildMonolithicBlobUploadURL(location, digest string) string { + return fmt.Sprintf("%s&digest=%s", location, digest) +} From de29bd4468c326fb7b95aee4269d774a738e610d Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 22 Apr 2016 17:32:25 +0800 Subject: [PATCH 3/6] modify httpclient --- utils/registry/httpclient.go | 132 ++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 39 deletions(-) diff --git a/utils/registry/httpclient.go b/utils/registry/httpclient.go index f23a2c064..541957857 100644 --- a/utils/registry/httpclient.go +++ b/utils/registry/httpclient.go @@ -16,6 +16,7 @@ package registry import ( + //"io/ioutil" "net/http" "github.com/vmware/harbor/utils/log" @@ -70,47 +71,100 @@ func NewAuthTransport(transport http.RoundTripper, handlers []auth.Handler) http // RoundTrip ... func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { - originResp, originErr := a.transport.RoundTrip(req) + // TODO remove duplicate codes + if req.Body == nil { + originResp, originErr := a.transport.RoundTrip(req) - if originErr != nil { - return originResp, originErr - } - - log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) - - if originResp.StatusCode != http.StatusUnauthorized { - return originResp, nil - } - - challenges := auth.ParseChallengeFromResponse(originResp) - - reqChanged := false - for _, challenge := range challenges { - - scheme := challenge.Scheme - - for _, handler := range a.handlers { - if scheme != handler.Schema() { - log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) - continue - } - - if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { - return nil, err - } - reqChanged = true + if originErr != nil { + return originResp, originErr } + + log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) + + if originResp.StatusCode != http.StatusUnauthorized { + return originResp, nil + } + + challenges := auth.ParseChallengeFromResponse(originResp) + + reqChanged := false + for _, challenge := range challenges { + + scheme := challenge.Scheme + + for _, handler := range a.handlers { + if scheme != handler.Schema() { + log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) + continue + } + + if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { + return nil, err + } + reqChanged = true + } + } + + if !reqChanged { + log.Warning("no handler match scheme") + return originResp, nil + } + + resp, err := a.transport.RoundTrip(req) + if err == nil { + log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) + } + + return resp, err + } else { + originReqBody := req.Body + + originReqContentLength := req.ContentLength + req.Body = nil + req.ContentLength = 0 + + originResp, originErr := a.transport.RoundTrip(req) + if originErr == nil { + log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) + } else { + log.Error(originErr) + } + + if originErr == nil && originResp.StatusCode == http.StatusUnauthorized { + challenges := auth.ParseChallengeFromResponse(originResp) + + reqChanged := false + for _, challenge := range challenges { + + scheme := challenge.Scheme + + for _, handler := range a.handlers { + if scheme != handler.Schema() { + log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) + continue + } + + if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { + return nil, err + } + reqChanged = true + } + } + + if !reqChanged { + log.Warning("no handler match scheme") + } + } + + req.ContentLength = originReqContentLength + req.Body = originReqBody + + resp, err := a.transport.RoundTrip(req) + if err == nil { + log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) + } + + return resp, err } - if !reqChanged { - log.Warning("no handler match scheme") - return originResp, nil - } - - resp, err := a.transport.RoundTrip(req) - if err == nil { - log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) - } - - return resp, err } From 5b32c4cfbb31738481abd1ce510eca55afd45145 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Mon, 25 Apr 2016 13:31:58 +0800 Subject: [PATCH 4/6] using medie type of distribution --- api/repository.go | 4 +- utils/registry/httpclient.go | 97 ++++++++++++++++++------------------ 2 files changed, 51 insertions(+), 50 deletions(-) diff --git a/api/repository.go b/api/repository.go index 6058b3608..b8d3745f8 100644 --- a/api/repository.go +++ b/api/repository.go @@ -23,6 +23,7 @@ import ( "strings" "time" + "github.com/docker/distribution/manifest/schema1" "github.com/vmware/harbor/dao" "github.com/vmware/harbor/models" svc_utils "github.com/vmware/harbor/service/utils" @@ -246,7 +247,8 @@ func (ra *RepositoryAPI) GetManifests() { item := models.RepoItem{} - _, _, payload, err := ra.registry.PullManifest(repoName, tag, registry.ManifestVersion1) + mediaTypes := []string{schema1.MediaTypeManifest} + _, _, payload, err := ra.registry.PullManifest(repoName, tag, mediaTypes) if err != nil { e, ok := errors.ParseError(err) if ok { diff --git a/utils/registry/httpclient.go b/utils/registry/httpclient.go index 541957857..137fbd8ea 100644 --- a/utils/registry/httpclient.go +++ b/utils/registry/httpclient.go @@ -115,56 +115,55 @@ func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) } - return resp, err - } else { - originReqBody := req.Body - - originReqContentLength := req.ContentLength - req.Body = nil - req.ContentLength = 0 - - originResp, originErr := a.transport.RoundTrip(req) - if originErr == nil { - log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) - } else { - log.Error(originErr) - } - - if originErr == nil && originResp.StatusCode == http.StatusUnauthorized { - challenges := auth.ParseChallengeFromResponse(originResp) - - reqChanged := false - for _, challenge := range challenges { - - scheme := challenge.Scheme - - for _, handler := range a.handlers { - if scheme != handler.Schema() { - log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) - continue - } - - if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { - return nil, err - } - reqChanged = true - } - } - - if !reqChanged { - log.Warning("no handler match scheme") - } - } - - req.ContentLength = originReqContentLength - req.Body = originReqBody - - resp, err := a.transport.RoundTrip(req) - if err == nil { - log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) - } - return resp, err } + originReqBody := req.Body + + originReqContentLength := req.ContentLength + req.Body = nil + req.ContentLength = 0 + + originResp, originErr := a.transport.RoundTrip(req) + if originErr == nil { + log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) + } else { + log.Error(originErr) + } + + if originErr == nil && originResp.StatusCode == http.StatusUnauthorized { + challenges := auth.ParseChallengeFromResponse(originResp) + + reqChanged := false + for _, challenge := range challenges { + + scheme := challenge.Scheme + + for _, handler := range a.handlers { + if scheme != handler.Schema() { + log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) + continue + } + + if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { + return nil, err + } + reqChanged = true + } + } + + if !reqChanged { + log.Warning("no handler match scheme") + } + } + + req.ContentLength = originReqContentLength + req.Body = originReqBody + + resp, err := a.transport.RoundTrip(req) + if err == nil { + log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) + } + + return resp, err } From 994b4e290b8b41c00cccb13f7256f9e643ba309b Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 27 Apr 2016 17:59:43 +0800 Subject: [PATCH 5/6] registry v2 API util --- api/repository.go | 121 ++++--- service/utils/cache.go | 43 ++- utils/registry/auth/authorizer.go | 59 +++ utils/registry/auth/credential.go | 43 +++ utils/registry/auth/handler.go | 196 ---------- utils/registry/auth/tokenhandler.go | 236 ++++++++++++ utils/registry/httpclient.go | 169 --------- utils/registry/registry.go | 458 +++--------------------- utils/registry/repository.go | 533 ++++++++++++++++++++++++++++ utils/registry/transport.go | 59 +++ 10 files changed, 1075 insertions(+), 842 deletions(-) create mode 100644 utils/registry/auth/authorizer.go create mode 100644 utils/registry/auth/credential.go delete mode 100644 utils/registry/auth/handler.go create mode 100644 utils/registry/auth/tokenhandler.go delete mode 100644 utils/registry/httpclient.go create mode 100644 utils/registry/repository.go create mode 100644 utils/registry/transport.go diff --git a/api/repository.go b/api/repository.go index b8d3745f8..e4e4ea62b 100644 --- a/api/repository.go +++ b/api/repository.go @@ -41,61 +41,20 @@ type RepositoryAPI struct { BaseAPI userID int username string - registry *registry.Registry } // Prepare will set a non existent user ID in case the request tries to view repositories under a project he doesn't has permission. func (ra *RepositoryAPI) Prepare() { userID, ok := ra.GetSession("userId").(int) if !ok { - ra.userID = dao.NonExistUserID - } else { - ra.userID = userID + userID = dao.NonExistUserID } + ra.userID = userID + username, ok := ra.GetSession("username").(string) - if !ok { - log.Warning("failed to get username from session") - ra.username = "" - } else { + if ok { ra.username = username } - - var client *http.Client - - //no session, initialize a standard auth handler - if ra.userID == dao.NonExistUserID && len(ra.username) == 0 { - username, password, _ := ra.Ctx.Request.BasicAuth() - - credential := auth.NewBasicAuthCredential(username, password) - client = registry.NewClientStandardAuthHandlerEmbeded(credential) - log.Debug("initializing standard auth handler") - - } else { - // session works, initialize a username auth handler - username := ra.username - if len(username) == 0 { - user, err := dao.GetUser(models.User{ - UserID: ra.userID, - }) - if err != nil { - log.Errorf("error occurred whiling geting user for initializing a username auth handler: %v", err) - return - } - - username = user.Username - } - - client = registry.NewClientUsernameAuthHandlerEmbeded(username) - log.Debug("initializing username auth handler: %s", username) - } - - endpoint := os.Getenv("REGISTRY_URL") - r, err := registry.New(endpoint, client) - if err != nil { - log.Fatalf("error occurred while initializing auth handler for repository API: %v", err) - } - - ra.registry = r } // Get ... @@ -157,10 +116,16 @@ func (ra *RepositoryAPI) Delete() { ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") } + rc, err := ra.initializeRepositoryClient(repoName) + if err != nil { + log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + tags := []string{} tag := ra.GetString("tag") if len(tag) == 0 { - tagList, err := ra.registry.ListTag(repoName) + tagList, err := rc.ListTag() if err != nil { e, ok := errors.ParseError(err) if ok { @@ -170,16 +135,14 @@ func (ra *RepositoryAPI) Delete() { log.Error(err) ra.CustomAbort(http.StatusInternalServerError, "internal error") } - } tags = append(tags, tagList...) - } else { tags = append(tags, tag) } for _, t := range tags { - if err := ra.registry.DeleteTag(repoName, t); err != nil { + if err := rc.DeleteTag(t); err != nil { e, ok := errors.ParseError(err) if ok { ra.CustomAbort(e.StatusCode, e.Message) @@ -219,16 +182,23 @@ type manifest struct { // GetTags handles GET /api/repositories/tags func (ra *RepositoryAPI) GetTags() { - - var tags []string - repoName := ra.GetString("repo_name") + if len(repoName) == 0 { + ra.CustomAbort(http.StatusBadRequest, "repo_name is nil") + } - tags, err := ra.registry.ListTag(repoName) + rc, err := ra.initializeRepositoryClient(repoName) + if err != nil { + log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + + tags := []string{} + + ts, err := rc.ListTag() if err != nil { e, ok := errors.ParseError(err) if ok { - log.Info(e) ra.CustomAbort(e.StatusCode, e.Message) } else { log.Error(err) @@ -236,6 +206,8 @@ func (ra *RepositoryAPI) GetTags() { } } + tags = append(tags, ts...) + ra.Data["json"] = tags ra.ServeJSON() } @@ -245,14 +217,23 @@ func (ra *RepositoryAPI) GetManifests() { repoName := ra.GetString("repo_name") tag := ra.GetString("tag") + if len(repoName) == 0 || len(tag) == 0 { + ra.CustomAbort(http.StatusBadRequest, "repo_name or tag is nil") + } + + rc, err := ra.initializeRepositoryClient(repoName) + if err != nil { + log.Errorf("error occurred while initializing repository client for %s: %v", repoName, err) + ra.CustomAbort(http.StatusInternalServerError, "internal error") + } + item := models.RepoItem{} mediaTypes := []string{schema1.MediaTypeManifest} - _, _, payload, err := ra.registry.PullManifest(repoName, tag, mediaTypes) + _, _, payload, err := rc.PullManifest(tag, mediaTypes) if err != nil { e, ok := errors.ParseError(err) if ok { - log.Info(e) ra.CustomAbort(e.StatusCode, e.Message) } else { log.Error(err) @@ -280,3 +261,31 @@ func (ra *RepositoryAPI) GetManifests() { ra.Data["json"] = item ra.ServeJSON() } + +func (ra *RepositoryAPI) initializeRepositoryClient(repoName string) (r *registry.Repository, err error) { + endpoint := os.Getenv("REGISTRY_URL") + + //no session, use basic auth + if ra.userID == dao.NonExistUserID { + username, password, _ := ra.Ctx.Request.BasicAuth() + credential := auth.NewBasicAuthCredential(username, password) + + return registry.NewRepositoryWithCredential(repoName, endpoint, credential) + + } + + //session exists, use username + if len(ra.username) == 0 { + u := models.User{ + UserID: ra.userID, + } + user, err := dao.GetUser(u) + if err != nil { + return nil, err + } + + ra.username = user.Username + } + + return registry.NewRepositoryWithUsername(repoName, endpoint, ra.username) +} diff --git a/service/utils/cache.go b/service/utils/cache.go index a97a4599c..bbb9bd0b8 100644 --- a/service/utils/cache.go +++ b/service/utils/cache.go @@ -25,10 +25,14 @@ import ( "github.com/astaxie/beego/cache" ) -// Cache is the global cache in system. -var Cache cache.Cache - -var registryClient *registry.Registry +var ( + // Cache is the global cache in system. + Cache cache.Cache + endpoint string + username string + registryClient *registry.Registry + repositoryClients map[string]*registry.Repository +) const catalogKey string = "catalog" @@ -39,17 +43,25 @@ func init() { log.Errorf("Failed to initialize cache, error:%v", err) } - endpoint := os.Getenv("REGISTRY_URL") - client := registry.NewClientUsernameAuthHandlerEmbeded("admin") - registryClient, err = registry.New(endpoint, client) - if err != nil { - log.Fatalf("error occurred while initializing authentication handler used by cache: %v", err) - } + endpoint = os.Getenv("REGISTRY_URL") + username = "admin" + repositoryClients = make(map[string]*registry.Repository, 10) } // RefreshCatalogCache calls registry's API to get repository list and write it to cache. func RefreshCatalogCache() error { log.Debug("refreshing catalog cache...") + + if registryClient == nil { + var err error + registryClient, err = registry.NewRegistryWithUsername(endpoint, username) + if err != nil { + log.Errorf("error occurred while initializing registry client used by cache: %v", err) + return err + } + } + + var err error rs, err := registryClient.Catalog() if err != nil { return err @@ -58,7 +70,16 @@ func RefreshCatalogCache() error { repos := []string{} for _, repo := range rs { - tags, err := registryClient.ListTag(repo) + rc, ok := repositoryClients[repo] + if !ok { + rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username) + if err != nil { + log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err) + return err + } + repositoryClients[repo] = rc + } + tags, err := rc.ListTag() if err != nil { log.Errorf("error occurred while list tag for %s: %v", repo, err) return err diff --git a/utils/registry/auth/authorizer.go b/utils/registry/auth/authorizer.go new file mode 100644 index 000000000..e560355ae --- /dev/null +++ b/utils/registry/auth/authorizer.go @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package auth + +import ( + "net/http" + + au "github.com/docker/distribution/registry/client/auth" +) + +// Handler authorizes requests according to the schema +type Handler interface { + // Schema : basic, bearer + Schema() string + //AuthorizeRequest adds basic auth or token auth to the header of request + AuthorizeRequest(req *http.Request, params map[string]string) error +} + +// RequestAuthorizer holds a handler list, which will authorize request +type RequestAuthorizer struct { + handlers []Handler + challenges []au.Challenge +} + +// NewRequestAuthorizer ... +func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *RequestAuthorizer { + return &RequestAuthorizer{ + handlers: handlers, + challenges: challenges, + } +} + +// ModifyRequest adds authorization to the request +func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error { + for _, handler := range r.handlers { + for _, challenge := range r.challenges { + if handler.Schema() == challenge.Scheme { + if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { + return err + } + } + } + } + + return nil +} diff --git a/utils/registry/auth/credential.go b/utils/registry/auth/credential.go new file mode 100644 index 000000000..68143e2f9 --- /dev/null +++ b/utils/registry/auth/credential.go @@ -0,0 +1,43 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package auth + +import ( + "net/http" +) + +// Credential ... +type Credential interface { + // AddAuthorization adds authorization information to request + AddAuthorization(req *http.Request) +} + +type basicAuthCredential struct { + username string + password string +} + +// NewBasicAuthCredential ... +func NewBasicAuthCredential(username, password string) Credential { + return &basicAuthCredential{ + username: username, + password: password, + } +} + +func (b *basicAuthCredential) AddAuthorization(req *http.Request) { + req.SetBasicAuth(b.username, b.password) +} diff --git a/utils/registry/auth/handler.go b/utils/registry/auth/handler.go deleted file mode 100644 index 845aa7fb6..000000000 --- a/utils/registry/auth/handler.go +++ /dev/null @@ -1,196 +0,0 @@ -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package auth - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "strings" - - token_util "github.com/vmware/harbor/service/token" - "github.com/vmware/harbor/utils/log" - registry_errors "github.com/vmware/harbor/utils/registry/errors" -) - -const ( - // credential type - basicAuth string = "basic_auth" - secretKey string = "secret_key" -) - -// Handler authorizes the request when encounters a 401 error -type Handler interface { - // Schema : basic, bearer - Schema() string - //AuthorizeRequest adds basic auth or token auth to the header of request - AuthorizeRequest(req *http.Request, params map[string]string) error -} - -// Credential ... -type Credential interface { - // AddAuthorization adds authorization information to request - AddAuthorization(req *http.Request) -} - -type basicAuthCredential struct { - username string - password string -} - -// NewBasicAuthCredential ... -func NewBasicAuthCredential(username, password string) Credential { - return &basicAuthCredential{ - username: username, - password: password, - } -} - -func (b *basicAuthCredential) AddAuthorization(req *http.Request) { - req.SetBasicAuth(b.username, b.password) -} - -type token struct { - Token string `json:"token"` -} - -type standardTokenHandler struct { - client *http.Client - credential Credential -} - -// NewStandardTokenHandler returns a standard token handler. The handler will request a token -// from token server whose URL is specified in the "WWW-authentication" header and add it to -// the origin request -// TODO deal with https -func NewStandardTokenHandler(credential Credential) Handler { - return &standardTokenHandler{ - client: &http.Client{ - Transport: http.DefaultTransport, - }, - credential: credential, - } -} - -// Schema implements the corresponding method in interface AuthHandler -func (t *standardTokenHandler) Schema() string { - return "bearer" -} - -// AuthorizeRequest implements the corresponding method in interface AuthHandler -func (t *standardTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { - realm, ok := params["realm"] - if !ok { - return errors.New("no realm") - } - - service := params["service"] - scope := params["scope"] - - u, err := url.Parse(realm) - if err != nil { - return err - } - - q := u.Query() - q.Add("service", service) - - for _, s := range strings.Split(scope, " ") { - q.Add("scope", s) - } - - u.RawQuery = q.Encode() - - r, err := http.NewRequest("GET", u.String(), nil) - if err != nil { - return err - } - - t.credential.AddAuthorization(r) - - resp, err := t.client.Do(r) - if err != nil { - return err - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - if resp.StatusCode != http.StatusOK { - return registry_errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - } - - tk := &token{} - - if err = json.Unmarshal(b, tk); err != nil { - return err - } - - req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", tk.Token)) - - log.Debugf("standardTokenHandler generated token successfully | %s %s", req.Method, req.URL) - - return nil -} - -type usernameTokenHandler struct { - username string -} - -// NewUsernameTokenHandler returns a handler which will generate -// a token according the user's privileges -func NewUsernameTokenHandler(username string) Handler { - return &usernameTokenHandler{ - username: username, - } -} - -// Schema implements the corresponding method in interface AuthHandler -func (u *usernameTokenHandler) Schema() string { - return "bearer" -} - -// AuthorizeRequest implements the corresponding method in interface AuthHandler -func (u *usernameTokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { - service := params["service"] - - scopes := []string{} - scope := params["scope"] - if len(scope) != 0 { - scopes = strings.Split(scope, " ") - } - - token, err := token_util.GenTokenForUI(u.username, service, scopes) - if err != nil { - return err - } - - req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token)) - - log.Debugf("usernameTokenHandler generated token successfully | %s %s", req.Method, req.URL) - - return nil -} diff --git a/utils/registry/auth/tokenhandler.go b/utils/registry/auth/tokenhandler.go new file mode 100644 index 000000000..3e0b6c30f --- /dev/null +++ b/utils/registry/auth/tokenhandler.go @@ -0,0 +1,236 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package auth + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strings" + "time" + + token_util "github.com/vmware/harbor/service/token" + "github.com/vmware/harbor/utils/log" + registry_errors "github.com/vmware/harbor/utils/registry/errors" +) + +type scope struct { + Type string + Name string + Actions []string +} + +func (s *scope) string() string { + return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ",")) +} + +type token struct { + token string + expiresIn time.Time +} + +type tokenGenerator func(realm, service string, scopes []string) (*token, error) + +type tokenHandler struct { + scope *scope + cache map[string]*token + tg tokenGenerator +} + +// Schema returns the schema that the handler can handle +func (t *tokenHandler) Schema() string { + return "bearer" +} + +// AuthorizeRequest will add authorization header which contains a token before the request is sent +func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error { + var token string + var scopes []*scope + + // TODO handle additional scope: xxx.xxx.xxx?from=repo + + scopes = append(scopes, t.scope) + key := cacheKey(scopes) + + value, ok := t.cache[key] + var expired bool + + if ok { + expired = value.expiresIn.Before(time.Now()) + } + + if ok && !expired { + token = value.token + log.Debugf("get token from cache: %s", key) + } else { + if ok && expired { + delete(t.cache, key) + log.Debugf("token is expired, remove from cache: %s", key) + } + + scopeStrs := []string{} + for _, scope := range scopes { + scopeStrs = append(scopeStrs, scope.string()) + } + tk, err := t.tg(params["realm"], params["service"], scopeStrs) + if err != nil { + return err + } + token = tk.token + t.cache[key] = tk + log.Debugf("add token to cache: %s", key) + } + + req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token)) + log.Debugf("add token to request: %s %s", req.Method, req.URL.String()) + + return nil +} + +// cacheKey returns a string which can identify the scope array and can be used as the key in cache map +func cacheKey(scopes []*scope) string { + key := "" + for _, scope := range scopes { + key = key + scope.string() + "|" + } + key = strings.TrimRight(key, "|") + + return key +} + +type standardTokenHandler struct { + tokenHandler + client *http.Client + credential Credential +} + +// NewStandardTokenHandler returns a standard token handler. The handler will request a token +// from token server and add it to the origin request +// TODO deal with https +func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, scopeActions ...string) Handler { + handler := &standardTokenHandler{ + client: &http.Client{ + Transport: http.DefaultTransport, + }, + credential: credential, + } + + handler.scope = &scope{ + Type: scopeType, + Name: scopeName, + Actions: scopeActions, + } + handler.cache = make(map[string]*token, 1) + handler.tg = handler.generateToken + + return handler +} + +func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) { + u, err := url.Parse(realm) + if err != nil { + return nil, err + } + q := u.Query() + q.Add("service", service) + for _, scope := range scopes { + q.Add("scope", scope) + } + u.RawQuery = q.Encode() + r, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + + s.credential.AddAuthorization(r) + + resp, err := s.client.Do(r) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, registry_errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + } + + tk := struct { + Token string `json:"token"` + }{} + if err = json.Unmarshal(b, &tk); err != nil { + return nil, err + } + + t := &token{ + token: tk.Token, + // TODO handle the expires time + expiresIn: time.Now().Add(5 * time.Minute), + } + + log.Debug("get token from token server") + + return t, nil +} + +type usernameTokenHandler struct { + tokenHandler + username string +} + +// NewUsernameTokenHandler returns a handler which will generate a token according to +// the user's privileges +func NewUsernameTokenHandler(username string, scopeType, scopeName string, scopeActions ...string) Handler { + handler := &usernameTokenHandler{ + username: username, + } + + handler.scope = &scope{ + Type: scopeType, + Name: scopeName, + Actions: scopeActions, + } + handler.cache = make(map[string]*token, 1) + + handler.tg = handler.generateToken + + return handler +} + +func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (*token, error) { + tk, err := token_util.GenTokenForUI(u.username, service, scopes) + if err != nil { + return nil, err + } + + t := &token{ + token: tk, + // TODO handle the expires time + expiresIn: time.Now().Add(5 * time.Minute), + } + + log.Debug("get token by calling GenTokenForUI directly") + + return t, nil +} diff --git a/utils/registry/httpclient.go b/utils/registry/httpclient.go deleted file mode 100644 index 137fbd8ea..000000000 --- a/utils/registry/httpclient.go +++ /dev/null @@ -1,169 +0,0 @@ -/* - Copyright (c) 2016 VMware, Inc. All Rights Reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package registry - -import ( - //"io/ioutil" - "net/http" - - "github.com/vmware/harbor/utils/log" - "github.com/vmware/harbor/utils/registry/auth" -) - -// NewClient returns a http.Client according to the handlers provided -func NewClient(handlers []auth.Handler) *http.Client { - transport := NewAuthTransport(http.DefaultTransport, handlers) - - return &http.Client{ - Transport: transport, - } -} - -// NewClientStandardAuthHandlerEmbeded return a http.Client which will authorize the request -// according to the credential provided and send it again when encounters a 401 error -func NewClientStandardAuthHandlerEmbeded(credential auth.Credential) *http.Client { - handlers := []auth.Handler{} - - tokenHandler := auth.NewStandardTokenHandler(credential) - - handlers = append(handlers, tokenHandler) - - return NewClient(handlers) -} - -// NewClientUsernameAuthHandlerEmbeded return a http.Client which will authorize the request -// according to the user's privileges and send it again when encounters a 401 error -func NewClientUsernameAuthHandlerEmbeded(username string) *http.Client { - handlers := []auth.Handler{} - - tokenHandler := auth.NewUsernameTokenHandler(username) - - handlers = append(handlers, tokenHandler) - - return NewClient(handlers) -} - -type authTransport struct { - transport http.RoundTripper - handlers []auth.Handler -} - -// NewAuthTransport wraps the AuthHandlers to be http.RounTripper -func NewAuthTransport(transport http.RoundTripper, handlers []auth.Handler) http.RoundTripper { - return &authTransport{ - transport: transport, - handlers: handlers, - } -} - -// RoundTrip ... -func (a *authTransport) RoundTrip(req *http.Request) (*http.Response, error) { - // TODO remove duplicate codes - if req.Body == nil { - originResp, originErr := a.transport.RoundTrip(req) - - if originErr != nil { - return originResp, originErr - } - - log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) - - if originResp.StatusCode != http.StatusUnauthorized { - return originResp, nil - } - - challenges := auth.ParseChallengeFromResponse(originResp) - - reqChanged := false - for _, challenge := range challenges { - - scheme := challenge.Scheme - - for _, handler := range a.handlers { - if scheme != handler.Schema() { - log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) - continue - } - - if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { - return nil, err - } - reqChanged = true - } - } - - if !reqChanged { - log.Warning("no handler match scheme") - return originResp, nil - } - - resp, err := a.transport.RoundTrip(req) - if err == nil { - log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) - } - - return resp, err - } - - originReqBody := req.Body - - originReqContentLength := req.ContentLength - req.Body = nil - req.ContentLength = 0 - - originResp, originErr := a.transport.RoundTrip(req) - if originErr == nil { - log.Debugf("%d | %s %s", originResp.StatusCode, req.Method, req.URL) - } else { - log.Error(originErr) - } - - if originErr == nil && originResp.StatusCode == http.StatusUnauthorized { - challenges := auth.ParseChallengeFromResponse(originResp) - - reqChanged := false - for _, challenge := range challenges { - - scheme := challenge.Scheme - - for _, handler := range a.handlers { - if scheme != handler.Schema() { - log.Debugf("scheme not match: %s %s, skip", scheme, handler.Schema()) - continue - } - - if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil { - return nil, err - } - reqChanged = true - } - } - - if !reqChanged { - log.Warning("no handler match scheme") - } - } - - req.ContentLength = originReqContentLength - req.Body = originReqBody - - resp, err := a.transport.RoundTrip(req) - if err == nil { - log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL) - } - - return resp, err -} diff --git a/utils/registry/registry.go b/utils/registry/registry.go index 5c7dc271c..1d2f48a3e 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -16,16 +16,14 @@ package registry import ( - "bytes" "encoding/json" "fmt" "io/ioutil" "net/http" "net/url" - "strconv" + "strings" - "github.com/docker/distribution/manifest/schema2" - "github.com/vmware/harbor/utils/log" + "github.com/vmware/harbor/utils/registry/auth" "github.com/vmware/harbor/utils/registry/errors" ) @@ -33,33 +31,66 @@ import ( type Registry struct { Endpoint *url.URL client *http.Client - ub *uRLBuilder } -type uRLBuilder struct { - root *url.URL -} +// NewRegistry returns an instance of registry +func NewRegistry(endpoint string, client *http.Client) (*Registry, error) { + endpoint = strings.TrimRight(endpoint, "/") -// New returns an instance of Registry -func New(endpoint string, client *http.Client) (*Registry, error) { u, err := url.Parse(endpoint) if err != nil { return nil, err } - return &Registry{ + registry := &Registry{ Endpoint: u, client: client, - ub: &uRLBuilder{ - root: u, + } + + log.Debugf("initialized a registry client: %s", endpoint) + + return registry, nil +} + +// NewRegistryWithUsername returns a Registry instance which will authorize the request +// according to the privileges of user +func NewRegistryWithUsername(endpoint, username string) (*Registry, error) { + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + resp, err := http.Get(buildPingURL(endpoint)) + if err != nil { + return nil, err + } + + var handlers []auth.Handler + handler := auth.NewUsernameTokenHandler(username, "registry", "catalog", "*") + handlers = append(handlers, handler) + + challenges := auth.ParseChallengeFromResponse(resp) + authorizer := auth.NewRequestAuthorizer(handlers, challenges) + + transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) + + registry := &Registry{ + Endpoint: u, + client: &http.Client{ + Transport: transport, }, - }, nil + } + + return registry, nil } // Catalog ... func (r *Registry) Catalog() ([]string, error) { repos := []string{} - req, err := http.NewRequest("GET", r.ub.buildCatalogURL(), nil) + + req, err := http.NewRequest("GET", buildCatalogURL(r.Endpoint.String()), nil) if err != nil { return repos, err } @@ -96,399 +127,6 @@ func (r *Registry) Catalog() ([]string, error) { } } -// ListTag ... -func (r *Registry) ListTag(name string) ([]string, error) { - tags := []string{} - req, err := http.NewRequest("GET", r.ub.buildTagListURL(name), nil) - if err != nil { - return tags, err - } - - resp, err := r.client.Do(req) - if err != nil { - return tags, err - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return tags, err - } - - if resp.StatusCode == http.StatusOK { - tagsResp := struct { - Tags []string `json:"tags"` - }{} - - if err := json.Unmarshal(b, &tagsResp); err != nil { - return tags, err - } - - tags = tagsResp.Tags - - return tags, nil - } - - return tags, errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - -} - -// ManifestExist ... -func (r *Registry) ManifestExist(name, reference string) (digest string, exist bool, err error) { - req, err := http.NewRequest("HEAD", r.ub.buildManifestURL(name, reference), nil) - if err != nil { - return - } - - // request Schema 2 manifest, if the registry does not support it, - // Schema 1 manifest will be returned - req.Header.Set(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest) - - resp, err := r.client.Do(req) - if err != nil { - return - } - - if resp.StatusCode == http.StatusOK { - exist = true - digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) - return - } - - if resp.StatusCode == http.StatusNotFound { - return - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - err = errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - return -} - -// PullManifest ... -func (r *Registry) PullManifest(name, reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) { - req, err := http.NewRequest("GET", r.ub.buildManifestURL(name, reference), nil) - if err != nil { - return - } - - for _, mediaType := range acceptMediaTypes { - req.Header.Set(http.CanonicalHeaderKey("Accept"), mediaType) - } - - resp, err := r.client.Do(req) - if err != nil { - return - } - - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode == http.StatusOK { - digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) - mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type")) - payload = b - return - } - - err = errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - - return -} - -// PushManifest ... -func (r *Registry) PushManifest(name, reference, mediaType string, payload []byte) (digest string, err error) { - req, err := http.NewRequest("PUT", r.ub.buildManifestURL(name, reference), - bytes.NewReader(payload)) - if err != nil { - return - } - req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType) - - resp, err := r.client.Do(req) - if err != nil { - return - } - - if resp.StatusCode == http.StatusCreated { - digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) - return - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - err = errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - - return -} - -// DeleteManifest ... -func (r *Registry) DeleteManifest(name, digest string) error { - req, err := http.NewRequest("DELETE", r.ub.buildManifestURL(name, digest), nil) - if err != nil { - return err - } - - resp, err := r.client.Do(req) - if err != nil { - return err - } - - if resp.StatusCode == http.StatusAccepted { - return nil - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - return errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } -} - -// DeleteTag ... -func (r *Registry) DeleteTag(name, tag string) error { - digest, exist, err := r.ManifestExist(name, tag) - if err != nil { - return err - } - - if !exist { - return errors.Error{ - StatusCode: http.StatusNotFound, - } - } - - return r.DeleteManifest(name, digest) -} - -// BlobExist ... -func (r *Registry) BlobExist(name, digest string) (bool, error) { - req, err := http.NewRequest("HEAD", r.ub.buildBlobURL(name, digest), nil) - if err != nil { - return false, err - } - - resp, err := r.client.Do(req) - if err != nil { - return false, err - } - - if resp.StatusCode == http.StatusOK { - return true, nil - } - - if resp.StatusCode == http.StatusNotFound { - return false, nil - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return false, err - } - - return false, errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } -} - -// PullBlob ... -func (r *Registry) PullBlob(name, digest string) (size int64, data []byte, err error) { - req, err := http.NewRequest("GET", r.ub.buildBlobURL(name, digest), nil) - if err != nil { - return - } - - resp, err := r.client.Do(req) - if err != nil { - return - } - - defer resp.Body.Close() - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - if resp.StatusCode == http.StatusOK { - contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length")) - size, err = strconv.ParseInt(contengLength, 10, 64) - if err != nil { - return - } - data = b - return - } - - err = errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - - return -} - -func (r *Registry) initiateBlobUpload(name string) (location, uploadUUID string, err error) { - req, err := http.NewRequest("POST", r.ub.buildInitiateBlobUploadURL(name), nil) - req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0") - - resp, err := r.client.Do(req) - if err != nil { - return - } - - if resp.StatusCode == http.StatusAccepted { - location = resp.Header.Get(http.CanonicalHeaderKey("Location")) - uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID")) - return - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return - } - - err = errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } - - return -} - -func (r *Registry) monolithicBlobUpload(location, digest string, size int64, data []byte) error { - req, err := http.NewRequest("PUT", r.ub.buildMonolithicBlobUploadURL(location, digest), bytes.NewReader(data)) - if err != nil { - return err - } - - resp, err := r.client.Do(req) - if err != nil { - return err - } - - if resp.StatusCode == http.StatusCreated { - return nil - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - return errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } -} - -// PushBlob ... -func (r *Registry) PushBlob(name, digest string, size int64, data []byte) error { - exist, err := r.BlobExist(name, digest) - if err != nil { - return err - } - - if exist { - log.Infof("blob already exists, skip pushing: %s %s", name, digest) - return nil - } - - location, _, err := r.initiateBlobUpload(name) - if err != nil { - return err - } - - return r.monolithicBlobUpload(location, digest, size, data) -} - -// DeleteBlob ... -func (r *Registry) DeleteBlob(name, digest string) error { - req, err := http.NewRequest("DELETE", r.ub.buildBlobURL(name, digest), nil) - if err != nil { - return err - } - - resp, err := r.client.Do(req) - if err != nil { - return err - } - - if resp.StatusCode == http.StatusAccepted { - return nil - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - - return errors.Error{ - StatusCode: resp.StatusCode, - Message: string(b), - } -} - -func (u *uRLBuilder) buildCatalogURL() string { - return fmt.Sprintf("%s/v2/_catalog", u.root.String()) -} - -func (u *uRLBuilder) buildTagListURL(name string) string { - return fmt.Sprintf("%s/v2/%s/tags/list", u.root.String(), name) -} - -func (u *uRLBuilder) buildManifestURL(name, reference string) string { - return fmt.Sprintf("%s/v2/%s/manifests/%s", u.root.String(), name, reference) -} - -func (u *uRLBuilder) buildBlobURL(name, reference string) string { - return fmt.Sprintf("%s/v2/%s/blobs/%s", u.root.String(), name, reference) -} - -func (u *uRLBuilder) buildInitiateBlobUploadURL(name string) string { - return fmt.Sprintf("%s/v2/%s/blobs/uploads/", u.root.String(), name) -} - -func (u *uRLBuilder) buildMonolithicBlobUploadURL(location, digest string) string { - return fmt.Sprintf("%s&digest=%s", location, digest) +func buildCatalogURL(endpoint string) string { + return fmt.Sprintf("%s/v2/_catalog", endpoint) } diff --git a/utils/registry/repository.go b/utils/registry/repository.go new file mode 100644 index 000000000..121b26741 --- /dev/null +++ b/utils/registry/repository.go @@ -0,0 +1,533 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "strconv" + "strings" + + "github.com/docker/distribution/manifest/schema1" + "github.com/docker/distribution/manifest/schema2" + "github.com/vmware/harbor/utils/log" + "github.com/vmware/harbor/utils/registry/auth" + "github.com/vmware/harbor/utils/registry/errors" +) + +// Repository holds information of a repository entity +type Repository struct { + Name string + Endpoint *url.URL + client *http.Client +} + +// TODO add agent to header of request, notifications need it + +// NewRepository returns an instance of Repository +func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) { + name = strings.TrimSpace(name) + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + repository := &Repository{ + Name: name, + Endpoint: u, + client: client, + } + + return repository, nil +} + +// NewRepositoryWithCredential returns a Repository instance which will authorize the request +// according to the credenttial +func NewRepositoryWithCredential(name, endpoint string, credential auth.Credential) (*Repository, error) { + name = strings.TrimSpace(name) + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + resp, err := http.Get(buildPingURL(endpoint)) + if err != nil { + return nil, err + } + + var handlers []auth.Handler + handler := auth.NewStandardTokenHandler(credential, "repository", name, "pull", "push") + handlers = append(handlers, handler) + + challenges := auth.ParseChallengeFromResponse(resp) + authorizer := auth.NewRequestAuthorizer(handlers, challenges) + + transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) + + repository := &Repository{ + Name: name, + Endpoint: u, + client: &http.Client{ + Transport: transport, + }, + } + + log.Debugf("initialized a repository client with credential: %s %s", endpoint, name) + + return repository, nil +} + +// NewRepositoryWithUsername returns a Repository instance which will authorize the request +// according to the privileges of user +func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, error) { + name = strings.TrimSpace(name) + endpoint = strings.TrimRight(endpoint, "/") + + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + + resp, err := http.Get(buildPingURL(endpoint)) + if err != nil { + return nil, err + } + + var handlers []auth.Handler + handler := auth.NewUsernameTokenHandler(username, "repository", name, "pull", "push") + handlers = append(handlers, handler) + + challenges := auth.ParseChallengeFromResponse(resp) + authorizer := auth.NewRequestAuthorizer(handlers, challenges) + + transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer}) + + repository := &Repository{ + Name: name, + Endpoint: u, + client: &http.Client{ + Transport: transport, + }, + } + + log.Debugf("initialized a repository client with username: %s %s", endpoint, name) + + return repository, nil +} + +// ListTag ... +func (r *Repository) ListTag() ([]string, error) { + tags := []string{} + req, err := http.NewRequest("GET", buildTagListURL(r.Endpoint.String(), r.Name), nil) + if err != nil { + return tags, err + } + + resp, err := r.client.Do(req) + if err != nil { + return tags, err + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return tags, err + } + + if resp.StatusCode == http.StatusOK { + tagsResp := struct { + Tags []string `json:"tags"` + }{} + + if err := json.Unmarshal(b, &tagsResp); err != nil { + return tags, err + } + + tags = tagsResp.Tags + + return tags, nil + } + + return tags, errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + +} + +// ManifestExist ... +func (r *Repository) ManifestExist(reference string) (digest string, exist bool, err error) { + req, err := http.NewRequest("HEAD", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil) + if err != nil { + return + } + + req.Header.Add(http.CanonicalHeaderKey("Accept"), schema1.MediaTypeManifest) + req.Header.Add(http.CanonicalHeaderKey("Accept"), schema2.MediaTypeManifest) + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusOK { + exist = true + digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + return + } + + if resp.StatusCode == http.StatusNotFound { + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + return +} + +// PullManifest ... +func (r *Repository) PullManifest(reference string, acceptMediaTypes []string) (digest, mediaType string, payload []byte, err error) { + req, err := http.NewRequest("GET", buildManifestURL(r.Endpoint.String(), r.Name, reference), nil) + if err != nil { + return + } + + for _, mediaType := range acceptMediaTypes { + req.Header.Add(http.CanonicalHeaderKey("Accept"), mediaType) + } + + resp, err := r.client.Do(req) + if err != nil { + return + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + if resp.StatusCode == http.StatusOK { + digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + mediaType = resp.Header.Get(http.CanonicalHeaderKey("Content-Type")) + payload = b + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +// PushManifest ... +func (r *Repository) PushManifest(reference, mediaType string, payload []byte) (digest string, err error) { + req, err := http.NewRequest("PUT", buildManifestURL(r.Endpoint.String(), r.Name, reference), + bytes.NewReader(payload)) + if err != nil { + return + } + req.Header.Set(http.CanonicalHeaderKey("Content-Type"), mediaType) + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusCreated { + digest = resp.Header.Get(http.CanonicalHeaderKey("Docker-Content-Digest")) + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +// DeleteManifest ... +func (r *Repository) DeleteManifest(digest string) error { + req, err := http.NewRequest("DELETE", buildManifestURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusAccepted { + return nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// DeleteTag ... +func (r *Repository) DeleteTag(tag string) error { + digest, exist, err := r.ManifestExist(tag) + if err != nil { + return err + } + + if !exist { + return errors.Error{ + StatusCode: http.StatusNotFound, + } + } + + return r.DeleteManifest(digest) +} + +// BlobExist ... +func (r *Repository) BlobExist(digest string) (bool, error) { + req, err := http.NewRequest("HEAD", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return false, err + } + + resp, err := r.client.Do(req) + if err != nil { + return false, err + } + + if resp.StatusCode == http.StatusOK { + return true, nil + } + + if resp.StatusCode == http.StatusNotFound { + return false, nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return false, err + } + + return false, errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// PullBlob ... +func (r *Repository) PullBlob(digest string) (size int64, data []byte, err error) { + req, err := http.NewRequest("GET", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return + } + + resp, err := r.client.Do(req) + if err != nil { + return + } + + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + if resp.StatusCode == http.StatusOK { + contengLength := resp.Header.Get(http.CanonicalHeaderKey("Content-Length")) + size, err = strconv.ParseInt(contengLength, 10, 64) + if err != nil { + return + } + data = b + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +func (r *Repository) initiateBlobUpload(name string) (location, uploadUUID string, err error) { + req, err := http.NewRequest("POST", buildInitiateBlobUploadURL(r.Endpoint.String(), r.Name), nil) + req.Header.Set(http.CanonicalHeaderKey("Content-Length"), "0") + + resp, err := r.client.Do(req) + if err != nil { + return + } + + if resp.StatusCode == http.StatusAccepted { + location = resp.Header.Get(http.CanonicalHeaderKey("Location")) + uploadUUID = resp.Header.Get(http.CanonicalHeaderKey("Docker-Upload-UUID")) + return + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } + + return +} + +func (r *Repository) monolithicBlobUpload(location, digest string, size int64, data []byte) error { + req, err := http.NewRequest("PUT", buildMonolithicBlobUploadURL(location, digest), bytes.NewReader(data)) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusCreated { + return nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +// PushBlob ... +func (r *Repository) PushBlob(digest string, size int64, data []byte) error { + exist, err := r.BlobExist(digest) + if err != nil { + return err + } + + if exist { + log.Infof("blob already exists, skip pushing: %s %s", r.Name, digest) + return nil + } + + location, _, err := r.initiateBlobUpload(r.Name) + if err != nil { + return err + } + + return r.monolithicBlobUpload(location, digest, size, data) +} + +// DeleteBlob ... +func (r *Repository) DeleteBlob(digest string) error { + req, err := http.NewRequest("DELETE", buildBlobURL(r.Endpoint.String(), r.Name, digest), nil) + if err != nil { + return err + } + + resp, err := r.client.Do(req) + if err != nil { + return err + } + + if resp.StatusCode == http.StatusAccepted { + return nil + } + + defer resp.Body.Close() + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + return errors.Error{ + StatusCode: resp.StatusCode, + Message: string(b), + } +} + +func buildPingURL(endpoint string) string { + return fmt.Sprintf("%s/v2/", endpoint) +} + +func buildTagListURL(endpoint, repoName string) string { + return fmt.Sprintf("%s/v2/%s/tags/list", endpoint, repoName) +} + +func buildManifestURL(endpoint, repoName, reference string) string { + return fmt.Sprintf("%s/v2/%s/manifests/%s", endpoint, repoName, reference) +} + +func buildBlobURL(endpoint, repoName, reference string) string { + return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repoName, reference) +} + +func buildInitiateBlobUploadURL(endpoint, repoName string) string { + return fmt.Sprintf("%s/v2/%s/blobs/uploads/", endpoint, repoName) +} + +func buildMonolithicBlobUploadURL(location, digest string) string { + return fmt.Sprintf("%s&digest=%s", location, digest) +} diff --git a/utils/registry/transport.go b/utils/registry/transport.go new file mode 100644 index 000000000..f2569b3cd --- /dev/null +++ b/utils/registry/transport.go @@ -0,0 +1,59 @@ +/* + Copyright (c) 2016 VMware, Inc. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package registry + +import ( + "net/http" + + "github.com/vmware/harbor/utils/log" +) + +// RequestModifier modifies request +type RequestModifier interface { + ModifyRequest(*http.Request) error +} + +// Transport holds information about base transport and modifiers +type Transport struct { + transport http.RoundTripper + modifiers []RequestModifier +} + +// NewTransport ... +func NewTransport(transport http.RoundTripper, modifiers []RequestModifier) *Transport { + return &Transport{ + transport: transport, + modifiers: modifiers, + } +} + +// RoundTrip ... +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + for _, modifier := range t.modifiers { + if err := modifier.ModifyRequest(req); err != nil { + return nil, err + } + } + + resp, err := t.transport.RoundTrip(req) + if err != nil { + return nil, err + } + + log.Debugf("%d | %s %s", resp.StatusCode, req.Method, req.URL.String()) + + return resp, err +} From 65fe14d2d37e9600e73c25915dea60a3ae591dbc Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Wed, 27 Apr 2016 18:03:56 +0800 Subject: [PATCH 6/6] add log --- utils/registry/registry.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/registry/registry.go b/utils/registry/registry.go index 1d2f48a3e..201e29f20 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -23,6 +23,7 @@ import ( "net/url" "strings" + "github.com/vmware/harbor/utils/log" "github.com/vmware/harbor/utils/registry/auth" "github.com/vmware/harbor/utils/registry/errors" )