From 2cded5990938579ed8b1a37682cc65f243961ba2 Mon Sep 17 00:00:00 2001 From: yhua123 Date: Thu, 23 Jun 2016 15:14:10 +0800 Subject: [PATCH 1/7] Update create-policy.directive.html delete 'usrname' & 'passwd' blank remind message --- .../replication/create-policy.directive.html | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/static/resources/js/components/replication/create-policy.directive.html b/static/resources/js/components/replication/create-policy.directive.html index 75fd8f35d..e08982861 100644 --- a/static/resources/js/components/replication/create-policy.directive.html +++ b/static/resources/js/components/replication/create-policy.directive.html @@ -59,19 +59,13 @@
- -
- // 'username_is_required' | tr // -
+
- -
- // 'password_is_required' | tr // -
+
@@ -89,4 +83,4 @@
- \ No newline at end of file + From fcc87d71a9d2b42cbc69c8126451ad45683654ab Mon Sep 17 00:00:00 2001 From: yhua123 Date: Thu, 23 Jun 2016 15:19:12 +0800 Subject: [PATCH 2/7] Update create-destination.directive.html delete 'username' and 'passwd' blank remind --- .../create-destination.directive.html | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/static/resources/js/components/system-management/create-destination.directive.html b/static/resources/js/components/system-management/create-destination.directive.html index 9f1e59140..07339d306 100644 --- a/static/resources/js/components/system-management/create-destination.directive.html +++ b/static/resources/js/components/system-management/create-destination.directive.html @@ -29,19 +29,13 @@
- -
- // 'username_is_required' | tr // -
+
- -
- // 'password_is_required' | tr // -
+
@@ -59,4 +53,4 @@
- \ No newline at end of file + From 7bd1888b7356df0ca3daf95ea2f26f2772802cb5 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Fri, 24 Jun 2016 00:21:14 +0800 Subject: [PATCH 3/7] handle pagination in catalog api --- utils/registry/registry.go | 81 ++++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 38 deletions(-) diff --git a/utils/registry/registry.go b/utils/registry/registry.go index baaf70e91..08b7411b1 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -17,7 +17,6 @@ package registry import ( "encoding/json" - "fmt" "io/ioutil" "net/http" "net/url" @@ -81,51 +80,57 @@ func NewRegistryWithUsername(endpoint, username string) (*Registry, error) { // Catalog ... func (r *Registry) Catalog() ([]string, error) { repos := []string{} + suffix := "/v2/_catalog" + var url string - req, err := http.NewRequest("GET", buildCatalogURL(r.Endpoint.String()), nil) - if err != nil { - return repos, err - } + for len(suffix) > 0 { + url = r.Endpoint.String() + suffix - resp, err := r.client.Do(req) - if err != nil { - ok, e := isUnauthorizedError(err) - if ok { - return repos, e + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return repos, err } - return repos, err - } - - defer resp.Body.Close() - - b, err := ioutil.ReadAll(resp.Body) - if err != nil { - return repos, err - } - - if resp.StatusCode == http.StatusOK { - catalogResp := struct { - Repositories []string `json:"repositories"` - }{} - - if err := json.Unmarshal(b, &catalogResp); err != nil { + resp, err := r.client.Do(req) + if err != nil { + ok, e := isUnauthorizedError(err) + if ok { + return repos, e + } return repos, err } - repos = catalogResp.Repositories + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return repos, err + } - return repos, nil + if resp.StatusCode == http.StatusOK { + catalogResp := struct { + Repositories []string `json:"repositories"` + }{} + + if err := json.Unmarshal(b, &catalogResp); err != nil { + return repos, err + } + + repos = append(repos, catalogResp.Repositories...) + //Link: ; rel="next" + link := resp.Header.Get("Link") + if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 { + suffix = link[strings.Index(link, "<")+1 : strings.Index(link, ">")] + } else { + suffix = "" + } + } else { + return repos, errors.Error{ + StatusCode: resp.StatusCode, + StatusText: resp.Status, + Message: string(b), + } + } } - - return repos, errors.Error{ - StatusCode: resp.StatusCode, - StatusText: resp.Status, - Message: string(b), - } -} - -func buildCatalogURL(endpoint string) string { - return fmt.Sprintf("%s/v2/_catalog", endpoint) + return repos, nil } func newClient(endpoint, username string, credential auth.Credential, From 23d1acb24abd4ca5444380484e20e71280bffde0 Mon Sep 17 00:00:00 2001 From: Tan Jiang Date: Fri, 24 Jun 2016 01:37:37 +0800 Subject: [PATCH 4/7] fixes #398 --- utils/registry/registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/registry/registry.go b/utils/registry/registry.go index 08b7411b1..c4b7677f5 100644 --- a/utils/registry/registry.go +++ b/utils/registry/registry.go @@ -80,7 +80,7 @@ func NewRegistryWithUsername(endpoint, username string) (*Registry, error) { // Catalog ... func (r *Registry) Catalog() ([]string, error) { repos := []string{} - suffix := "/v2/_catalog" + suffix := "/v2/_catalog?n=1000" var url string for len(suffix) > 0 { From d113ed22a2f10368363071a31a77c557d77a73fc Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 24 Jun 2016 12:34:15 +0800 Subject: [PATCH 5/7] add error logs to job log file when error occurs --- job/replication/transfer.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/job/replication/transfer.go b/job/replication/transfer.go index 30bbf88ae..09dcc7308 100644 --- a/job/replication/transfer.go +++ b/job/replication/transfer.go @@ -102,6 +102,7 @@ func InitBaseHandler(repository, srcURL, srcSecret, srcClient, err := newRepositoryClient(base.srcURL, base.insecure, srcCred, base.repository, "repository", base.repository, "pull", "push", "*") if err != nil { + base.logger.Errorf("an error occurred while creating source repository client: %v", err) return nil, err } base.srcClient = srcClient @@ -110,6 +111,7 @@ func InitBaseHandler(repository, srcURL, srcSecret, dstClient, err := newRepositoryClient(base.dstURL, base.insecure, dstCred, base.repository, "repository", base.repository, "pull", "push", "*") if err != nil { + base.logger.Errorf("an error occurred while creating destination repository client: %v", err) return nil, err } base.dstClient = dstClient @@ -117,6 +119,7 @@ func InitBaseHandler(repository, srcURL, srcSecret, if len(base.tags) == 0 { tags, err := base.srcClient.ListTag() if err != nil { + base.logger.Errorf("an error occurred while listing tags for source repository: %v", err) return nil, err } base.tags = tags From 97150ee1f89412c48251fa4df91bdd17fa1a2b30 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 24 Jun 2016 13:00:56 +0800 Subject: [PATCH 6/7] add insecure filed to job log --- job/replication/delete.go | 4 ++-- job/replication/transfer.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/job/replication/delete.go b/job/replication/delete.go index 8bf7813f8..1bec6378f 100644 --- a/job/replication/delete.go +++ b/job/replication/delete.go @@ -62,8 +62,8 @@ func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, dstClient: dstClient, logger: logger, } - deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, destination user: %s", - deleter.repository, deleter.tags, deleter.dstURL, deleter.dstUsr) + deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, insecure: %v, destination user: %s", + deleter.repository, deleter.tags, deleter.dstURL, deleter.insecure, deleter.dstUsr) return deleter, nil } diff --git a/job/replication/transfer.go b/job/replication/transfer.go index 09dcc7308..349026f4d 100644 --- a/job/replication/transfer.go +++ b/job/replication/transfer.go @@ -80,8 +80,8 @@ type BaseHandler struct { func InitBaseHandler(repository, srcURL, srcSecret, dstURL, dstUsr, dstPwd string, insecure bool, tags []string, logger *log.Logger) (*BaseHandler, error) { - logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s", - repository, tags, srcURL, dstURL, dstUsr) + logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, insecure: %v, destination user: %s", + repository, tags, srcURL, dstURL, insecure, dstUsr) base := &BaseHandler{ repository: repository, @@ -125,8 +125,8 @@ func InitBaseHandler(repository, srcURL, srcSecret, base.tags = tags } - base.logger.Infof("initialization completed: project: %s, repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s", - base.project, base.repository, base.tags, base.srcURL, base.dstURL, base.dstUsr) + base.logger.Infof("initialization completed: project: %s, repository: %s, tags: %v, source URL: %s, destination URL: %s, insecure: %v, destination user: %s", + base.project, base.repository, base.tags, base.srcURL, base.dstURL, base.insecure, base.dstUsr) return base, nil } From 6c2dc9759a1f65664623214f3ccfed336e15d533 Mon Sep 17 00:00:00 2001 From: Wenkai Yin Date: Fri, 24 Jun 2016 13:57:48 +0800 Subject: [PATCH 7/7] pass variable insecure to base handler --- job/replication/transfer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/job/replication/transfer.go b/job/replication/transfer.go index 349026f4d..dd2fba546 100644 --- a/job/replication/transfer.go +++ b/job/replication/transfer.go @@ -90,6 +90,7 @@ func InitBaseHandler(repository, srcURL, srcSecret, dstURL: dstURL, dstUsr: dstUsr, dstPwd: dstPwd, + insecure: insecure, blobsExistence: make(map[string]bool, 10), logger: logger, }