Merge branch 'new-ui-with-sync-image' of https://github.com/vmware/harbor into new-ui-with-sync-image

Deploy/harbor.cfg

@@ -38,6 +38,10 @@ self_registration = on
 #Number of job workers in job service, default is 3 
 max_job_workers = 3 
 
+#Toggle on and off to tell job service wheter or not verify the ssl cert
+#when it tries to access a remote registry
+verify_remote_cert = on
+
 #Turn on or off the customize your certificate for registry's token.
 #If the value is on, the prepare script will generate new root cert and private key 
 #for generating token to access the image in registry.

Deploy/prepare

@@ -47,6 +47,7 @@ crt_organizationalunit = rcp.get("configuration", "crt_organizationalunit")
 crt_commonname = rcp.get("configuration", "crt_commonname")
 crt_email = rcp.get("configuration", "crt_email")
 max_job_workers = rcp.get("configuration", "max_job_workers")
+verify_remote_cert = rcp.get("configuration", "verify_remote_cert")
 ########
 
 ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  
@@ -122,7 +123,8 @@ render(os.path.join(templates_dir, "jobservice", "env"),
         db_password=db_password,
         ui_secret=ui_secret,
         max_job_workers=max_job_workers,
-        ui_url=ui_url)
+        ui_url=ui_url,
+        verify_remote_cert=verify_remote_cert)
 
 def validate_crt_subj(dirty_subj):
     subj_list = [item for item in dirty_subj.strip().split("/") \

Deploy/templates/jobservice/env

@@ -3,7 +3,10 @@ MYSQL_PORT=3306
 MYSQL_USR=root
 MYSQL_PWD=$db_password
 UI_SECRET=$ui_secret
-HARBOR_URL=$ui_url
+REGISTRY_URL=http://registry:5000
+VERIFY_REMOTE_CERT=$verify_remote_cert
 MAX_JOB_WORKERS=$max_job_workers
 LOG_LEVEL=debug
 GODEBUG=netdns=cgo
+EXT_ENDPOINT=$ui_url
+TOKEN_URL=http://ui

Deploy/templates/ui/env

@@ -3,10 +3,11 @@ MYSQL_PORT=3306
 MYSQL_USR=root
 MYSQL_PWD=$db_password
 REGISTRY_URL=http://registry:5000
+UI_URL=http://ui
 CONFIG_PATH=/etc/ui/app.conf
 HARBOR_REG_URL=$hostname
 HARBOR_ADMIN_PASSWORD=$harbor_admin_password
-HARBOR_URL=$hostname
+HARBOR_URL=$ui_url
 AUTH_MODE=$auth_mode
 LDAP_URL=$ldap_url
 LDAP_BASE_DN=$ldap_basedn
@@ -14,3 +15,5 @@ UI_SECRET=$ui_secret
 SELF_REGISTRATION=$self_registration
 LOG_LEVEL=debug
 GODEBUG=netdns=cgo
+EXT_ENDPOINT=$ui_url
+TOKEN_URL=http://ui

api/jobs/replication.go

@@ -1,16 +1,16 @@
 /*
-    Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
-    Licensed under the Apache License, Version 2.0 (the "License");
+   Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
+   you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
+   You may obtain a copy of the License at
-        
+
-        http://www.apache.org/licenses/LICENSE-2.0
+       http://www.apache.org/licenses/LICENSE-2.0
-        
+
-    Unless required by applicable law or agreed to in writing, software
+   Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
+   distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
+   See the License for the specific language governing permissions and
-    limitations under the License.
+   limitations under the License.
 */
 
 package api
@@ -165,7 +165,7 @@ func getRepoList(projectID int64) ([]string, error) {
 			uiPwd = "Harbor12345"
 		}
 	*/
-	uiURL := config.LocalHarborURL()
+	uiURL := config.LocalUIURL()
 	client := &http.Client{}
 	req, err := http.NewRequest("GET", uiURL+"/api/repositories?project_id="+strconv.Itoa(int(projectID)), nil)
 	if err != nil {

api/project.go

@@ -159,7 +159,7 @@ func (p *ProjectAPI) List() {
 	if len(isPublic) > 0 {
 		public, err = strconv.Atoi(isPublic)
 		if err != nil {
-			log.Errorf("Error parsing public property: %d, error: %v", isPublic, err)
+			log.Errorf("Error parsing public property: %v, error: %v", isPublic, err)
 			p.CustomAbort(http.StatusBadRequest, "invalid project Id")
 		}
 	}

api/repository.go

@@ -255,11 +255,13 @@ func (ra *RepositoryAPI) GetManifests() {
 
 func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repository, err error) {
 	endpoint := os.Getenv("REGISTRY_URL")
+	// TODO read variable from config file
+	insecure := true
 
 	username, password, ok := ra.Ctx.Request.BasicAuth()
 	if ok {
-		credential := auth.NewBasicAuthCredential(username, password)
+		return newRepositoryClient(endpoint, insecure, username, password,
-		return registry.NewRepositoryWithCredential(repoName, endpoint, credential)
+			repoName, "repository", repoName, "pull", "push", "*")
 	}
 
 	username, err = ra.getUsername()
@@ -267,7 +269,8 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo
 		return nil, err
 	}
 
-	return registry.NewRepositoryWithUsername(repoName, endpoint, username)
+	return cache.NewRepositoryClient(endpoint, insecure, username, repoName,
+		"repository", repoName, "pull", "push", "*")
 }
 
 func (ra *RepositoryAPI) getUsername() (string, error) {
@@ -327,3 +330,21 @@ func (ra *RepositoryAPI) GetTopRepos() {
 	ra.Data["json"] = repos
 	ra.ServeJSON()
 }
+
+func newRepositoryClient(endpoint string, insecure bool, username, password, repository, scopeType, scopeName string,
+	scopeActions ...string) (*registry.Repository, error) {
+
+	credential := auth.NewBasicAuthCredential(username, password)
+	authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}

api/target.go

@@ -26,7 +26,7 @@ import (
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/utils"
 	"github.com/vmware/harbor/utils/log"
-	registry_util "github.com/vmware/harbor/utils/registry"
+	"github.com/vmware/harbor/utils/registry"
 	"github.com/vmware/harbor/utils/registry/auth"
 	registry_error "github.com/vmware/harbor/utils/registry/error"
 )
@@ -92,8 +92,10 @@ func (t *TargetAPI) Ping() {
 		password = t.GetString("password")
 	}
 
-	credential := auth.NewBasicAuthCredential(username, password)
+	// TODO read variable from config file
-	registry, err := registry_util.NewRegistryWithCredential(endpoint, credential)
+	insecure := true
+	registry, err := newRegistryClient(endpoint, insecure, username, password,
+		"", "", "")
 	if err != nil {
 		// timeout, dns resolve error, connection refused, etc.
 		if urlErr, ok := err.(*url.Error); ok {
@@ -314,6 +316,23 @@ func (t *TargetAPI) Delete() {
 	}
 }
 
+func newRegistryClient(endpoint string, insecure bool, username, password, scopeType, scopeName string,
+	scopeActions ...string) (*registry.Registry, error) {
+	credential := auth.NewBasicAuthCredential(username, password)
+	authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := registry.NewRegistryWithModifiers(endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
 // ListPolicies ...
 func (t *TargetAPI) ListPolicies() {
 	id := t.GetIDFromURL()

job/config/config.go

@@ -1,16 +1,16 @@
 /*
-    Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
-    Licensed under the Apache License, Version 2.0 (the "License");
+   Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
+   you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
+   You may obtain a copy of the License at
-        
+
-        http://www.apache.org/licenses/LICENSE-2.0
+       http://www.apache.org/licenses/LICENSE-2.0
-        
+
-    Unless required by applicable law or agreed to in writing, software
+   Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
+   distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
+   See the License for the specific language governing permissions and
-    limitations under the License.
+   limitations under the License.
 */
 
 package config
@@ -26,9 +26,11 @@ import (
 const defaultMaxWorkers int = 10
 
 var maxJobWorkers int
-var localURL string
+var localUIURL string
+var localRegURL string
 var logDir string
 var uiSecret string
+var verifyRemoteCert string
 
 func init() {
 	maxWorkersEnv := os.Getenv("MAX_JOB_WORKERS")
@@ -39,9 +41,14 @@ func init() {
 		maxJobWorkers = defaultMaxWorkers
 	}
 
-	localURL = os.Getenv("HARBOR_URL")
+	localRegURL = os.Getenv("REGISTRY_URL")
-	if len(localURL) == 0 {
+	if len(localRegURL) == 0 {
-		localURL = "http://registry:5000/"
+		localRegURL = "http://registry:5000"
+	}
+
+	localUIURL = os.Getenv("UI_URL")
+	if len(localUIURL) == 0 {
+		localUIURL = "http://ui"
 	}
 
 	logDir = os.Getenv("LOG_DIR")
@@ -67,8 +74,15 @@ func init() {
 		panic("UI Secret is not set")
 	}
 
+	verifyRemoteCert = os.Getenv("VERIFY_REMOTE_CERT")
+	if len(verifyRemoteCert) == 0 {
+		verifyRemoteCert = "on"
+	}
+
 	log.Debugf("config: maxJobWorkers: %d", maxJobWorkers)
-	log.Debugf("config: localHarborURL: %s", localURL)
+	log.Debugf("config: localUIURL: %s", localUIURL)
+	log.Debugf("config: localRegURL: %s", localRegURL)
+	log.Debugf("config: verifyRemoteCert: %s", verifyRemoteCert)
 	log.Debugf("config: logDir: %s", logDir)
 	log.Debugf("config: uiSecret: ******")
 }
@@ -78,9 +92,14 @@ func MaxJobWorkers() int {
 	return maxJobWorkers
 }
 
-// LocalHarborURL returns the local registry url, job service will use this URL to pull manifest and repository.
+// LocalUIURL returns the local ui url, job service will use this URL to call API hosted on ui process
-func LocalHarborURL() string {
+func LocalUIURL() string {
-	return localURL
+	return localUIURL
+}
+
+// LocalRegURL returns the local registry url, job service will use this URL to pull image from the registry
+func LocalRegURL() string {
+	return localRegURL
 }
 
 // LogDir returns the absolute path to which the log file will be written
@@ -92,3 +111,8 @@ func LogDir() string {
 func UISecret() string {
 	return uiSecret
 }
+
+// VerifyRemoteCert return the flag to tell jobservice whether or not verify the cert of remote registry
+func VerifyRemoteCert() bool {
+	return verifyRemoteCert != "off"
+}

job/replication/delete.go

@@ -16,13 +16,10 @@
 package replication
 
 import (
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/utils/log"
+	"github.com/vmware/harbor/utils/registry"
+	"github.com/vmware/harbor/utils/registry/auth"
 )
 
 const (
@@ -39,22 +36,35 @@ type Deleter struct {
 	dstUsr string // username ...
 	dstPwd string // username ...
 
+	insecure bool
+
+	dstClient *registry.Repository
+
 	logger *log.Logger
 }
 
 // NewDeleter returns a Deleter
-func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, logger *log.Logger) *Deleter {
+func NewDeleter(repository string, tags []string, dstURL, dstUsr, dstPwd string, insecure bool, logger *log.Logger) (*Deleter, error) {
+	dstCred := auth.NewBasicAuthCredential(dstUsr, dstPwd)
+	dstClient, err := newRepositoryClient(dstURL, insecure, dstCred,
+		repository, "repository", repository, "pull", "push", "*")
+	if err != nil {
+		return nil, err
+	}
+
 	deleter := &Deleter{
 		repository: repository,
 		tags:       tags,
 		dstURL:     dstURL,
 		dstUsr:     dstUsr,
 		dstPwd:     dstPwd,
+		insecure:   insecure,
+		dstClient:  dstClient,
 		logger:     logger,
 	}
-	deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, destination user: %s",
+	deleter.logger.Infof("initialization completed: repository: %s, tags: %v, destination URL: %s, insecure: %v, destination user: %s",
-		deleter.repository, deleter.tags, deleter.dstURL, deleter.dstUsr)
+		deleter.repository, deleter.tags, deleter.dstURL, deleter.insecure, deleter.dstUsr)
-	return deleter
+	return deleter, nil
 }
 
 // Exit ...
@@ -64,25 +74,22 @@ func (d *Deleter) Exit() error {
 
 // Enter deletes repository or tags
 func (d *Deleter) Enter() (string, error) {
-	url := strings.TrimRight(d.dstURL, "/") + "/api/repositories/"
 
-	// delete repository
 	if len(d.tags) == 0 {
-		u := url + "?repo_name=" + d.repository
+		tags, err := d.dstClient.ListTag()
-		if err := del(u, d.dstUsr, d.dstPwd); err != nil {
+		if err != nil {
-			d.logger.Errorf("an error occurred while deleting repository %s on %s with user %s: %v", d.repository, d.dstURL, d.dstUsr, err)
+			d.logger.Errorf("an error occurred while listing tags of repository %s on %s with user %s: %v", d.repository, d.dstURL, d.dstUsr, err)
 			return "", err
 		}
 
-		d.logger.Infof("repository %s on %s has been deleted", d.repository, d.dstURL)
+		d.tags = append(d.tags, tags...)
-
-		return models.JobFinished, nil
 	}
 
-	// delele tags
+	d.logger.Infof("tags %v will be deleted", d.tags)
+
 	for _, tag := range d.tags {
-		u := url + "?repo_name=" + d.repository + "&tag=" + tag
+
-		if err := del(u, d.dstUsr, d.dstPwd); err != nil {
+		if err := d.dstClient.DeleteTag(tag); err != nil {
 			d.logger.Errorf("an error occurred while deleting repository %s:%s on %s with user %s: %v", d.repository, tag, d.dstURL, d.dstUsr, err)
 			return "", err
 		}
@@ -92,28 +99,3 @@ func (d *Deleter) Enter() (string, error) {
 
 	return models.JobFinished, nil
 }
-
-func del(url, username, password string) error {
-	req, err := http.NewRequest("DELETE", url, nil)
-	if err != nil {
-		return err
-	}
-
-	req.SetBasicAuth(username, password)
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode == http.StatusOK {
-		return nil
-	}
-
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-
-	return fmt.Errorf("%d %s", resp.StatusCode, string(b))
-}

job/replication/transfer.go

@@ -17,6 +17,7 @@ package replication
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -61,6 +62,8 @@ type BaseHandler struct {
 	dstUsr string // username ...
 	dstPwd string // password ...
 
+	insecure bool // whether skip secure check when using https
+
 	srcClient *registry.Repository
 	dstClient *registry.Repository
 
@@ -75,10 +78,10 @@ type BaseHandler struct {
 // InitBaseHandler initializes a BaseHandler: creating clients for source and destination registry,
 // listing tags of the repository if parameter tags is nil.
 func InitBaseHandler(repository, srcURL, srcSecret,
-	dstURL, dstUsr, dstPwd string, tags []string, logger *log.Logger) (*BaseHandler, error) {
+	dstURL, dstUsr, dstPwd string, insecure bool, tags []string, logger *log.Logger) (*BaseHandler, error) {
 
-	logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s",
+	logger.Infof("initializing: repository: %s, tags: %v, source URL: %s, destination URL: %s, insecure: %v, destination user: %s",
-		repository, tags, srcURL, dstURL, dstUsr)
+		repository, tags, srcURL, dstURL, insecure, dstUsr)
 
 	base := &BaseHandler{
 		repository:     repository,
@@ -87,6 +90,7 @@ func InitBaseHandler(repository, srcURL, srcSecret,
 		dstURL:         dstURL,
 		dstUsr:         dstUsr,
 		dstPwd:         dstPwd,
+		insecure:       insecure,
 		blobsExistence: make(map[string]bool, 10),
 		logger:         logger,
 	}
@@ -96,15 +100,19 @@ func InitBaseHandler(repository, srcURL, srcSecret,
 	c := &http.Cookie{Name: models.UISecretCookie, Value: srcSecret}
 	srcCred := auth.NewCookieCredential(c)
 	//	srcCred := auth.NewBasicAuthCredential("admin", "Harbor12345")
-	srcClient, err := registry.NewRepositoryWithCredential(base.repository, base.srcURL, srcCred)
+	srcClient, err := newRepositoryClient(base.srcURL, base.insecure, srcCred,
+		base.repository, "repository", base.repository, "pull", "push", "*")
 	if err != nil {
+		base.logger.Errorf("an error occurred while creating source repository client: %v", err)
 		return nil, err
 	}
 	base.srcClient = srcClient
 
 	dstCred := auth.NewBasicAuthCredential(base.dstUsr, base.dstPwd)
-	dstClient, err := registry.NewRepositoryWithCredential(base.repository, base.dstURL, dstCred)
+	dstClient, err := newRepositoryClient(base.dstURL, base.insecure, dstCred,
+		base.repository, "repository", base.repository, "pull", "push", "*")
 	if err != nil {
+		base.logger.Errorf("an error occurred while creating destination repository client: %v", err)
 		return nil, err
 	}
 	base.dstClient = dstClient
@@ -112,13 +120,14 @@ func InitBaseHandler(repository, srcURL, srcSecret,
 	if len(base.tags) == 0 {
 		tags, err := base.srcClient.ListTag()
 		if err != nil {
+			base.logger.Errorf("an error occurred while listing tags for source repository: %v", err)
 			return nil, err
 		}
 		base.tags = tags
 	}
 
-	base.logger.Infof("initialization completed: project: %s, repository: %s, tags: %v, source URL: %s, destination URL: %s, destination user: %s",
+	base.logger.Infof("initialization completed: project: %s, repository: %s, tags: %v, source URL: %s, destination URL: %s, insecure: %v, destination user: %s",
-		base.project, base.repository, base.tags, base.srcURL, base.dstURL, base.dstUsr)
+		base.project, base.repository, base.tags, base.srcURL, base.dstURL, base.insecure, base.dstUsr)
 
 	return base, nil
 }
@@ -186,7 +195,16 @@ func (c *Checker) projectExist() (exist, canWrite bool, err error) {
 	}
 
 	req.SetBasicAuth(c.dstUsr, c.dstPwd)
-	resp, err := http.DefaultClient.Do(req)
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: c.insecure,
+			},
+		},
+	}
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return
 	}
@@ -255,27 +273,38 @@ func (c *Checker) createProject() error {
 	}
 
 	req.SetBasicAuth(c.dstUsr, c.dstPwd)
-	resp, err := http.DefaultClient.Do(req)
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: c.insecure,
+			},
+		},
+	}
+
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
 
-	if resp.StatusCode != http.StatusCreated {
+	// version 0.1.1's reponse code is 200
-		if resp.StatusCode == http.StatusConflict {
+	if resp.StatusCode == http.StatusCreated ||
-			return ErrConflict
+		resp.StatusCode == http.StatusOK {
-		}
+		return nil
-
-		defer resp.Body.Close()
-		message, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			c.logger.Errorf("an error occurred while reading message from response: %v", err)
-		}
-
-		return fmt.Errorf("failed to create project %s on %s with user %s: %d %s",
-			c.project, c.dstURL, c.dstUsr, resp.StatusCode, string(message))
 	}
 
-	return nil
+	if resp.StatusCode == http.StatusConflict {
+		return ErrConflict
+	}
+
+	defer resp.Body.Close()
+	message, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		c.logger.Errorf("an error occurred while reading message from response: %v", err)
+	}
+
+	return fmt.Errorf("failed to create project %s on %s with user %s: %d %s",
+		c.project, c.dstURL, c.dstUsr, resp.StatusCode, string(message))
 }
 
 // ManifestPuller pulls the manifest of a tag. And if no tag needs to be pulled,
@@ -416,3 +445,34 @@ func (m *ManifestPusher) Enter() (string, error) {
 
 	return StatePullManifest, nil
 }
+
+func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, repository, scopeType, scopeName string,
+	scopeActions ...string) (*registry.Repository, error) {
+
+	authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	uam := &userAgentModifier{
+		userAgent: "harbor-registry-client",
+	}
+
+	client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store, uam)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+type userAgentModifier struct {
+	userAgent string
+}
+
+// Modify adds user-agent header to the request
+func (u *userAgentModifier) Modify(req *http.Request) error {
+	req.Header.Set(http.CanonicalHeaderKey("User-Agent"), u.userAgent)
+	return nil
+}

job/statemachine.go

@@ -1,16 +1,16 @@
 /*
-    Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
-    Licensed under the Apache License, Version 2.0 (the "License");
+   Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
+   you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
+   You may obtain a copy of the License at
-        
+
-        http://www.apache.org/licenses/LICENSE-2.0
+       http://www.apache.org/licenses/LICENSE-2.0
-        
+
-    Unless required by applicable law or agreed to in writing, software
+   Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
+   distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
+   See the License for the specific language governing permissions and
-    limitations under the License.
+   limitations under the License.
 */
 
 package job
@@ -38,6 +38,7 @@ type RepJobParm struct {
 	Tags           []string
 	Enabled        int
 	Operation      string
+	Insecure       bool
 }
 
 // SM is the state machine to handle job, it handles one job at a time.
@@ -205,11 +206,12 @@ func (sm *SM) Reset(jid int64) error {
 		return fmt.Errorf("The policy doesn't exist in DB, policy id:%d", job.PolicyID)
 	}
 	sm.Parms = &RepJobParm{
-		LocalRegURL: config.LocalHarborURL(),
+		LocalRegURL: config.LocalRegURL(),
 		Repository:  job.Repository,
 		Tags:        job.TagList,
 		Enabled:     policy.Enabled,
 		Operation:   job.Operation,
+		Insecure:    !config.VerifyRemoteCert(),
 	}
 	if policy.Enabled == 0 {
 		//worker will cancel this job
@@ -260,7 +262,7 @@ func (sm *SM) Reset(jid int64) error {
 func addImgTransferTransition(sm *SM) error {
 	base, err := replication.InitBaseHandler(sm.Parms.Repository, sm.Parms.LocalRegURL, config.UISecret(),
 		sm.Parms.TargetURL, sm.Parms.TargetUsername, sm.Parms.TargetPassword,
-		sm.Parms.Tags, sm.Logger)
+		sm.Parms.Insecure, sm.Parms.Tags, sm.Logger)
 	if err != nil {
 		return err
 	}
@@ -274,8 +276,11 @@ func addImgTransferTransition(sm *SM) error {
 }
 
 func addImgDeleteTransition(sm *SM) error {
-	deleter := replication.NewDeleter(sm.Parms.Repository, sm.Parms.Tags, sm.Parms.TargetURL,
+	deleter, err := replication.NewDeleter(sm.Parms.Repository, sm.Parms.Tags, sm.Parms.TargetURL,
-		sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Logger)
+		sm.Parms.TargetUsername, sm.Parms.TargetPassword, sm.Parms.Insecure, sm.Logger)
+	if err != nil {
+		return err
+	}
 
 	sm.AddTransition(models.JobRunning, replication.StateDelete, deleter)
 	sm.AddTransition(replication.StateDelete, models.JobFinished, &StatusUpdater{DummyHandler{JobID: sm.JobID}, models.JobFinished})

service/cache/cache.go

@@ -21,17 +21,16 @@ import (
 
 	"github.com/vmware/harbor/utils/log"
 	"github.com/vmware/harbor/utils/registry"
+	"github.com/vmware/harbor/utils/registry/auth"
 
 	"github.com/astaxie/beego/cache"
 )
 
 var (
 	// Cache is the global cache in system.
-	Cache             cache.Cache
+	Cache    cache.Cache
-	endpoint          string
+	endpoint string
-	username          string
+	username string
-	registryClient    *registry.Registry
-	repositoryClients map[string]*registry.Repository
 )
 
 const catalogKey string = "catalog"
@@ -45,23 +44,18 @@ func init() {
 
 	endpoint = os.Getenv("REGISTRY_URL")
 	username = "admin"
-	repositoryClients = make(map[string]*registry.Repository, 10)
 }
 
 // RefreshCatalogCache calls registry's API to get repository list and write it to cache.
 func RefreshCatalogCache() error {
 	log.Debug("refreshing catalog cache...")
 
-	if registryClient == nil {
+	registryClient, err := NewRegistryClient(endpoint, true, username,
-		var err error
+		"registry", "catalog", "*")
-		registryClient, err = registry.NewRegistryWithUsername(endpoint, username)
+	if err != nil {
-		if err != nil {
+		return err
-			log.Errorf("error occurred while initializing registry client used by cache: %v", err)
-			return err
-		}
 	}
 
-	var err error
 	rs, err := registryClient.Catalog()
 	if err != nil {
 		return err
@@ -70,15 +64,13 @@ func RefreshCatalogCache() error {
 	repos := []string{}
 
 	for _, repo := range rs {
-		rc, ok := repositoryClients[repo]
+		rc, err := NewRepositoryClient(endpoint, true, username,
-		if !ok {
+			repo, "repository", repo, "pull", "push", "*")
-			rc, err = registry.NewRepositoryWithUsername(repo, endpoint, username)
+		if err != nil {
-			if err != nil {
+			log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err)
-				log.Errorf("error occurred while initializing repository client used by cache: %s %v", repo, err)
+			continue
-				continue
-			}
-			repositoryClients[repo] = rc
 		}
+
 		tags, err := rc.ListTag()
 		if err != nil {
 			log.Errorf("error occurred while list tag for %s: %v", repo, err)
@@ -112,3 +104,38 @@ func GetRepoFromCache() ([]string, error) {
 	}
 	return result.([]string), nil
 }
+
+// NewRegistryClient ...
+func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scopeName string,
+	scopeActions ...string) (*registry.Registry, error) {
+	authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := registry.NewRegistryWithModifiers(endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+// NewRepositoryClient ...
+func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string,
+	scopeActions ...string) (*registry.Repository, error) {
+
+	authorizer := auth.NewUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}

service/notification.go

@@ -25,7 +25,6 @@ import (
 	"github.com/vmware/harbor/models"
 	"github.com/vmware/harbor/service/cache"
 	"github.com/vmware/harbor/utils/log"
-	"github.com/vmware/harbor/utils/registry"
 
 	"github.com/astaxie/beego"
 )
@@ -57,7 +56,7 @@ func (n *NotificationHandler) Post() {
 			matched = false
 		}
 		if matched && (strings.HasPrefix(e.Request.UserAgent, "docker") ||
-			strings.ToLower(strings.TrimSpace(e.Request.UserAgent)) == strings.ToLower(registry.UserAgent)) {
+			strings.ToLower(strings.TrimSpace(e.Request.UserAgent)) == "harbor-registry-client") {
 			username = e.Actor.Name
 			action = e.Action
 			repo = e.Target.Repository

static/resources/js/components/replication/create-policy.directive.html

@@ -67,19 +67,13 @@
             <div class="form-group col-md-12 form-group-custom">
               <label for="username" class="col-md-3 control-label">// 'username' | tr //:</label>
               <div class="col-md-9">
-                <input type="text" class="form-control" id="username" ng-model="replication.destination.username" name="uUsername" ng-value="vm.username" required ng-disabled="!vm.targetEditable">                 
+                <input type="text" class="form-control" id="username" ng-model="replication.destination.username" name="uUsername" ng-value="vm.username" ng-disabled="!vm.targetEditable">                 
-                <div ng-messages="form.$submitted && form.uUsername.$error">
-                  <span ng-message="required">// 'username_is_required' | tr //</span>
-                </div>
               </div> 
             </div>
             <div class="form-group col-md-12 form-group-custom">
               <label for="password" class="col-md-3 control-label">// 'password' | tr //:</label>
               <div class="col-md-9">
-                <input type="password" class="form-control" id="password" ng-model="replication.destination.password" name="uPassword" ng-value="vm.password" required ng-disabled="!vm.targetEditable">                 
+                <input type="password" class="form-control" id="password" ng-model="replication.destination.password" name="uPassword" ng-value="vm.password" ng-disabled="!vm.targetEditable">                 
-                <div ng-messages="form.$submitted && form.uPassword.$error">
-                  <span ng-message="required">// 'password_is_required' | tr //</span>
-                </div>
               </div> 
             </div>
             <div class="form-group col-md-12 form-group-custom">
@@ -97,4 +91,4 @@
     </form>
     </div><!-- /.modal-content -->
   </div><!-- /.modal-dialog -->
-</div><!-- /.modal -->
+</div><!-- /.modal -->

static/resources/js/components/system-management/create-destination.directive.html

@@ -29,19 +29,13 @@
             <div class="form-group col-md-12 form-group-custom">
               <label for="username" class="col-md-3 control-label">// 'username' | tr //:</label>
               <div class="col-md-9">
-                <input type="text" class="form-control" id="username" ng-model="destination.username" name="uUsername" required ng-disabled="!vm.editable">                 
+                <input type="text" class="form-control" id="username" ng-model="destination.username" name="uUsername" ng-disabled="!vm.editable">                 
-                <div ng-messages="form.$submitted && form.uUsername.$error">
-                  <span ng-message="required">// 'username_is_required' | tr //</span>
-                </div>
               </div> 
             </div>
             <div class="form-group col-md-12 form-group-custom">
               <label for="password" class="col-md-3 control-label">// 'password' | tr //:</label>
               <div class="col-md-9">
-                <input type="password" class="form-control" id="password" ng-model="destination.password" name="uPassword" required ng-disabled="!vm.editable">                 
+                <input type="password" class="form-control" id="password" ng-model="destination.password" name="uPassword" ng-disabled="!vm.editable">                 
-                <div ng-messages="form.$submitted && form.uPassword.$error">
-                  <span ng-message="required">// 'password_is_required' | tr //</span>
-                </div>
               </div> 
             </div>
             <div class="form-group col-md-12 form-group-custom">
@@ -59,4 +53,4 @@
       </div><!-- /.modal-content -->
     </form>
   </div><!-- /.modal-dialog -->
-</div><!-- /.modal -->
+</div><!-- /.modal -->

utils/registry/auth/authorizer.go

@@ -16,40 +16,63 @@
 package auth
 
 import (
+	"crypto/tls"
+	"fmt"
 	"net/http"
 
 	au "github.com/docker/distribution/registry/client/auth"
+	"github.com/vmware/harbor/utils/registry/utils"
 )
 
-// Handler authorizes requests according to the schema
+// Authorizer authorizes requests according to the schema
-type Handler interface {
+type Authorizer interface {
 	// Scheme : basic, bearer
 	Scheme() string
-	//AuthorizeRequest adds basic auth or token auth to the header of request
+	//Authorize adds basic auth or token auth to the header of request
-	AuthorizeRequest(req *http.Request, params map[string]string) error
+	Authorize(req *http.Request, params map[string]string) error
 }
 
-// RequestAuthorizer holds a handler list, which will authorize request.
+// AuthorizerStore holds a authorizer list, which will authorize request.
-// Implements interface RequestModifier
+// And it implements interface Modifier
-type RequestAuthorizer struct {
+type AuthorizerStore struct {
-	handlers   []Handler
+	authorizers []Authorizer
-	challenges []au.Challenge
+	challenges  []au.Challenge
 }
 
-// NewRequestAuthorizer ...
+// NewAuthorizerStore ...
-func NewRequestAuthorizer(handlers []Handler, challenges []au.Challenge) *RequestAuthorizer {
+func NewAuthorizerStore(endpoint string, insecure bool, authorizers ...Authorizer) (*AuthorizerStore, error) {
-	return &RequestAuthorizer{
+	endpoint = utils.FormatEndpoint(endpoint)
-		handlers:   handlers,
+
-		challenges: challenges,
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: insecure,
+			},
+		},
 	}
+
+	resp, err := client.Get(buildPingURL(endpoint))
+	if err != nil {
+		return nil, err
+	}
+
+	challenges := ParseChallengeFromResponse(resp)
+	return &AuthorizerStore{
+		authorizers: authorizers,
+		challenges:  challenges,
+	}, nil
 }
 
-// ModifyRequest adds authorization to the request
+func buildPingURL(endpoint string) string {
-func (r *RequestAuthorizer) ModifyRequest(req *http.Request) error {
+	return fmt.Sprintf("%s/v2/", endpoint)
-	for _, challenge := range r.challenges {
+}
-		for _, handler := range r.handlers {
+
-			if handler.Scheme() == challenge.Scheme {
+// Modify adds authorization to the request
-				if err := handler.AuthorizeRequest(req, challenge.Parameters); err != nil {
+func (a *AuthorizerStore) Modify(req *http.Request) error {
+	for _, challenge := range a.challenges {
+		for _, authorizer := range a.authorizers {
+			if authorizer.Scheme() == challenge.Scheme {
+				if err := authorizer.Authorize(req, challenge.Parameters); err != nil {
 					return err
 				}
 			}

utils/registry/auth/challenge.go

@@ -19,14 +19,11 @@ import (
 	"net/http"
 
 	au "github.com/docker/distribution/registry/client/auth"
-	"github.com/vmware/harbor/utils/log"
 )
 
 // ParseChallengeFromResponse ...
 func ParseChallengeFromResponse(resp *http.Response) []au.Challenge {
 	challenges := au.ResponseChallenges(resp)
 
-	log.Debugf("challenges: %v", challenges)
-
 	return challenges
 }

utils/registry/auth/tokenauthorizer.go

@@ -16,11 +16,13 @@
 package auth
 
 import (
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -42,8 +44,8 @@ func (s *scope) string() string {
 
 type tokenGenerator func(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error)
 
-// Implements interface Handler
+// Implements interface Authorizer
-type tokenHandler struct {
+type tokenAuthorizer struct {
 	scope     *scope
 	tg        tokenGenerator
 	cache     string     // cached token
@@ -53,12 +55,12 @@ type tokenHandler struct {
 }
 
 // Scheme returns the scheme that the handler can handle
-func (t *tokenHandler) Scheme() string {
+func (t *tokenAuthorizer) Scheme() string {
 	return "bearer"
 }
 
 // AuthorizeRequest will add authorization header which contains a token before the request is sent
-func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]string) error {
+func (t *tokenAuthorizer) Authorize(req *http.Request, params map[string]string) error {
 	var scopes []*scope
 	var token string
 
@@ -100,26 +102,23 @@ func (t *tokenHandler) AuthorizeRequest(req *http.Request, params map[string]str
 
 		if !hasFrom {
 			t.updateCachedToken(to, expiresIn, issuedAt)
-			log.Debug("add token to cache")
 		}
 	} else {
 		token = cachedToken
-		log.Debug("get token from cache")
 	}
 
 	req.Header.Add(http.CanonicalHeaderKey("Authorization"), fmt.Sprintf("Bearer %s", token))
-	log.Debugf("add token to request: %s %s", req.Method, req.URL.String())
 
 	return nil
 }
 
-func (t *tokenHandler) getCachedToken() (string, int, *time.Time) {
+func (t *tokenAuthorizer) getCachedToken() (string, int, *time.Time) {
 	t.Lock()
 	defer t.Unlock()
 	return t.cache, t.expiresIn, t.issuedAt
 }
 
-func (t *tokenHandler) updateCachedToken(token string, expiresIn int, issuedAt *time.Time) {
+func (t *tokenAuthorizer) updateCachedToken(token string, expiresIn int, issuedAt *time.Time) {
 	t.Lock()
 	defer t.Unlock()
 	t.cache = token
@@ -127,38 +126,45 @@ func (t *tokenHandler) updateCachedToken(token string, expiresIn int, issuedAt *
 	t.issuedAt = issuedAt
 }
 
-// Implements interface Handler
+// Implements interface Authorizer
-type standardTokenHandler struct {
+type standardTokenAuthorizer struct {
-	tokenHandler
+	tokenAuthorizer
 	client     *http.Client
 	credential Credential
 }
 
-// NewStandardTokenHandler returns a standard token handler. The handler will request a token
+// NewStandardTokenAuthorizer returns a standard token authorizer. The authorizer will request a token
 // from token server and add it to the origin request
-// TODO deal with https
+func NewStandardTokenAuthorizer(credential Credential, insecure bool, scopeType, scopeName string, scopeActions ...string) Authorizer {
-func NewStandardTokenHandler(credential Credential, scopeType, scopeName string, scopeActions ...string) Handler {
+	t := &http.Transport{
-	handler := &standardTokenHandler{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: insecure,
+		},
+	}
+
+	authorizer := &standardTokenAuthorizer{
 		client: &http.Client{
-			Transport: http.DefaultTransport,
+			Transport: t,
 		},
 		credential: credential,
 	}
 
 	if len(scopeType) != 0 || len(scopeName) != 0 {
-		handler.scope = &scope{
+		authorizer.scope = &scope{
 			Type:    scopeType,
 			Name:    scopeName,
 			Actions: scopeActions,
 		}
 	}
 
-	handler.tg = handler.generateToken
+	authorizer.tg = authorizer.generateToken
 
-	return handler
+	return authorizer
 }
 
-func (s *standardTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
+func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
+	realm = tokenURL(realm)
+
 	u, err := url.Parse(realm)
 	if err != nil {
 		return
@@ -217,37 +223,50 @@ func (s *standardTokenHandler) generateToken(realm, service string, scopes []str
 		}
 	}
 
-	log.Debug("get token from token server")
-
 	return
 }
 
+// when the registry client is used inside Harbor, the token request
+// can be posted to token service directly rather than going through nginx.
+// this solution can resolve two problems:
+// 1. performance issue
+// 2. the realm field returned by registry is an IP which can not reachable
+// inside Harbor
+func tokenURL(realm string) string {
+	extEndpoint := os.Getenv("EXT_ENDPOINT")
+	tokenURL := os.Getenv("TOKEN_URL")
+	if len(extEndpoint) != 0 && len(tokenURL) != 0 &&
+		strings.Contains(realm, extEndpoint) {
+		realm = strings.TrimRight(tokenURL, "/") + "/service/token"
+	}
+	return realm
+}
+
 // Implements interface Handler
-type usernameTokenHandler struct {
+type usernameTokenAuthorizer struct {
-	tokenHandler
+	tokenAuthorizer
 	username string
 }
 
-// NewUsernameTokenHandler returns a handler which will generate a token according to
+// NewUsernameTokenAuthorizer returns a authorizer which will generate a token according to
 // the user's privileges
-func NewUsernameTokenHandler(username string, scopeType, scopeName string, scopeActions ...string) Handler {
+func NewUsernameTokenAuthorizer(username string, scopeType, scopeName string, scopeActions ...string) Authorizer {
-	handler := &usernameTokenHandler{
+	authorizer := &usernameTokenAuthorizer{
 		username: username,
 	}
 
-	handler.scope = &scope{
+	authorizer.scope = &scope{
 		Type:    scopeType,
 		Name:    scopeName,
 		Actions: scopeActions,
 	}
 
-	handler.tg = handler.generateToken
+	authorizer.tg = authorizer.generateToken
 
-	return handler
+	return authorizer
 }
 
-func (u *usernameTokenHandler) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
+func (u *usernameTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
 	token, expiresIn, issuedAt, err = token_util.GenTokenForUI(u.username, service, scopes)
-	log.Debug("get token by calling GenTokenForUI directly")
 	return
 }

utils/registry/modifier.go

@@ -0,0 +1,25 @@
+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package registry
+
+import (
+	"net/http"
+)
+
+// Modifier modifies request
+type Modifier interface {
+	Modify(*http.Request) error
+}

utils/registry/registry.go

@@ -3,9 +3,7 @@
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
-
        http://www.apache.org/licenses/LICENSE-2.0
-
    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,21 +14,15 @@
 package registry
 
 import (
+	"crypto/tls"
 	"encoding/json"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 	"strings"
 
-	"github.com/vmware/harbor/utils/log"
-	"github.com/vmware/harbor/utils/registry/auth"
 	registry_error "github.com/vmware/harbor/utils/registry/error"
-)
+	"github.com/vmware/harbor/utils/registry/utils"
-
-const (
-	// UserAgent is used to decorate the request so it can be identified by webhook.
-	UserAgent string = "registry-client"
 )
 
 // Registry holds information of a registry entity
@@ -41,9 +33,7 @@ type Registry struct {
 
 // NewRegistry returns an instance of registry
 func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
-	endpoint = strings.TrimRight(endpoint, "/")
+	u, err := utils.ParseEndpoint(endpoint)
-
-	u, err := url.Parse(endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -53,105 +43,81 @@ func NewRegistry(endpoint string, client *http.Client) (*Registry, error) {
 		client:   client,
 	}
 
-	log.Debugf("initialized a registry client: %s", endpoint)
-
 	return registry, nil
 }
 
-// NewRegistryWithUsername returns a Registry instance which will authorize the request
+// NewRegistryWithModifiers returns an instance of Registry according to the modifiers
-// according to the privileges of user
+func NewRegistryWithModifiers(endpoint string, insecure bool, modifiers ...Modifier) (*Registry, error) {
-func NewRegistryWithUsername(endpoint, username string) (*Registry, error) {
+	u, err := utils.ParseEndpoint(endpoint)
-	endpoint = strings.TrimRight(endpoint, "/")
-
-	u, err := url.Parse(endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	client, err := newClient(endpoint, username, nil, "registry", "catalog", "*")
+	t := &http.Transport{
-	if err != nil {
+		TLSClientConfig: &tls.Config{
-		return nil, err
+			InsecureSkipVerify: insecure,
+		},
 	}
 
-	registry := &Registry{
+	transport := NewTransport(t, modifiers...)
+
+	return &Registry{
 		Endpoint: u,
-		client:   client,
+		client: &http.Client{
-	}
+			Transport: transport,
-
+		},
-	log.Debugf("initialized a registry client with username: %s %s", endpoint, username)
+	}, nil
-
-	return registry, nil
-}
-
-// NewRegistryWithCredential returns a Registry instance which associate to a crendential.
-// And Credential is essentially a decorator for client to docorate the request before sending it to the registry.
-func NewRegistryWithCredential(endpoint string, credential auth.Credential) (*Registry, error) {
-	endpoint = strings.TrimSpace(endpoint)
-	endpoint = strings.TrimRight(endpoint, "/")
-	if !strings.HasPrefix(endpoint, "http://") &&
-		!strings.HasPrefix(endpoint, "https://") {
-		endpoint = "http://" + endpoint
-	}
-
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	client, err := newClient(endpoint, "", credential, "", "", "")
-	if err != nil {
-		return nil, err
-	}
-
-	registry := &Registry{
-		Endpoint: u,
-		client:   client,
-	}
-
-	log.Debugf("initialized a registry client with credential: %s", endpoint)
-
-	return registry, nil
 }
 
 // Catalog ...
 func (r *Registry) Catalog() ([]string, error) {
 	repos := []string{}
+	suffix := "/v2/_catalog?n=1000"
+	var url string
 
-	req, err := http.NewRequest("GET", buildCatalogURL(r.Endpoint.String()), nil)
+	for len(suffix) > 0 {
-	if err != nil {
+		url = r.Endpoint.String() + suffix
-		return repos, err
-	}
 
-	resp, err := r.client.Do(req)
+		req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
+		if err != nil {
-		return repos, parseError(err)
+			return repos, err
-	}
+		}
+		resp, err := r.client.Do(req)
+		if err != nil {
+			return nil, parseError(err)
+		}
 
-	defer resp.Body.Close()
+		defer resp.Body.Close()
-
+		b, err := ioutil.ReadAll(resp.Body)
-	b, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
-	if err != nil {
-		return repos, err
-	}
-
-	if resp.StatusCode == http.StatusOK {
-		catalogResp := struct {
-			Repositories []string `json:"repositories"`
-		}{}
-
-		if err := json.Unmarshal(b, &catalogResp); err != nil {
 			return repos, err
 		}
 
-		repos = catalogResp.Repositories
+		if resp.StatusCode == http.StatusOK {
+			catalogResp := struct {
+				Repositories []string `json:"repositories"`
+			}{}
 
-		return repos, nil
+			if err := json.Unmarshal(b, &catalogResp); err != nil {
-	}
+				return repos, err
+			}
 
-	return repos, &registry_error.Error{
+			repos = append(repos, catalogResp.Repositories...)
-		StatusCode: resp.StatusCode,
+			//Link: </v2/_catalog?last=library%2Fhello-world-25&n=100>; rel="next"
-		Detail:     string(b),
+			link := resp.Header.Get("Link")
+			if strings.HasSuffix(link, `rel="next"`) && strings.Index(link, "<") >= 0 && strings.Index(link, ">") >= 0 {
+				suffix = link[strings.Index(link, "<")+1 : strings.Index(link, ">")]
+			} else {
+				suffix = ""
+			}
+		} else {
+			return repos, &registry_error.Error{
+				StatusCode: resp.StatusCode,
+				Detail:     string(b),
+			}
+		}
 	}
+	return repos, nil
 }
 
 // Ping ...
@@ -163,16 +129,6 @@ func (r *Registry) Ping() error {
 
 	resp, err := r.client.Do(req)
 	if err != nil {
-		//		if urlErr, ok := err.(*url.Error); ok {
-		//			if regErr, ok := urlErr.Err.(*registry_error.Error); ok {
-		//				return &registry_error.Error{
-		//					StatusCode: regErr.StatusCode,
-		//					Detail:     regErr.Detail,
-		//				}
-		//			}
-		//			return urlErr.Err
-		//		}
-
 		return parseError(err)
 	}
 
@@ -192,36 +148,3 @@ func (r *Registry) Ping() error {
 		Detail:     string(b),
 	}
 }
-
-func buildCatalogURL(endpoint string) string {
-	return fmt.Sprintf("%s/v2/_catalog", endpoint)
-}
-
-func newClient(endpoint, username string, credential auth.Credential,
-	scopeType, scopeName string, scopeActions ...string) (*http.Client, error) {
-
-	endpoint = strings.TrimRight(endpoint, "/")
-	resp, err := http.Get(buildPingURL(endpoint))
-	if err != nil {
-		return nil, err
-	}
-
-	var handlers []auth.Handler
-	var handler auth.Handler
-	if credential != nil {
-		handler = auth.NewStandardTokenHandler(credential, scopeType, scopeName, scopeActions...)
-	} else {
-		handler = auth.NewUsernameTokenHandler(username, scopeType, scopeName, scopeActions...)
-	}
-
-	handlers = append(handlers, handler)
-
-	challenges := auth.ParseChallengeFromResponse(resp)
-	authorizer := auth.NewRequestAuthorizer(handlers, challenges)
-	headerModifier := NewHeaderModifier(map[string]string{http.CanonicalHeaderKey("User-Agent"): UserAgent})
-
-	transport := NewTransport(http.DefaultTransport, []RequestModifier{authorizer, headerModifier})
-	return &http.Client{
-		Transport: transport,
-	}, nil
-}

utils/registry/repository.go

@@ -17,6 +17,7 @@ package registry
 
 import (
 	"bytes"
+	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -28,9 +29,9 @@ import (
 
 	"github.com/docker/distribution/manifest/schema1"
 	"github.com/docker/distribution/manifest/schema2"
-	"github.com/vmware/harbor/utils/log"
+
-	"github.com/vmware/harbor/utils/registry/auth"
 	registry_error "github.com/vmware/harbor/utils/registry/error"
+	"github.com/vmware/harbor/utils/registry/utils"
 )
 
 // Repository holds information of a repository entity
@@ -40,14 +41,11 @@ type Repository struct {
 	client   *http.Client
 }
 
-// TODO add agent to header of request, notifications need it
-
 // NewRepository returns an instance of Repository
 func NewRepository(name, endpoint string, client *http.Client) (*Repository, error) {
 	name = strings.TrimSpace(name)
-	endpoint = strings.TrimRight(endpoint, "/")
 
-	u, err := url.Parse(endpoint)
+	u, err := utils.ParseEndpoint(endpoint)
 	if err != nil {
 		return nil, err
 	}
@@ -61,55 +59,30 @@ func NewRepository(name, endpoint string, client *http.Client) (*Repository, err
 	return repository, nil
 }
 
-// NewRepositoryWithCredential returns a Repository instance which will authorize the request
+// NewRepositoryWithModifiers returns an instance of Repository according to the modifiers
-// according to the credenttial
+func NewRepositoryWithModifiers(name, endpoint string, insecure bool, modifiers ...Modifier) (*Repository, error) {
-func NewRepositoryWithCredential(name, endpoint string, credential auth.Credential) (*Repository, error) {
 	name = strings.TrimSpace(name)
-	endpoint = strings.TrimRight(endpoint, "/")
 
-	u, err := url.Parse(endpoint)
+	u, err := utils.ParseEndpoint(endpoint)
 	if err != nil {
 		return nil, err
 	}
 
-	client, err := newClient(endpoint, "", credential, "repository", name, "pull", "push")
+	t := &http.Transport{
-	if err != nil {
+		TLSClientConfig: &tls.Config{
-		return nil, err
+			InsecureSkipVerify: insecure,
+		},
 	}
 
-	repository := &Repository{
+	transport := NewTransport(t, modifiers...)
+
+	return &Repository{
 		Name:     name,
 		Endpoint: u,
-		client:   client,
+		client: &http.Client{
-	}
+			Transport: transport,
-
+		},
-	log.Debugf("initialized a repository client with credential: %s %s", endpoint, name)
+	}, nil
-
-	return repository, nil
-}
-
-// NewRepositoryWithUsername returns a Repository instance which will authorize the request
-// according to the privileges of user
-func NewRepositoryWithUsername(name, endpoint, username string) (*Repository, error) {
-	name = strings.TrimSpace(name)
-	endpoint = strings.TrimRight(endpoint, "/")
-
-	u, err := url.Parse(endpoint)
-	if err != nil {
-		return nil, err
-	}
-
-	client, err := newClient(endpoint, username, nil, "repository", name, "pull", "push")
-
-	repository := &Repository{
-		Name:     name,
-		Endpoint: u,
-		client:   client,
-	}
-
-	log.Debugf("initialized a repository client with username: %s %s %s", endpoint, name, username)
-
-	return repository, nil
 }
 
 func parseError(err error) error {

utils/registry/repository_test.go

@@ -26,7 +26,7 @@ import (
 	"time"
 
 	"github.com/vmware/harbor/utils/registry/auth"
-	"github.com/vmware/harbor/utils/registry/error"
+	registry_error "github.com/vmware/harbor/utils/registry/error"
 )
 
 var (
@@ -139,7 +139,8 @@ func serveToken(w http.ResponseWriter, r *http.Request) {
 }
 
 func TestListTag(t *testing.T) {
-	client, err := NewRepositoryWithCredential(repo, registryServer.URL, credential)
+	client, err := newRepositoryClient(registryServer.URL, true, credential,
+		repo, "repository", repo, "pull", "push", "*")
 	if err != nil {
 		t.Error(err)
 	}
@@ -158,13 +159,14 @@ func TestListTag(t *testing.T) {
 
 func TestListTagWithInvalidCredential(t *testing.T) {
 	credential := auth.NewBasicAuthCredential(username, "wrong_password")
-	client, err := NewRepositoryWithCredential(repo, registryServer.URL, credential)
+	client, err := newRepositoryClient(registryServer.URL, true, credential,
+		repo, "repository", repo, "pull", "push", "*")
 	if err != nil {
 		t.Error(err)
 	}
 
 	if _, err = client.ListTag(); err != nil {
-		e, ok := err.(*error.Error)
+		e, ok := err.(*registry_error.Error)
 		if ok && e.StatusCode == http.StatusUnauthorized {
 			return
 		}
@@ -173,3 +175,20 @@ func TestListTagWithInvalidCredential(t *testing.T) {
 		return
 	}
 }
+
+func newRepositoryClient(endpoint string, insecure bool, credential auth.Credential, repository, scopeType, scopeName string,
+	scopeActions ...string) (*Repository, error) {
+
+	authorizer := auth.NewStandardTokenAuthorizer(credential, insecure, scopeType, scopeName, scopeActions...)
+
+	store, err := auth.NewAuthorizerStore(endpoint, true, authorizer)
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := NewRepositoryWithModifiers(repository, endpoint, insecure, store)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}

utils/registry/transport.go

@@ -21,39 +21,14 @@ import (
 	"github.com/vmware/harbor/utils/log"
 )
 
-// RequestModifier modifies request
-type RequestModifier interface {
-	ModifyRequest(*http.Request) error
-}
-
-// HeaderModifier adds headers to request
-type HeaderModifier struct {
-	headers map[string]string
-}
-
-// NewHeaderModifier ...
-func NewHeaderModifier(headers map[string]string) *HeaderModifier {
-	return &HeaderModifier{
-		headers: headers,
-	}
-}
-
-// ModifyRequest adds headers to the request
-func (h *HeaderModifier) ModifyRequest(req *http.Request) error {
-	for key, value := range h.headers {
-		req.Header.Add(key, value)
-	}
-	return nil
-}
-
 // Transport holds information about base transport and modifiers
 type Transport struct {
 	transport http.RoundTripper
-	modifiers []RequestModifier
+	modifiers []Modifier
 }
 
 // NewTransport ...
-func NewTransport(transport http.RoundTripper, modifiers []RequestModifier) *Transport {
+func NewTransport(transport http.RoundTripper, modifiers ...Modifier) *Transport {
 	return &Transport{
 		transport: transport,
 		modifiers: modifiers,
@@ -63,7 +38,7 @@ func NewTransport(transport http.RoundTripper, modifiers []RequestModifier) *Tra
 // RoundTrip ...
 func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
 	for _, modifier := range t.modifiers {
-		if err := modifier.ModifyRequest(req); err != nil {
+		if err := modifier.Modify(req); err != nil {
 			return nil, err
 		}
 	}

utils/registry/utils/utils.go

@@ -0,0 +1,44 @@
+/*
+   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package utils
+
+import (
+	"net/url"
+	"strings"
+)
+
+// FormatEndpoint formats endpoint
+func FormatEndpoint(endpoint string) string {
+	endpoint = strings.TrimSpace(endpoint)
+	endpoint = strings.TrimRight(endpoint, "/")
+	if !strings.HasPrefix(endpoint, "http://") &&
+		!strings.HasPrefix(endpoint, "https://") {
+		endpoint = "http://" + endpoint
+	}
+
+	return endpoint
+}
+
+// ParseEndpoint parses endpoint to a URL
+func ParseEndpoint(endpoint string) (*url.URL, error) {
+	endpoint = FormatEndpoint(endpoint)
+
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, err
+	}
+	return u, nil
+}