Merge branch 'master' of https://github.com/goharbor/harbor into fix_api_faild_always_show_loading

2024-12-18 06:38:19 +01:00 · 2019-05-08 14:27:45 +08:00 · 2019-05-08 14:27:45 +08:00 · fe85d23a90
commit fe85d23a90
parent 97acf44fbc 709406eb75
43 changed files with 444 additions and 233 deletions
--- a/make/harbor.yml
+++ b/make/harbor.yml
@ -75,7 +75,7 @@ log:
  location: /var/log/harbor

 #This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
-_version: 1.7.0
+_version: 1.8.0

 # Uncomment external_database if using external database. And the password will replace the the password setting in database.
 # And currently only support postgres.
@ -95,3 +95,7 @@ _version: 1.7.0
 #   registry_db_index: 1
 #   jobservice_db_index: 2
 #   chartmuseum_db_index: 3
+
+# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
+# uaa:
+#   ca_file: /path/to/ca
--- a/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja
+++ b/make/photon/prepare/templates/docker_compose/docker-compose.yml.jinja
@ -1,4 +1,4 @@
-version: '2'
+version: '2.3'
 services:
  log:
    image: goharbor/harbor-log:{{version}}
@ -130,12 +130,16 @@ services:
      - SETUID
    volumes:
      - ./common/config/core/app.conf:/etc/core/app.conf:z
-      - ./common/config/core/certificates/:/etc/core/certificates/:z
      - {{data_volume}}/secret/core/private_key.pem:/etc/core/private_key.pem:z
      - {{data_volume}}/secret/keys/secretkey:/etc/core/key:z
      - {{data_volume}}/ca_download/:/etc/core/ca/:z
      - {{data_volume}}/psc/:/etc/core/token/:z
      - {{data_volume}}/:/data/:z
+{% if uaa_ca_file %}
+      - type: bind
+        source: {{uaa_ca_file}}
+        target: /etc/core/certificates/uaa_ca.pem
+{% endif %}
    networks:
      harbor:
 {% if with_notary %}
--- a/make/photon/prepare/templates/jobservice/config.yml.jinja
+++ b/make/photon/prepare/templates/jobservice/config.yml.jinja
@ -37,5 +37,3 @@ job_loggers:
 loggers:
  - name: "STD_OUTPUT" # Same with above
    level: "INFO"
-#Admin server endpoint
-admin_server: "http://adminserver:8080/"
--- a/make/photon/prepare/utils/configs.py
+++ b/make/photon/prepare/utils/configs.py
@ -190,4 +190,7 @@ def parse_yaml_config(config_file_path):
     # Admiral configs
    config_dict['admiral_url'] = configs.get("admiral_url") or ""

+    # UAA configs
+    config_dict['uaa'] = configs.get('uaa') or {}
+
    return config_dict
--- a/make/photon/prepare/utils/docker_compose.py
+++ b/make/photon/prepare/utils/docker_compose.py
@ -43,4 +43,8 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
        rendering_variables['cert_key_path'] = configs['cert_key_path']
        rendering_variables['cert_path'] = configs['cert_path']

+    uaa_config = configs.get('uaa') or {}
+    if uaa_config.get('ca_file'):
+        rendering_variables['uaa_ca_file'] = uaa_config['ca_file']
+
    render_jinja(docker_compose_template_path, docker_compose_yml_path, **rendering_variables)
--- a/src/common/http/client.go
+++ b/src/common/http/client.go
@ -17,9 +17,13 @@ package http
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/url"
+	"reflect"
+	"strings"

 	"github.com/goharbor/harbor/src/common/http/modifier"
 )
@ -168,3 +172,63 @@ func (c *Client) do(req *http.Request) ([]byte, error) {

 	return data, nil
 }
+
+// GetAndIteratePagination iterates the pagination header and returns all resources
+// The parameter "v" must be a pointer to a slice
+func (c *Client) GetAndIteratePagination(endpoint string, v interface{}) error {
+	url, err := url.Parse(endpoint)
+	if err != nil {
+		return err
+	}
+
+	rv := reflect.ValueOf(v)
+	if rv.Kind() != reflect.Ptr {
+		return errors.New("v should be a pointer to a slice")
+	}
+	elemType := rv.Elem().Type()
+	if elemType.Kind() != reflect.Slice {
+		return errors.New("v should be a pointer to a slice")
+	}
+
+	resources := reflect.Indirect(reflect.New(elemType))
+	for len(endpoint) > 0 {
+		req, err := http.NewRequest(http.MethodGet, endpoint, nil)
+		if err != nil {
+			return err
+		}
+		resp, err := c.Do(req)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode < 200 || resp.StatusCode > 299 {
+			return &Error{
+				Code:    resp.StatusCode,
+				Message: string(data),
+			}
+		}
+
+		res := reflect.New(elemType)
+		if err = json.Unmarshal(data, res.Interface()); err != nil {
+			return err
+		}
+		resources = reflect.AppendSlice(resources, reflect.Indirect(res))
+
+		endpoint = ""
+		link := resp.Header.Get("Link")
+		for _, str := range strings.Split(link, ",") {
+			if strings.HasSuffix(str, `rel="next"`) &&
+				strings.Index(str, "<") >= 0 &&
+				strings.Index(str, ">") >= 0 {
+				endpoint = url.Scheme + "://" + url.Host + str[strings.Index(str, "<")+1:strings.Index(str, ">")]
+				break
+			}
+		}
+	}
+	rv.Elem().Set(resources)
+	return nil
+}
--- a/src/common/utils/oidc/helper.go
+++ b/src/common/utils/oidc/helper.go
@ -208,5 +208,9 @@ func RefreshToken(ctx context.Context, token *Token) (*Token, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &Token{Token: *t, IDToken: t.Extra("id_token").(string)}, nil
+	it, ok := t.Extra("id_token").(string)
+	if !ok {
+		return nil, fmt.Errorf("failed to get id_token from refresh response")
+	}
+	return &Token{Token: *t, IDToken: it}, nil
 }
--- a/src/core/api/user.go
+++ b/src/core/api/user.go
@ -203,7 +203,10 @@ func (ua *UserAPI) List() {
 		ua.SendInternalServerError(fmt.Errorf("failed to get users: %v", err))
 		return
 	}
-
+	for i := range users {
+		user := &users[i]
+		user.Password = ""
+	}
 	ua.SetPaginationHeader(total, page, size)
 	ua.Data["json"] = users
 	ua.ServeJSON()
--- a/src/core/proxy/interceptor_test.go
+++ b/src/core/proxy/interceptor_test.go
@ -79,6 +79,53 @@ func TestMatchPullManifest(t *testing.T) {
 	assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7)
 }

+func TestMatchPushManifest(t *testing.T) {
+	assert := assert.New(t)
+	req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/library/ubuntu/manifests/14.04", nil)
+	res1, _, _ := MatchPushManifest(req1)
+	assert.False(res1, "%s %v is not a request to push manifest", req1.Method, req1.URL)
+
+	req2, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
+	res2, repo2, tag2 := MatchPushManifest(req2)
+	assert.True(res2, "%s %v is a request to push manifest", req2.Method, req2.URL)
+	assert.Equal("library/ubuntu", repo2)
+	assert.Equal("14.04", tag2)
+
+	req3, _ := http.NewRequest("GET", "https://192.168.0.5:443/v1/library/ubuntu/manifests/14.04", nil)
+	res3, _, _ := MatchPushManifest(req3)
+	assert.False(res3, "%s %v is not a request to push manifest", req3.Method, req3.URL)
+
+	req4, _ := http.NewRequest("PUT", "https://192.168.0.5/v2/library/ubuntu/manifests/14.04", nil)
+	res4, repo4, tag4 := MatchPushManifest(req4)
+	assert.True(res4, "%s %v is a request to push manifest", req4.Method, req4.URL)
+	assert.Equal("library/ubuntu", repo4)
+	assert.Equal("14.04", tag4)
+
+	req5, _ := http.NewRequest("PUT", "https://myregistry.com/v2/path1/path2/golang/manifests/1.6.2", nil)
+	res5, repo5, tag5 := MatchPushManifest(req5)
+	assert.True(res5, "%s %v is a request to push manifest", req5.Method, req5.URL)
+	assert.Equal("path1/path2/golang", repo5)
+	assert.Equal("1.6.2", tag5)
+
+	req6, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/registry/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
+	res6, repo6, tag6 := MatchPushManifest(req6)
+	assert.True(res6, "%s %v is a request to push manifest", req6.Method, req6.URL)
+	assert.Equal("myproject/registry", repo6)
+	assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag6)
+
+	req7, _ := http.NewRequest("PUT", "https://myregistry.com/v2/myproject/manifests/sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", nil)
+	res7, repo7, tag7 := MatchPushManifest(req7)
+	assert.True(res7, "%s %v is a request to push manifest", req7.Method, req7.URL)
+	assert.Equal("myproject", repo7)
+	assert.Equal("sha256:ca4626b691f57d16ce1576231e4a2e2135554d32e13a85dcff380d51fdd13f6a", tag7)
+
+	req8, _ := http.NewRequest("PUT", "http://192.168.0.3:80/v2/library/ubuntu/manifests/14.04", nil)
+	res8, repo8, tag8 := MatchPushManifest(req8)
+	assert.True(res8, "%s %v is a request to push manifest", req8.Method, req8.URL)
+	assert.Equal("library/ubuntu", repo8)
+	assert.Equal("14.04", tag8)
+}
+
 func TestMatchListRepos(t *testing.T) {
 	assert := assert.New(t)
 	req1, _ := http.NewRequest("POST", "http://127.0.0.1:5000/v2/_catalog", nil)
--- a/src/core/proxy/interceptors.go
+++ b/src/core/proxy/interceptors.go
@ -43,6 +43,18 @@ func MatchPullManifest(req *http.Request) (bool, string, string) {
 	if req.Method != http.MethodGet {
 		return false, "", ""
 	}
+	return matchManifestURL(req)
+}
+
+// MatchPushManifest checks if the request looks like a request to push manifest.  If it is returns the image and tag/sha256 digest as 2nd and 3rd return values
+func MatchPushManifest(req *http.Request) (bool, string, string) {
+	if req.Method != http.MethodPut {
+		return false, "", ""
+	}
+	return matchManifestURL(req)
+}
+
+func matchManifestURL(req *http.Request) (bool, string, string) {
 	re := regexp.MustCompile(manifestURLPattern)
 	s := re.FindStringSubmatch(req.URL.Path)
 	if len(s) == 3 {
@ -168,6 +180,25 @@ func (rh readonlyHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	rh.next.ServeHTTP(rw, req)
 }

+type multipleManifestHandler struct {
+	next http.Handler
+}
+
+// The handler is responsible for blocking request to upload manifest list by docker client, which is not supported so far by Harbor.
+func (mh multipleManifestHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
+	match, _, _ := MatchPushManifest(req)
+	if match {
+		contentType := req.Header.Get("Content-type")
+		// application/vnd.docker.distribution.manifest.list.v2+json
+		if strings.Contains(contentType, "manifest.list.v2") {
+			log.Debugf("Content-type: %s is not supported, failing the response.", contentType)
+			http.Error(rw, marshalError("UNSUPPORTED_MEDIA_TYPE", "Manifest.list is not supported."), http.StatusUnsupportedMediaType)
+			return
+		}
+	}
+	mh.next.ServeHTTP(rw, req)
+}
+
 type listReposHandler struct {
 	next http.Handler
 }
--- a/src/core/proxy/proxy.go
+++ b/src/core/proxy/proxy.go
@ -38,7 +38,7 @@ func Init(urls ...string) error {
 		return err
 	}
 	Proxy = httputil.NewSingleHostReverseProxy(targetURL)
-	handlers = handlerChain{head: readonlyHandler{next: urlHandler{next: listReposHandler{next: contentTrustHandler{next: vulnerableHandler{next: Proxy}}}}}}
+	handlers = handlerChain{head: readonlyHandler{next: urlHandler{next: multipleManifestHandler{next: listReposHandler{next: contentTrustHandler{next: vulnerableHandler{next: Proxy}}}}}}}
 	return nil
 }

--- a/src/core/service/notifications/registry/handler.go
+++ b/src/core/service/notifications/registry/handler.go
@ -93,20 +93,24 @@ func (n *NotificationHandler) Post() {
 		}()

 		if action == "push" {
-			go func() {
-				exist := dao.RepositoryExists(repository)
-				if exist {
-					return
-				}
-				log.Debugf("Add repository %s into DB.", repository)
-				repoRecord := models.RepoRecord{
-					Name:      repository,
-					ProjectID: pro.ProjectID,
-				}
-				if err := dao.AddRepository(repoRecord); err != nil {
-					log.Errorf("Error happens when adding repository: %v", err)
-				}
-			}()
+			// discard the notification without tag.
+			if tag != "" {
+				go func() {
+					exist := dao.RepositoryExists(repository)
+					if exist {
+						return
+					}
+					log.Debugf("Add repository %s into DB.", repository)
+					repoRecord := models.RepoRecord{
+						Name:      repository,
+						ProjectID: pro.ProjectID,
+					}
+					if err := dao.AddRepository(repoRecord); err != nil {
+						log.Errorf("Error happens when adding repository: %v", err)
+					}
+				}()
+			}
+
 			if !coreutils.WaitForManifestReady(repository, tag, 5) {
 				log.Errorf("Manifest for image %s:%s is not ready, skip the follow up actions.", repository, tag)
 				return
--- a/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts
+++ b/src/portal/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts
@ -185,7 +185,7 @@ export class CreateEditEndpointComponent
      this.endpointService.getEndpoint(targetId).subscribe(
        target => {
          this.target = target;
-          this.urlDisabled = this.target.type === 'dockerHub' ? true : false;
+          this.urlDisabled = this.target.type === 'docker-hub' ? true : false;
          // Keep data cache
          this.initVal = clone(target);
          this.initVal.credential.access_secret = FAKE_PASSWORD;
@ -212,7 +212,7 @@ export class CreateEditEndpointComponent

  adapterChange($event): void {
    let selectValue = this.targetForm.controls.adapter.value;
-    if (selectValue === 'dockerHub') {
+    if (selectValue === 'docker-hub') {
      this.urlDisabled = true;
      this.targetForm.controls.endpointUrl.setValue(DOCKERHUB_URL);
    } else {
--- a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html
+++ b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.html
@ -22,7 +22,7 @@
        <div class="form-group form-group-override">
          <label class="form-group-label-override">{{'REPLICATION.REPLI_MODE' | translate}}</label>
          <div class="radio-inline" [class.disabled]="policyId >= 0 || onGoing">
-            <input type="radio" id="push_base" name="replicationMode" [value]=true  [disabled]="policyId >= 0 || onGoing" [(ngModel)]="isPushMode" (change)="modeChange()" [ngModelOptions]="{standalone: true}">
+            <input type="radio" id="push_base" name="replicationMode" [value]=true  [disabled]="policyId >= 0 || onGoing" [(ngModel)]="isPushMode" (change)="pushModeChange()" [ngModelOptions]="{standalone: true}">
            <label for="push_base">Push-based</label>
            <clr-tooltip class="mode-tooltip">
              <clr-icon clrTooltipTrigger shape="info-circle" size="24"></clr-icon>
@ -32,7 +32,7 @@
            </clr-tooltip>
          </div>
          <div class="radio-inline" [class.disabled]="policyId >= 0 || onGoing">          
-            <input type="radio" id="pull_base" name="replicationMode" [value]=false [disabled]="policyId >= 0 || onGoing" [(ngModel)]="isPushMode" [ngModelOptions]="{standalone: true}">
+            <input type="radio" id="pull_base" name="replicationMode" [value]=false [disabled]="policyId >= 0 || onGoing" [(ngModel)]="isPushMode" (change)="pullModeChange()" [ngModelOptions]="{standalone: true}">
            <label for="pull_base">Pull-based</label>
            <clr-tooltip class="mode-tooltip">
              <clr-icon clrTooltipTrigger shape="info-circle" size="24"></clr-icon>
--- a/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts
+++ b/src/portal/lib/src/create-edit-rule/create-edit-rule.component.ts
@ -133,11 +133,19 @@ export class CreateEditRuleComponent implements OnInit, OnDestroy {
  equals(c1: any, c2: any): boolean {
    return c1 && c2 ? c1.id === c2.id : c1 === c2;
  }
-  modeChange(): void {
+  pushModeChange(): void {
    this.setFilter([]);
    this.initRegistryInfo(0);
  }

+  pullModeChange(): void {
+    let selectId = this.ruleForm.get('src_registry').value;
+    if (selectId) {
+      this.setFilter([]);
+      this.initRegistryInfo(selectId.id);
+    }
+  }
+
  sourceChange($event): void {
    this.noSelectedEndpoint = false;
    let selectId = this.ruleForm.get('src_registry').value;
--- a/src/portal/lib/src/cron-schedule/cron-schedule.component.ts
+++ b/src/portal/lib/src/cron-schedule/cron-schedule.component.ts
@ -73,12 +73,15 @@ export class CronScheduleComponent implements OnChanges {
  }

  save(): void {
-    if (this.scheduleType === SCHEDULE_TYPE.CUSTOM && this.cronString === '') {
-      this.dateInvalid = true;
-    }
-    if (this.dateInvalid) {
-      return;
+    if (this.scheduleType === SCHEDULE_TYPE.CUSTOM ) {
+      if (this.cronString === '') {
+        this.dateInvalid = true;
+      }
+      if (this.dateInvalid) {
+        return;
+      }
    }
+
    let scheduleTerm: string = "";
    if (this.scheduleType && this.scheduleType === SCHEDULE_TYPE.NONE) {
      scheduleTerm = "";
--- a/src/portal/lib/src/inline-alert/inline-alert.component.ts
+++ b/src/portal/lib/src/inline-alert/inline-alert.component.ts
@ -24,7 +24,7 @@ import { Observable,  Subscription } from "rxjs";
  styleUrls: ["./inline-alert.component.scss"]
 })
 export class InlineAlertComponent {
-  inlineAlertType: string = "alert-danger";
+  inlineAlertType: string = "danger";
  inlineAlertClosable: boolean = false;
  alertClose: boolean = true;
  displayedText: string = "";
@ -51,7 +51,7 @@ export class InlineAlertComponent {
        .subscribe((res: string) => (this.displayedText = res));
    }

-    this.inlineAlertType = "alert-danger";
+    this.inlineAlertType = "danger";
    this.showCancelAction = false;
    this.inlineAlertClosable = true;
    this.alertClose = false;
@ -66,7 +66,7 @@ export class InlineAlertComponent {
        .get(warning.message)
        .subscribe((res: string) => (this.displayedText = res));
    }
-    this.inlineAlertType = "alert-warning";
+    this.inlineAlertType = "warning";
    this.showCancelAction = true;
    this.inlineAlertClosable = false;
    this.alertClose = false;
@ -81,7 +81,7 @@ export class InlineAlertComponent {
        .get(info.message)
        .subscribe((res: string) => (this.displayedText = res));
    }
-    this.inlineAlertType = "alert-success";
+    this.inlineAlertType = "success";
    this.showCancelAction = false;
    this.inlineAlertClosable = true;
    this.alertClose = false;
--- a/src/portal/src/app/account/account-settings/account-settings-modal.component.html
+++ b/src/portal/src/app/account/account-settings/account-settings-modal.component.html
@ -34,9 +34,9 @@
                    </label><span class="spinner spinner-inline" [hidden]="!checkProgress"></span>
                </div>
                <div class="form-group form-group-override">
-                    <label for="account_settings_full_name" class="required form-group-label-override">{{'PROFILE.FULL_NAME' | translate}}</label>
+                    <label for="account_settings_full_name" class="form-group-label-override" [class.required]="!account.oidc_user_meta">{{'PROFILE.FULL_NAME' | translate}}</label>
                    <label for="account_settings_full_name" aria-haspopup="true" role="tooltip" class="tooltip tooltip-validation tooltip-md tooltip-top-left" [class.invalid]='!getValidationState("account_settings_full_name")'>
-                      <input type="text" name="account_settings_full_name" #fullNameInput="ngModel" [(ngModel)]="account.realname" required maxLengthExt="20" id="account_settings_full_name" size="30" (input)='handleValidation("account_settings_full_name", false)' (blur)='handleValidation("account_settings_full_name", true)'>
+                      <input type="text" name="account_settings_full_name" #fullNameInput="ngModel" [(ngModel)]="account.realname" [required]="!account.oidc_user_meta"  maxLengthExt="20" id="account_settings_full_name" size="30" (input)='handleValidation("account_settings_full_name", false)' (blur)='handleValidation("account_settings_full_name", true)'>
                      <span class="tooltip-content">
                          {{'TOOLTIP.FULL_NAME' | translate}}
                      </span>
@ -45,7 +45,7 @@
                <div class="form-group form-group-override">
                    <label for="account_settings_comments" class="form-group-label-override">{{'PROFILE.COMMENT' | translate}}</label>
                    <label for="account_settings_comments" aria-haspopup="true" role="tooltip" class="tooltip tooltip-validation tooltip-md tooltip-top-left" [class.invalid]="commentInput.invalid && (commentInput.dirty || commentInput.touched)">
-                    <input type="text" #commentInput="ngModel" maxLengthExt="20" name="account_settings_comments" [(ngModel)]="account.comment" id="account_settings_comments" size="30">
+                    <input type="text" #commentInput="ngModel" maxLengthExt="30" name="account_settings_comments" [(ngModel)]="account.comment" id="account_settings_comments" size="30">
                    <span class="tooltip-content">
                        {{'TOOLTIP.COMMENT' | translate}}
                    </span>
--- a/src/portal/src/app/config/auth/config-auth.component.html
+++ b/src/portal/src/app/config/auth/config-auth.component.html
@ -322,7 +322,6 @@
    </section>

    <section class="form-block" *ngIf="showOIDC">
-        <div class="oidc-tip">{{ 'CONFIG.OIDC.OIDC_REDIREC_URL' | translate}} <span>{{redirectUrl}}/c/oidc/callback</span></div>
        <div class="form-group">
            <label for="oidcName" class="required">{{'CONFIG.OIDC.OIDC_PROVIDER' | translate}}</label>
            <label for="oidcName" aria-haspopup="true" role="tooltip"
@ -348,7 +347,7 @@
                class="tooltip tooltip-validation tooltip-lg tooltip-top-right">
                <input name="oidcEndpoint" type="text" #oidcEndpointInput="ngModel" required
                    [(ngModel)]="currentConfig.oidc_endpoint.value" id="oidcEndpoint" size="40"
-                    [disabled]="disabled(currentConfig.oidc_endpoint)" pattern="^([hH][tT]{2}[pP][sS])(.*?)*$">
+                    [disabled]="disabled(currentConfig.oidc_endpoint)" pattern="^([hH][tT]{2}[pP][sS]:\/\/)(.*?)*$">
                <span class="tooltip-content">
                    {{'TOOLTIP.OIDC_ENDPOIT_FORMAT' | translate}}
                </span>
@ -394,7 +393,7 @@
                    [(ngModel)]="currentConfig.oidc_scope.value" id="oidcScope" size="40" required
                    [disabled]="disabled(currentConfig.oidc_scope)" pattern="^(\w+,){0,}openid(,\w+){0,}$">
                <span class="tooltip-content">
-                    {{'TOOLTIP.ITEM_REQUIRED' | translate}}
+                    {{'TOOLTIP.SCOPE_REQUIRED' | translate}}
                </span>
            </label>
            <a href="javascript:void(0)" role="tooltip" aria-haspopup="true"
@ -416,6 +415,7 @@
                <span class="tooltip-content">{{'TOOLTIP.OIDC_VERIFYCERT' | translate}}</span>
            </a>
        </div>
+        <div class="oidc-tip">{{ 'CONFIG.OIDC.OIDC_REDIREC_URL' | translate}} <span>{{redirectUrl}}/c/oidc/callback</span></div>
    </section>
 </form>
 <div>
--- a/src/portal/src/app/config/auth/config-auth.component.scss
+++ b/src/portal/src/app/config/auth/config-auth.component.scss
@ -16,7 +16,5 @@ clr-tooltip {
    top: -5px;
 }
 .oidc-tip {
-    position: absolute;
-    top: 170px;
-    color: rgb(210, 74, 112);
+    color: rgb(10, 74, 112);
 }
--- a/src/portal/src/app/global-message/message.ts
+++ b/src/portal/src/app/global-message/message.ts
@ -22,15 +22,15 @@ export class Message {
  get type(): string {
    switch (this.alertType) {
      case AlertType.DANGER:
-        return 'alert-danger';
+        return 'danger';
      case AlertType.INFO:
-        return 'alert-info';
+        return 'info';
      case AlertType.SUCCESS:
-        return 'alert-success';
+        return 'success';
      case AlertType.WARNING:
-        return 'alert-warning';
+        return 'warning';
      default:
-        return 'alert-warning';
+        return 'warning';
    }
  }

--- a/src/portal/src/app/project/create-project/create-project.component.ts
+++ b/src/portal/src/app/project/create-project/create-project.component.ts
@ -32,6 +32,7 @@ import { InlineAlertComponent } from "../../shared/inline-alert/inline-alert.com

 import { Project } from "../project";
 import { ProjectService } from "../project.service";
+import { errorHandler } from '@angular/platform-browser/src/browser';



@ -119,21 +120,7 @@ export class CreateProjectComponent implements OnInit, OnDestroy {
      },
      error => {
        this.isSubmitOnGoing = false;
-
-        let errorMessage: string;
-        if (error instanceof Response) {
-          switch (error.status) {
-            case 409:
-              this.translateService.get("PROJECT.NAME_ALREADY_EXISTS").subscribe(res => errorMessage = res);
-              break;
-            case 400:
-              this.translateService.get("PROJECT.NAME_IS_ILLEGAL").subscribe(res => errorMessage = res);
-              break;
-            default:
-              this.translateService.get("PROJECT.UNKNOWN_ERROR").subscribe(res => errorMessage = res);
-          }
-        this.messageHandlerService.handleError(error);
-        }
+        this.inlineAlert.showInlineError(error);
      });
  }

--- a/src/portal/src/app/project/project-routing-resolver.service.ts
+++ b/src/portal/src/app/project/project-routing-resolver.service.ts
@ -59,10 +59,10 @@ export class ProjectRoutingResolver implements Resolve<Project> {
          this.router.navigate(['/harbor', 'projects']);
          return null;
        }
-      }), catchError (error => {
+      }, catchError (error => {
        this.router.navigate(['/harbor', 'projects']);
        return null;
-      }));
+      })));

  }
 }
--- a/src/portal/src/app/shared/inline-alert/inline-alert.component.ts
+++ b/src/portal/src/app/shared/inline-alert/inline-alert.component.ts
@ -23,7 +23,7 @@ import { Subscription } from "rxjs";
    styleUrls: ['inline-alert.component.scss']
 })
 export class InlineAlertComponent {
-    inlineAlertType: string = 'alert-danger';
+    inlineAlertType: string = 'danger';
    inlineAlertClosable: boolean = false;
    alertClose: boolean = true;
    displayedText: string = "";
@ -49,7 +49,7 @@ export class InlineAlertComponent {
            this.translate.get(this.displayedText).subscribe((res: string) => this.displayedText = res);
        }

-        this.inlineAlertType = 'alert-danger';
+        this.inlineAlertType = 'danger';
        this.showCancelAction = false;
        this.inlineAlertClosable = true;
        this.alertClose = false;
@ -62,7 +62,7 @@ export class InlineAlertComponent {
        if (warning && warning.message) {
            this.translate.get(warning.message).subscribe((res: string) => this.displayedText = res);
        }
-        this.inlineAlertType = 'alert-warning';
+        this.inlineAlertType = 'warning';
        this.showCancelAction = true;
        this.inlineAlertClosable = false;
        this.alertClose = false;
@ -75,7 +75,7 @@ export class InlineAlertComponent {
        if (warning && warning.message) {
            this.translate.get(warning.message).subscribe((res: string) => this.displayedText = res);
        }
-        this.inlineAlertType = 'alert-warning';
+        this.inlineAlertType = 'warning';
        this.showCancelAction = false;
        this.inlineAlertClosable = true;
        this.alertClose = false;
@ -88,7 +88,7 @@ export class InlineAlertComponent {
        if (info && info.message) {
            this.translate.get(info.message).subscribe((res: string) => this.displayedText = res);
        }
-        this.inlineAlertType = 'alert-success';
+        this.inlineAlertType = 'success';
        this.showCancelAction = false;
        this.inlineAlertClosable = true;
        this.alertClose = false;
--- a/src/portal/src/app/sign-in/sign-in.component.ts
+++ b/src/portal/src/app/sign-in/sign-in.component.ts
@ -145,7 +145,8 @@ export class SignInComponent implements AfterViewChecked, OnInit {
        return this.appConfig.auth_mode === 'oidc_auth';
    }
    public get showForgetPwd(): boolean {
-        return this.appConfig.auth_mode !== 'ldap_auth' && this.appConfig.auth_mode !== 'uaa_auth';
+        return this.appConfig.auth_mode !== 'ldap_auth' && this.appConfig.auth_mode !== 'uaa_auth'
+         && this.appConfig.auth_mode !== 'oidc_auth';
    }
    clickRememberMe($event: any): void {
        if ($event && $event.target) {
--- a/src/portal/src/app/user/user.component.ts
+++ b/src/portal/src/app/user/user.component.ts
@ -106,7 +106,7 @@ export class UserComponent implements OnInit, OnDestroy {
  public get canCreateUser(): boolean {
    let appConfig = this.appConfigService.getConfig();
    if (appConfig) {
-      return !(appConfig.auth_mode === 'ldap_auth' || appConfig.auth_mode === 'uaa_auth');
+      return !(appConfig.auth_mode === 'ldap_auth' || appConfig.auth_mode === 'uaa_auth' || appConfig.auth_mode === 'oidc_auth');
    } else {
      return true;
    }
--- a/src/portal/src/i18n/lang/en-us-lang.json
+++ b/src/portal/src/i18n/lang/en-us-lang.json
@ -62,7 +62,7 @@
        "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.",
        "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.",
        "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.",
-        "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.",
+        "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.",
        "EMAIL": "Email should be a valid email address like name@example.com.",
        "USER_NAME": "Cannot contain special characters and maximum length should be 255 characters.",
        "FULL_NAME": "Maximum length should be 20 characters.",
@ -75,6 +75,7 @@
        "SIGN_UP_MAIL": "Email is only used for resetting your password.",
        "SIGN_UP_REAL_NAME": "First and last name",
        "ITEM_REQUIRED": "Field is required.",
+        "SCOPE_REQUIRED": "Field is required and should be in scope format.",
        "NUMBER_REQUIRED": "Field is required and should be numbers.",
        "PORT_REQUIRED": "Field is required and should be valid port number.",
        "CRON_REQUIRED": "Field is required and should be in cron format.",
@ -374,7 +375,7 @@
        "TOTAL": "Total",
        "OVERRIDE": "Override",
        "ENABLED_RULE": "Enable rule",
-        "OVERRIDE_INFO": "Replace the destination resources if name exists",
+        "OVERRIDE_INFO": "Override",
        "OPERATION": "Operation",
        "CURRENT": "current",
        "FILTER_PLACEHOLDER": "Filter Tasks",
--- a/src/portal/src/i18n/lang/es-es-lang.json
+++ b/src/portal/src/i18n/lang/es-es-lang.json
@ -62,7 +62,7 @@
        "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.",
        "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.",
        "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.",
-        "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.",
+        "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.",
        "EMAIL": "El email debe ser una dirección válida como nombre@ejemplo.com.",
        "USER_NAME": "Debe tener una longitud máxima de 255 caracteres y no puede contener caracteres especiales.",
        "FULL_NAME": "La longitud máxima debería ser de 20 caracteres.",
@ -75,6 +75,7 @@
        "SIGN_UP_MAIL": "La dirección de email solamente se utilizar para restablecer la contraseña.",
        "SIGN_UP_REAL_NAME": "Nombre y apellidos",
        "ITEM_REQUIRED": "Campo obligatorio.",
+        "SCOPE_REQUIRED": "Field is required and should be in scope format.",
        "NUMBER_REQUIRED": "El campo es obligatorio y debería ser un número.",
        "PORT_REQUIRED": "El campo es obligatorio y debería ser un número de puerto válido.",
        "CRON_REQUIRED": "El campo es obligatorio y debe estar en formato cron.",
@ -374,7 +375,7 @@
        "TOTAL": "Total",
        "OVERRIDE": "Override",
        "ENABLED_RULE": "Enable rule",
-        "OVERRIDE_INFO": "Replace the destination resources if name exists",
+        "OVERRIDE_INFO": "Override",
        "CURRENT": "current",
        "FILTER_PLACEHOLDER": "Filter Tasks",
        "STOP_TITLE": "Confirme Stop Executions",
--- a/src/portal/src/i18n/lang/fr-fr-lang.json
+++ b/src/portal/src/i18n/lang/fr-fr-lang.json
@ -59,7 +59,7 @@
        "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.",
        "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.",
        "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.",
-        "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.",
+        "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.",
        "EMAIL": "L'email doit être une adresse email valide comme name@example.com.",
        "USER_NAME": "Ne peut pas contenir de caractères spéciaux et la longueur maximale doit être de 255 caractères.",
        "FULL_NAME": "La longueur maximale doit être de 20 caractères.",
@ -72,6 +72,7 @@
        "SIGN_UP_MAIL": "L'email n'est utilisé que pour réinitialiser votre mot de passe.",
        "SIGN_UP_REAL_NAME": "Prénom et nom",
        "ITEM_REQUIRED": "Le champ est obligatoire.",
+        "SCOPE_REQUIRED": "Field is required and should be in scope format.",
        "NUMBER_REQUIRED": "Le champ est obligatoire et doit être numérique.",
        "PORT_REQUIRED": "Le champ est obligatoire et doit être un numéro de port valide.",
        "CRON_REQUIRED": "Le champ est obligatoire et doit être au format cron.",
@ -366,7 +367,7 @@
        "TOTAL": "Total",
        "OVERRIDE": "Override",
        "ENABLED_RULE": "Enable rule",
-        "OVERRIDE_INFO": "Replace the destination resources if name exists",
+        "OVERRIDE_INFO": "Override",
        "CURRENT": "current",
        "FILTER_PLACEHOLDER": "Filter Tasks",
        "STOP_TITLE": "Confirmer arrêter les exécutions",
--- a/src/portal/src/i18n/lang/pt-br-lang.json
+++ b/src/portal/src/i18n/lang/pt-br-lang.json
@ -62,7 +62,7 @@
        "PUSH_BASED": "Push the resources from the local Harbor to the remote registry.",
        "PULL_BASED": "Pull the resources from the remote registry to the local Harbor.",
        "DESTINATION_NAMESPACE": "Specify the destination namespace. If empty, the resources will be put under the same namespace as the source.",
-        "OVERRIDE": "Specify whether to replace the resources at the destination if a resource with the same name exists.",
+        "OVERRIDE": "Specify whether to override the resources at the destination if a resource with the same name exists.",
        "EMAIL": "Email deve ser um  endereço de email válido como nome@exemplo.com.",
        "USER_NAME": "Não pode conter caracteres especiais e o tamanho máximo deve ser de 255 caracteres.",
        "FULL_NAME": "Tamanho máximo deve ser de 20 caracteres.",
@ -75,6 +75,7 @@
        "SIGN_UP_MAIL": "Email é apenas utilizado para redefinir a senha.",
        "SIGN_UP_REAL_NAME": "Primeiro e último nome",
        "ITEM_REQUIRED": "Campo é obrigatório.",
+        "SCOPE_REQUIRED": "Field is required and should be in scope format.",
        "NUMBER_REQUIRED": "Campo é obrigatório e deve ser numerico.",
        "PORT_REQUIRED": "Campo é obrigatório e deve ser um número de porta válido.",
        "CRON_REQUIRED": "O campo é obrigatório e deve estar no formato cron.",
@ -372,7 +373,7 @@
        "TOTAL": "Total",
        "OVERRIDE": "Override",
        "ENABLED_RULE": "Enable rule",
-        "OVERRIDE_INFO": "Replace the destination resources if name exists",
+        "OVERRIDE_INFO": "Override",
        "CURRENT": "current",
        "FILTER_PLACEHOLDER": "Filter Tasks",
        "STOP_TITLE": "Confirme as execuções de parada",
--- a/src/portal/src/i18n/lang/zh-cn-lang.json
+++ b/src/portal/src/i18n/lang/zh-cn-lang.json
@ -62,7 +62,7 @@
        "PUSH_BASED": "把资源由本地Harbor推送到远端仓库。",
        "PULL_BASED": "把资源由远端仓库拉取到本地Harbor。",
        "DESTINATION_NAMESPACE": "指定目的端名称空间。如果不填，资源会被放到和源相同的名称空间下。",
-        "OVERRIDE": "如果存在具有相同名称的资源，请指定是否替换目标上的资源。",
+        "OVERRIDE": "如果存在具有相同名称的资源，请指定是否覆盖目标上的资源。",
        "EMAIL": "请使用正确的邮箱地址，比如name@example.com。",
        "USER_NAME": "不能包含特殊字符且长度不能超过255。",
        "FULL_NAME": "长度不能超过20。",
@ -75,6 +75,7 @@
        "SIGN_UP_MAIL": "邮件地址仅用来重置您的密码。",
        "SIGN_UP_REAL_NAME": "全名",
        "ITEM_REQUIRED": "此项为必填项。",
+        "SCOPE_REQUIRED": "此项为必填项且为scope格式。",
        "NUMBER_REQUIRED": "此项为必填项且为数字。",
        "PORT_REQUIRED": "此项为必填项且为合法端口号。",
        "CRON_REQUIRED": "此项为必填项且为cron格式。",
@ -373,7 +374,7 @@
        "TOTAL": "总数",
        "OVERRIDE": "覆盖",
        "ENABLED_RULE": "启用规则",
-        "OVERRIDE_INFO": "如果名称存在，则替换目标资源",
+        "OVERRIDE_INFO": "覆盖",
        "CURRENT": "当前仓库",
        "FILTER_PLACEHOLDER": "过滤任务",
        "STOP_TITLE": "确认停止任务",
--- a/src/replication/adapter/harbor/adapter.go
+++ b/src/replication/adapter/harbor/adapter.go
@ -211,8 +211,8 @@ type project struct {

 func (a *adapter) getProjects(name string) ([]*project, error) {
 	projects := []*project{}
-	url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=1000", a.coreServiceURL, name)
-	if err := a.client.Get(url, &projects); err != nil {
+	url := fmt.Sprintf("%s/api/projects?name=%s&page=1&page_size=500", a.coreServiceURL, name)
+	if err := a.client.GetAndIteratePagination(url, &projects); err != nil {
 		return nil, err
 	}
 	return projects, nil
@ -243,3 +243,12 @@ func (a *adapter) getProject(name string) (*project, error) {
 	}
 	return nil, nil
 }
+
+func (a *adapter) getRepositories(projectID int64) ([]*repository, error) {
+	repositories := []*repository{}
+	url := fmt.Sprintf("%s/api/repositories?project_id=%d&page=1&page_size=500", a.coreServiceURL, projectID)
+	if err := a.client.GetAndIteratePagination(url, &repositories); err != nil {
+		return nil, err
+	}
+	return repositories, nil
+}
--- a/src/replication/adapter/harbor/image_registry.go
+++ b/src/replication/adapter/harbor/image_registry.go
@ -64,9 +64,8 @@ func (a *adapter) FetchImages(filters []*model.Filter) ([]*model.Resource, error
 	}
 	resources := []*model.Resource{}
 	for _, project := range projects {
-		repositories := []*repository{}
-		url := fmt.Sprintf("%s/api/repositories?project_id=%d", a.coreServiceURL, project.ID)
-		if err = a.client.Get(url, &repositories); err != nil {
+		repositories, err := a.getRepositories(project.ID)
+		if err != nil {
 			return nil, err
 		}
 		repositories, err = filterRepositories(repositories, filters)
--- a/src/replication/adapter/native/adapter.go
+++ b/src/replication/adapter/native/adapter.go
@ -20,16 +20,14 @@ import (
 	"github.com/goharbor/harbor/src/replication/model"
 )

-const registryTypeNative model.RegistryType = "native"
-
 func init() {
-	if err := adp.RegisterFactory(registryTypeNative, func(registry *model.Registry) (adp.Adapter, error) {
+	if err := adp.RegisterFactory(model.RegistryTypeDockerRegistry, func(registry *model.Registry) (adp.Adapter, error) {
 		return newAdapter(registry)
 	}); err != nil {
-		log.Errorf("failed to register factory for %s: %v", registryTypeNative, err)
+		log.Errorf("failed to register factory for %s: %v", model.RegistryTypeDockerRegistry, err)
 		return
 	}
-	log.Infof("the factory for adapter %s registered", registryTypeNative)
+	log.Infof("the factory for adapter %s registered", model.RegistryTypeDockerRegistry)
 }

 func newAdapter(registry *model.Registry) (*native, error) {
@ -52,7 +50,7 @@ var _ adp.Adapter = native{}

 func (native) Info() (info *model.RegistryInfo, err error) {
 	return &model.RegistryInfo{
-		Type: registryTypeNative,
+		Type: model.RegistryTypeDockerRegistry,
 		SupportedResourceTypes: []model.ResourceType{
 			model.ResourceTypeImage,
 		},
--- a/src/replication/adapter/native/adapter_test.go
+++ b/src/replication/adapter/native/adapter_test.go
@ -57,7 +57,7 @@ func Test_native_Info(t *testing.T) {
 	var info, err = adapter.Info()
 	assert.Nil(t, err)
 	assert.NotNil(t, info)
-	assert.Equal(t, registryTypeNative, info.Type)
+	assert.Equal(t, model.RegistryTypeDockerRegistry, info.Type)
 	assert.Equal(t, 1, len(info.SupportedResourceTypes))
 	assert.Equal(t, 2, len(info.SupportedResourceFilters))
 	assert.Equal(t, 2, len(info.SupportedTriggers))
--- a/src/replication/adapter/native/image_registry_test.go
+++ b/src/replication/adapter/native/image_registry_test.go
@ -65,7 +65,7 @@ func Test_native_FetchImages(t *testing.T) {
 	fmt.Println("mockNativeRegistry URL: ", mock.URL)

 	var registry = &model.Registry{
-		Type:     registryTypeNative,
+		Type:     model.RegistryTypeDockerRegistry,
 		URL:      mock.URL,
 		Insecure: true,
 	}
--- a/src/replication/model/registry.go
+++ b/src/replication/model/registry.go
@ -22,10 +22,10 @@ import (

 // const definition
 const (
-	// RegistryTypeHarbor indicates registry type harbor
-	RegistryTypeHarbor    RegistryType = "harbor"
-	RegistryTypeDockerHub RegistryType = "dockerHub"
-	RegistryTypeHuawei    RegistryType = "Huawei"
+	RegistryTypeHarbor         RegistryType = "harbor"
+	RegistryTypeDockerHub      RegistryType = "docker-hub"
+	RegistryTypeDockerRegistry RegistryType = "docker-registry"
+	RegistryTypeHuawei         RegistryType = "huawei"

 	FilterStyleTypeText  = "input"
 	FilterStyleTypeRadio = "radio"
--- a/tools/migration/Dockerfile
+++ b/tools/migration/Dockerfile
@ -7,7 +7,7 @@ RUN tdnf distro-sync -y \
    && tdnf remove -y toybox \
    && tdnf install -y sed shadow procps-ng gawk gzip sudo net-tools glibc-i18n >> /dev/null\
    && groupadd -r -g 10000 mysql && useradd --no-log-init -r -g 10000 -u 10000 mysql \
-    && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc \
+    && tdnf install -y mariadb-server mariadb mariadb-devel python2 python2-devel python-pip gcc PyYAML python-jinja2\
    linux-api-headers glibc-devel binutils zlib-devel openssl-devel postgresql python-psycopg2 >> /dev/null \
    && pip install mysqlclient alembic \
    && mkdir /docker-entrypoint-initdb.d /docker-entrypoint-updatedb.d \
--- a/tools/migration/cfg/migrator_1_8_0/init.py
+++ b/tools/migration/cfg/migrator_1_8_0/init.py
@ -1,14 +1,13 @@
 from __future__ import print_function
 import utils
 import os
+from jinja2 import Environment, FileSystemLoader
 acceptable_versions = ['1.7.0']
 keys = [
    'hostname',
    'ui_url_protocol',
-    'customize_crt',
    'ssl_cert',
    'ssl_cert_key',
-    'secretkey_path',
    'admiral_url',
    'log_rotate_count',
    'log_rotate_size',
@ -19,18 +18,15 @@ keys = [
    'db_password',
    'db_port',
    'db_user',
-    'clair_db_host',
-    'clair_db_password',
-    'clair_db_port',
-    'clair_db_username',
-    'clair_db',
-    'uaa_endpoint',
-    'uaa_clientid',
-    'uaa_clientsecret',
-    'uaa_verify_cert',
-    'uaa_ca_cert',
+    'redis_host',
+    'redis_port',
+    'redis_password',
+    'redis_db_index',
+    'clair_updaters_interval',
+    'max_job_workers',
    'registry_storage_provider_name',
-    'registry_storage_provider_config'
+    'registry_storage_provider_config',
+    'registry_custom_ca_bundle'
    ]

 def migrate(input_cfg, output_cfg):
@ -38,5 +34,26 @@ def migrate(input_cfg, output_cfg):
    val = {}
    for k in keys:
        val[k] = d.get(k,'')
-    tpl_path = os.path.join(os.path.dirname(__file__), 'harbor.yml.tpl')
-    utils.render(tpl_path, output_cfg, **val)
+    if val['db_host'] == 'postgresql' and val['db_port'] == 5432 and val['db_user'] == 'postgres':
+        val['external_db'] = False
+    else:
+        val['external_db'] = True
+    # If using default filesystem, didn't need registry_storage_provider_config config
+    if val['registry_storage_provider_name'] == 'filesystem' and not val.get('registry_storage_provider_config'):
+        val['storage_provider_info'] = ''
+    else:
+        val['storage_provider_info'] = utils.get_storage_provider_info(
+            val['registry_storage_provider_name'],
+            val['registry_storage_provider_config']
+            )
+    if val['redis_host'] == 'redis' and val['redis_port'] == 6379 and not val['redis_password'] and val['redis_db_index'] == '1,2,3':
+        val['external_redis'] = False
+    else:
+        val['registry_db_index'], val['jobservice_db_index'], val['chartmuseum_db_index'] = map(int, val['redis_db_index'].split(','))
+        val['external_redis'] = True
+
+    this_dir = os.path.dirname(__file__)
+    tpl = Environment(loader=FileSystemLoader(this_dir)).get_template('harbor.yml.jinja')
+
+    with open(output_cfg, 'w') as f:
+        f.write(tpl.render(**val))
--- a/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja
+++ b/tools/migration/cfg/migrator_1_8_0/harbor.yml.jinja
@ -0,0 +1,113 @@
+## Configuration file of Harbor
+
+#The IP address or hostname to access admin UI and registry service.
+#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
+hostname: {{hostname}}
+
+# http related comfig
+http:
+  # port for http, default is 80. If https enabled, this port will redirect to https port
+  port: 80
+
+{% if ui_url_protocol == 'https'%}
+https:
+  port: 443
+  #The path of cert and key files for nginx
+  certificate: {{ ssl_cert }}
+  private_key: {{ ssl_cert_key }}
+{% else %}
+# https:
+#   port: 443
+#   #The path of cert and key files for nginx
+#   certificate: /your/certificate/path
+#   private_key: /your/private/key/path
+{% endif %}
+
+# Uncomment extearnal_url if you want to enable external proxy
+# And when it enabled the hostname will no longger used
+# external_url: https://reg.mydomain.com:8433
+
+# The initial password of Harbor admin
+# It only works in first time to install harbor
+# Remember Change the admin password from UI after launching Harbor.
+harbor_admin_password: Harbor12345
+
+## Harbor DB configuration
+database:
+  #The password for the root user of Harbor DB. Change this before any production use.
+  password: {{ db_password }}
+
+# The default data volume
+data_volume: /data
+
+# Harbor Storage settings by default is using data_volume dir on local filesystem
+# Uncomment storage_service setting If you want to using external storage
+storage_service:
+  # ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
+  # of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
+  ca_bundle: {{ registry_custom_ca_bundle }}
+
+  {{storage_provider_info}}
+
+# Clair configuration
+clair:
+  # The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
+  updaters_interval: {{ clair_updaters_interval }}
+
+  # Config http proxy for Clair, e.g. http://my.proxy.com:3128
+  # Clair doesn't need to connect to harbor internal components via http proxy.
+  http_proxy: {{ http_proxy }}
+  https_proxy: {{ https_proxy }}
+  no_proxy: {{ no_proxy }}
+
+jobservice:
+  # Maximum number of job workers in job service  
+  max_job_workers: {{ max_job_workers }}
+
+# Log configurations
+log:
+  # options are debug, info, warn, error
+  level: info
+  # Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+  rotate_count: {{ log_rotate_count }}
+  # Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
+  # If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
+  # are all valid.
+  rotate_size: {{ log_rotate_size }}
+  # The directory on your host that store log
+  location: /var/log/harbor
+
+#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
+_version: 1.8.0
+
+{% if external_db %}
+# Uncomment external_database if using external database. And the password will replace the the password setting in database.
+# And currently ontly support postgres.
+external_database:
+  host: {{ db_host }}
+  port: {{ db_port }}
+  username: {{ db_user }}
+  password: {{ db_password }}
+  ssl_mode: disable
+{% endif %}
+
+{% if external_redis %}
+external_redis:
+  host: {{ redis_host }}
+  port: {{ redis_port }}
+  password: {{ redis_password }}
+  # db_index 0 is for core, it's unchangeable
+  registry_db_index: {{ registry_db_index }}
+  jobservice_db_index: {{ jobservice_db_index }}
+  chartmuseum_db_index: {{ chartmuseum_db_index }}
+{% else %}
+# Umcomments external_redis if using external Redis server
+# external_redis:
+#   host: redis
+#   port: 6379
+#   password:
+#   # db_index 0 is for core, it's unchangeable
+#   registry_db_index: 1
+#   jobservice_db_index: 2
+#   chartmuseum_db_index: 3
+{% endif %}
--- a/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl
+++ b/tools/migration/cfg/migrator_1_8_0/harbor.yml.tpl
@ -1,114 +0,0 @@
-## Configuration file of Harbor
-
-#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
-_version: 1.7.0
-#The IP address or hostname to access admin UI and registry service.
-#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
-#DO NOT comment out this line, modify the value of "hostname" directly, or the installation will fail.
-hostname: $hostname
-
-#The protocol for accessing the UI and token/notification service, by default it is http.
-#It can be set to https if ssl is enabled on nginx.
-ui_url_protocol: $ui_url_protocol
-
-#Maximum number of job workers in job service  
-max_job_workers: 10
-
-#Determine whether or not to generate certificate for the registry's token.
-#If the value is on, the prepare script creates new root cert and private key 
-#for generating token to access the registry. If the value is off the default key/cert will be used.
-#This flag also controls the creation of the notary signer's cert.
-customize_crt: $customize_crt
-
-# The default data volume
-data_volume: /data
-
-#The path of cert and key files for nginx, they are applied only the protocol is set to https
-ssl_cert: $ssl_cert
-ssl_cert_key: $ssl_cert_key
-
-#The path of secretkey storage
-secretkey_path: $secretkey_path
-
-#Admiral's url, comment this attribute, or set its value to NA when Harbor is standalone
-admiral_url: $admiral_url
-
-log:
-  #Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
-  rotate_count: $log_rotate_count
-  #Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes. 
-  #If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G 
-  #are all valid.
-  rotate_size: $log_rotate_size
-
-  # The directory that store log files
-  location: /var/log/harbor
-
-#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
-#only take effect in the first boot, the subsequent changes of these properties 
-#should be performed on web ui
-
-##The initial password of Harbor admin, only works for the first time when Harbor starts.
-#It has no effect after the first launch of Harbor.
-#Change the admin password from UI after launching Harbor.
-harbor_admin_password: Harbor12345
-
-database:
-  #The address of the Harbor database. Only need to change when using external db.
-  host: $db_host
-  #The port of Harbor database host
-  port: $db_port
-  #The user name of Harbor database
-  username: $db_user
-  #The password for the root user of Harbor DB. Change this before any production use.
-  password: $db_password
-
-redis:
-  # Redis connection address
-  redis_host: redis
-  # Redis connection port
-  redis_port: 6379
-  # Redis connection password
-  redis_password:
-  # Redis connection db index
-  # db_index 1,2,3 is for registry, jobservice and chartmuseum. 
-  # db_index 0 is for UI, it's unchangeable
-  redis_db_index: 1,2,3
-
-clair:
-  #Clair DB host address. Only change it when using an exteral DB.
-  db_host: $clair_db_host
-  #The password of the Clair's postgres database. Only effective when Harbor is deployed with Clair.
-  #Please update it before deployment. Subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
-  db_password: $clair_db_password
-  #Clair DB connect port
-  db_port: $clair_db_port
-  #Clair DB username
-  db_username: $clair_db_username
-  #Clair default database
-  db: $clair_db
-  #The interval of clair updaters, the unit is hour, set to 0 to disable the updaters.
-  updaters_interval: 12
-  #Config http proxy for Clair, e.g. http://my.proxy.com:3128
-  #Clair doesn't need to connect to harbor internal components via http proxy.
-  http_proxy: $http_proxy
-  https_proxy: $https_proxy
-  no_proxy: $no_proxy
-
-storage:
-  #Please be aware that the following storage settings will be applied to both docker registry and helm chart repository.
-  #registry_storage_provider can be: filesystem, s3, gcs, azure, etc.
-  registry_storage_provider_name: $registry_storage_provider_name
-  #registry_storage_provider_config is a comma separated "key: value" pairs, e.g. "key1: value, key2: value2".
-  #To avoid duplicated configurations, both docker registry and chart repository follow the same storage configuration specifications of docker registry.
-  #Refer to https://docs.docker.com/registry/configuration/#storage for all available configuration.
-  registry_storage_provider_config: $registry_storage_provider_config
-  #registry_custom_ca_bundle is the path to the custom root ca certificate, which will be injected into the truststore
-  #of registry's and chart repository's containers.  This is usually needed when the user hosts a internal storage with self signed certificate.
-  registry_custom_ca_bundle:
-
-#If reload_config=true, all settings which present in harbor.yml take effect after prepare and restart harbor, it overwrites exsiting settings.
-#reload_config=true
-#Regular expression to match skipped environment variables
-#skip_reload_env_pattern: (^EMAIL.*)|(^LDAP.*)
-
--- a/tools/migration/cfg/run.py
+++ b/tools/migration/cfg/run.py
@ -13,7 +13,7 @@ import shutil
 import sys

 def main():
-    target_version = '1.7.0'
+    target_version = '1.8.0'
    parser = argparse.ArgumentParser(description='migrator of harbor.cfg') 
    parser.add_argument('--input', '-i', action="store", dest='input_path', required=True, help='The path to the old harbor.cfg that provides input value, this required value')
    parser.add_argument('--output','-o', action="store", dest='output_path', required=False, help='The path of the migrated harbor.cfg, if not set the input file will be overwritten')
@ -63,6 +63,5 @@ def search(basedir, input_ver, target_ver, l):
                return True
    return False

-
 if __name__ == "__main__":
    main()
--- a/tools/migration/cfg/utils.py
+++ b/tools/migration/cfg/utils.py
@ -43,3 +43,25 @@ def render(src, dest, **kw):
    t = Template(open(src, 'r').read())
    with open(dest, 'w') as f:
        f.write(t.substitute(**kw))
+
+def get_storage_provider_info(provider_name, provider_config):
+    provider_config = provider_config.strip('" ')
+    if not provider_config.strip(" "):
+        return ''
+
+    storage_provider_cfg_map = {}
+    for k_v in provider_config.split(","):
+        if k_v > 0:
+            kvs = k_v.split(": ") # add space suffix to avoid existing ":" in the value
+            if len(kvs) == 2:
+                #key must not be empty
+                if kvs[0].strip() != "":
+                    storage_provider_cfg_map[kvs[0].strip()] = kvs[1].strip()
+
+    # generate storage configuration section in yaml format
+
+    storage_provider_conf_list = [provider_name + ':']
+    for config in storage_provider_cfg_map.items():
+        storage_provider_conf_list.append('{}: {}'.format(*config))
+    storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list)
+    return storage_provider_info