Fix conflicts in Makefiles and prepare script files with upstream

.drone.yml

@@ -50,6 +50,8 @@ pipeline:
       - mail_pwd
       - npm_password
       - npm_username
+      - docker_hub_username
+      - docker_hub_password
     commands:
       - export DOMAIN=${CI_DOMAIN}
       - export HOST_CONTAINER_ID=$(hostname)

.gitmessage

@@ -15,7 +15,7 @@ SUBJECT HERE
 # Description of your commit should go below. Make sure to leave
 # one empty line after your description.
 #
-#
+
 BODY LINE1
 BODY LINE2
 

.travis.yml

@@ -80,7 +80,7 @@ script:
   - sudo mkdir -p /harbor
   - sudo mv ./VERSION /harbor/UIVERSION
   - sudo service postgresql stop
-  - sudo make run_clarity_ut CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.1
+  - sudo make run_clarity_ut CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.0
   - cat ./src/ui_ng/npm-ut-test-results
   - sudo ./tests/testprepare.sh 
   - sudo make -f make/photon/Makefile _build_db _build_registry -e VERSIONTAG=dev -e CLAIRDBVERSION=dev -e REGISTRYVERSION=v2.6.2
@@ -106,7 +106,7 @@ script:
   - sudo rm -rf /data/config/*
   - sudo rm -rf /data/database/*
   - ls /data/cert
-  - sudo make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.1 NOTARYFLAG=true CLAIRFLAG=true
+  - sudo make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.0 NOTARYFLAG=true CLAIRFLAG=true
   - sleep 10  
   - docker ps 
   - ./tests/validatecontainers.sh

Makefile

@@ -102,9 +102,9 @@ NGINXVERSION=$(VERSIONTAG)
 PHOTONVERSION=1.0
 NOTARYVERSION=v0.5.1
 MARIADBVERSION=$(VERSIONTAG)
-CLAIRVERSION=v2.0.1
+CLAIRVERSION=v2.0.4
 CLAIRDBVERSION=$(VERSIONTAG)
-MIGRATORVERSION=v1.5.0
+MIGRATORVERSION=$(VERSIONTAG)
 REDISVERSION=$(VERSIONTAG)
 # version of chartmuseum
 CHARTMUSEUMVERSION=v0.7.1
@@ -140,10 +140,12 @@ GOIMAGEBUILD=$(GOIMAGEBUILDCMD) build
 GOBUILDPATH_ADMINSERVER=$(GOBUILDPATH)/src/adminserver
 GOBUILDPATH_UI=$(GOBUILDPATH)/src/ui
 GOBUILDPATH_JOBSERVICE=$(GOBUILDPATH)/src/jobservice
+GOBUILDPATH_REGISTRYCTL=$(GOBUILDPATH)/src/registryctl
 GOBUILDMAKEPATH=$(GOBUILDPATH)/make
 GOBUILDMAKEPATH_ADMINSERVER=$(GOBUILDMAKEPATH)/dev/adminserver
 GOBUILDMAKEPATH_UI=$(GOBUILDMAKEPATH)/dev/ui
 GOBUILDMAKEPATH_JOBSERVICE=$(GOBUILDMAKEPATH)/dev/jobservice
+GOBUILDMAKEPATH_REGISTRYCTL=$(GOBUILDMAKEPATH)/dev/registryctl
 GOLANGDOCKERFILENAME=Dockerfile.golang
 
 # binary
@@ -153,6 +155,8 @@ UIBINARYPATH=$(MAKEDEVPATH)/ui
 UIBINARYNAME=harbor_ui
 JOBSERVICEBINARYPATH=$(MAKEDEVPATH)/jobservice
 JOBSERVICEBINARYNAME=harbor_jobservice
+REGISTRYCTLBINARYPATH=$(MAKEDEVPATH)/registryctl
+REGISTRYCTLBINARYNAME=harbor_registryctl
 
 # configfile
 CONFIGPATH=$(MAKEPATH)
@@ -188,6 +192,7 @@ DOCKERIMAGENAME_LOG=vmware/harbor-log
 DOCKERIMAGENAME_DB=vmware/harbor-db
 DOCKERIMAGENAME_CLARITY=vmware/harbor-clarity-ui-builder
 DOCKERIMAGENAME_CHART_SERVER=vmware/chartmuseum-photon
+DOCKERIMAGENAME_REGCTL=vmware/harbor-registryctl
 
 # docker-compose files
 DOCKERCOMPOSEFILEPATH=$(MAKEPATH)
@@ -220,8 +225,8 @@ DOCKERSAVE_PARA=$(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) \
 		$(DOCKERIMAGENAME_LOG):$(VERSIONTAG) \
 		$(DOCKERIMAGENAME_DB):$(VERSIONTAG) \
 		$(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) \
+		$(DOCKERIMAGENAME_REGCTL):$(VERSIONTAG) \
 		vmware/redis-photon:$(REDISVERSION) \
-		vmware/harbor-migrator:$(VERSIONTAG) \
 		vmware/nginx-photon:$(NGINXVERSION) vmware/registry-photon:$(REGISTRYVERSION)-$(VERSIONTAG) \
 		vmware/photon:$(PHOTONVERSION)
 PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(PKGVERSIONTAG).tgz \
@@ -229,12 +234,13 @@ PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(PKGVERSIONTAG).tgz \
 				  $(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
 				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
-				  $(HARBORPKG)/ha
+				  $(HARBORPKG)/open_source_license $(HARBORPKG)/ha
 PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(PKGVERSIONTAG).tgz \
 		          $(HARBORPKG)/common/templates $(HARBORPKG)/prepare \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/NOTICE \
 				  $(HARBORPKG)/install.sh $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
-				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/ha
+				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/ha \
+				  $(HARBORPKG)/open_source_license
 DOCKERCOMPOSE_LIST=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 
 ifeq ($(NOTARYFLAG), true)
@@ -292,6 +298,10 @@ compile_golangimage: compile_clarity
 	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_JOBSERVICE) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_JOBSERVICE)/$(JOBSERVICEBINARYNAME)
 	@echo "Done."
 
+	@echo "compiling binary for harbor regsitry controller (golang image)..."
+	@$(DOCKERCMD) run --rm -v $(BUILDPATH):$(GOBUILDPATH) -w $(GOBUILDPATH_REGISTRYCTL) $(GOBUILDIMAGE) $(GOIMAGEBUILD) -o $(GOBUILDMAKEPATH_REGISTRYCTL)/$(REGISTRYCTLBINARYNAME)
+	@echo "Done."
+
 compile:check_environment compile_golangimage
 	
 prepare:
@@ -302,7 +312,7 @@ build:
 	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e MARIADBVERSION=$(MARIADBVERSION) \
 	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) \
 	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION)
+	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION)
 
 modify_composefile: modify_composefile_notary modify_composefile_clair modify_composefile_chartmuseum
 	@echo "preparing docker-compose file..."
@@ -356,6 +366,7 @@ package_online: modify_composefile
 		$(HARBORPKG)/ha/docker-compose.yml ; \
 	fi
 	@cp LICENSE $(HARBORPKG)/LICENSE
+	@cp open_source_license $(HARBORPKG)/open_source_license
 	@cp NOTICE $(HARBORPKG)/NOTICE
 	
 	@$(TARCMD) $(PACKAGE_ONLINE_PARA)
@@ -366,13 +377,9 @@ package_offline: compile version build modify_sourcefiles modify_composefile
 	@echo "packing offline package ..."
 	@cp -r make $(HARBORPKG)
 	@cp LICENSE $(HARBORPKG)/LICENSE
+	@cp open_source_license $(HARBORPKG)/open_source_license
 	@cp NOTICE $(HARBORPKG)/NOTICE
-	@cp $(HARBORPKG)/photon/db/registry.sql $(HARBORPKG)/ha/
+	@cp $(HARBORPKG)/photon/db/initial-registry.sql $(HARBORPKG)/ha/
-	
-	@if [ "$(MIGRATORFLAG)" = "true" ] ; then \
-		echo "pulling Harbor migrator..."; \
-		$(DOCKERPULL) vmware/harbor-migrator:$(MIGRATORVERSION); \
-	fi
 
 	@echo "saving harbor docker image"
 	@$(DOCKERSAVE) $(DOCKERSAVE_PARA) > $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tar

README.md

@@ -38,6 +38,8 @@ Refer to **[User Guide](docs/user_guide.md)** for more details on how to use Har
 **Developer Group:** Join Harbor developer group: [harbor-dev@googlegroups.com](https://groups.google.com/forum/#!forum/harbor-dev) for discussion on Harbor development and contribution. To subscribe, send an email to [harbor-dev+subscribe@googlegroups.com](mailto:harbor-dev+subscribe@googlegroups.com).  
 **Slack:** Join Harbor's community for discussion and ask questions: [VMware {code}](https://code.vmware.com/join/), channel: [#harbor](https://vmwarecode.slack.com/messages/harbor).
 
+**Demo Server:** Harbor provides a demo environment with the latest Harbor stable build installed. If you want to have a try, please refer to **[Demo Server](docs/demo_server.md)** for more details.
+
 More info on [partners and users](partners.md).
 
 ### Contribution
@@ -56,7 +58,6 @@ Harbor is available under the [Apache 2 license](LICENSE).
 This project uses open source components which have additional licensing terms.  The official docker images and licensing terms for these open source components can be found at the following locations:
 
 * Photon OS 1.0: [docker image](https://hub.docker.com/_/photon/), [license](https://github.com/vmware/photon/blob/master/COPYING)
-* MySQL 5.6: [docker image](https://hub.docker.com/_/mysql/), [license](https://github.com/docker-library/mysql/blob/master/LICENSE)
 
 ### Commercial Support
 If you need commercial support of Harbor, please contact us for more information: <img alt="email" src="docs/img/harbor_email.png" valigin="middle" height="18"/>.

contrib/helm/harbor/requirements.lock

@@ -1,7 +1,4 @@
 dependencies:
-- name: postgresql
-  repository: https://kubernetes-charts.storage.googleapis.com
-  version: 0.9.1
 - name: redis
   repository: https://kubernetes-charts.storage.googleapis.com
   version: 3.2.5

contrib/helm/harbor/requirements.yaml

@@ -1,7 +1,4 @@
 dependencies:
-- name: postgresql
-  version: 0.9.1
-  repository: https://kubernetes-charts.storage.googleapis.com
 - name: redis
   version: 3.2.5
   repository: https://kubernetes-charts.storage.googleapis.com

contrib/helm/harbor/templates/NOTES.txt

@@ -1,26 +1,3 @@
-
 Please wait for several minutes for Harbor deployment to complete.
-Then follow the steps below to use Harbor.
+Then you should be able to visit the UI portal at {{ template "harbor.externalURL" . }}. 
-
+For more details, please visit https://github.com/vmware/harbor.
-1. Add the Harbor CA certificate to Docker by executing the following command:
-
-  sudo mkdir -p /etc/docker/certs.d/{{ .Values.externalDomain }}
-  kubectl get secret \
-    --namespace {{ .Release.Namespace }} {{ template "harbor.fullname" . }}-ingress \
-    -o jsonpath="{.data.ca\.crt}" | base64 --decode | \
-    sudo tee /etc/docker/certs.d/{{ .Values.externalDomain }}/ca.crt
-
-2. Get Harbor admin password by executing the following command:
-
-  kubectl get secret --namespace {{ .Release.Namespace }} {{ template "harbor.fullname" . }}-adminserver -o jsonpath="{.data.HARBOR_ADMIN_PASSWORD}" | base64 --decode; echo
-
-3. Add DNS resolution entry for Harbor FQDN {{ .Values.externalDomain }} to K8s Ingress Controller IP on DNS Server or in file /etc/hosts.
-{{- if .Values.notary.enabled }}
-   Add DNS resolution entry for Notary FQDN {{ template "harbor.notaryFQDN" . }} to K8s Ingress Controller IP on DNS Server or in file /etc/hosts.
-{{- end }}
-
-4. Access Harbor UI via https://{{ .Values.externalDomain }}
-
-5. Login Harbor with Docker CLI:
-
-  docker login {{ .Values.externalDomain }}

contrib/helm/harbor/templates/_helpers.tpl

@@ -32,9 +32,9 @@ app: "{{ template "harbor.name" . }}"
 
 {{- define "harbor.externalURL" -}}
 {{- if .Values.externalPort -}}
-{{- printf "%s:%s" .Values.externalDomain (toString .Values.externalPort) -}}
+{{- printf "%s://%s:%s" .Values.externalProtocol .Values.externalDomain (toString .Values.externalPort) -}}
 {{- else -}}
-{{- .Values.externalDomain -}}
+{{- printf "%s://%s" .Values.externalProtocol .Values.externalDomain -}}
 {{- end -}}
 {{- end -}}
 
@@ -57,3 +57,87 @@ so it can match Harbor service FQDN and Notary service FQDN.
 {{- define "harbor.notaryServiceName" -}}
 {{- printf "%s-notary-server" (include "harbor.fullname" .) -}}
 {{- end -}}
+
+{{- define "harbor.database.host" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- template "harbor.fullname" . }}-database
+  {{- else -}}
+    {{- .Values.database.external.host -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.port" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- printf "%s" "5432" -}}
+  {{- else -}}
+    {{- .Values.database.external.port -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.username" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- printf "%s" "postgres" -}}
+  {{- else -}}
+    {{- .Values.database.external.username -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.password" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- .Values.database.internal.password | b64enc | quote -}}
+  {{- else -}}
+    {{- .Values.database.external.password | b64enc | quote -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.rawPassword" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- .Values.database.internal.password -}}
+  {{- else -}}
+    {{- .Values.database.external.password -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.coreDatabase" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- printf "%s" "registry" -}}
+  {{- else -}}
+    {{- .Values.database.external.coreDatabase -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.clairDatabase" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- printf "%s" "postgres" -}}
+  {{- else -}}
+    {{- .Values.database.external.clairDatabase -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.notaryServerDatabase" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- printf "%s" "notaryserver" -}}
+  {{- else -}}
+    {{- .Values.database.external.notaryServerDatabase -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.notarySignerDatabase" -}}
+  {{- if eq .Values.database.type "internal" -}}
+    {{- printf "%s" "notarysigner" -}}
+  {{- else -}}
+    {{- .Values.database.external.notarySignerDatabase -}}
+  {{- end -}}
+{{- end -}}
+
+{{- define "harbor.database.clair" -}}
+postgres://{{ template "harbor.database.username" . }}:{{ template "harbor.database.rawPassword" . }}@{{ template "harbor.database.host" . }}:{{ template "harbor.database.port" . }}/{{ template "harbor.database.clairDatabase" . }}?sslmode=disable
+{{- end -}}
+
+{{- define "harbor.database.notaryServer" -}}
+postgres://{{ template "harbor.database.username" . }}:{{ template "harbor.database.rawPassword" . }}@{{ template "harbor.database.host" . }}:{{ template "harbor.database.port" . }}/{{ template "harbor.database.notaryServerDatabase" . }}?sslmode=disable
+{{- end -}}
+
+{{- define "harbor.database.notarySigner" -}}
+postgres://{{ template "harbor.database.username" . }}:{{ template "harbor.database.rawPassword" . }}@{{ template "harbor.database.host" . }}:{{ template "harbor.database.port" . }}/{{ template "harbor.database.notarySignerDatabase" . }}?sslmode=disable
+{{- end -}}

contrib/helm/harbor/templates/adminserver/adminserver-cm.yaml

@@ -6,22 +6,18 @@ metadata:
 {{ include "harbor.labels" . | indent 4 }}
     component: adminserver
 data:
-  {{ if .Values.mysql.host -}}
+  POSTGRESQL_HOST: "{{ template "harbor.database.host" . }}"
-  MYSQL_HOST: "{{ .Values.mysql.host }}"
+  POSTGRESQL_PORT: "{{ template "harbor.database.port" . }}"
-  {{ else -}}
+  POSTGRESQL_USERNAME: "{{ template "harbor.database.username" . }}"
-  MYSQL_HOST: "{{ template "harbor.fullname" . }}-mysql"
+  POSTGRESQL_DATABASE: "{{ template "harbor.database.coreDatabase" . }}"
-  {{ end -}}
+  EMAIL_HOST: "{{ .Values.email.host }}"
-  MYSQL_PORT: "{{ .Values.mysql.port }}"
+  EMAIL_PORT: "{{ .Values.email.port }}"
-  MYSQL_USR: "{{ .Values.mysql.user }}"
+  EMAIL_USR: "{{ .Values.email.username }}"
-  MYSQL_DATABASE: "{{ .Values.mysql.database }}"
+  EMAIL_SSL: "{{ .Values.email.ssl }}"
-  EMAIL_HOST: "{{ .Values.adminserver.emailHost }}"
+  EMAIL_FROM: "{{ .Values.email.from }}"
-  EMAIL_PORT: "{{ .Values.adminserver.emailPort }}"
+  EMAIL_IDENTITY: "{{ .Values.email.identity }}"
-  EMAIL_USR: "{{ .Values.adminserver.emailUser }}"
+  EMAIL_INSECURE: "{{ .Values.email.insecure }}"
-  EMAIL_SSL: "{{ .Values.adminserver.emailSsl }}"
+  EXT_ENDPOINT: "{{ template "harbor.externalURL" . }}"
-  EMAIL_FROM: "{{ .Values.adminserver.emailFrom }}"
-  EMAIL_IDENTITY: "{{ .Values.adminserver.emailIdentity }}"
-  EMAIL_INSECURE: "{{ .Values.adminserver.emailInsecure }}"
-  EXT_ENDPOINT: "https://{{ .Values.externalDomain }}"
   UI_URL: "http://{{ template "harbor.fullname" . }}-ui"
   JOBSERVICE_URL: "http://{{ template "harbor.fullname" . }}-jobservice"
   REGISTRY_URL: "http://{{ template "harbor.fullname" . }}-registry:5000"
@@ -30,17 +26,17 @@ data:
   NOTARY_URL: "http://{{ template "harbor.notaryServiceName" . }}:4443"
   LOG_LEVEL: "info"
   IMAGE_STORE_PATH: "/" # This is a temporary hack.
-  AUTH_MODE: "{{ .Values.adminserver.authenticationMode }}"
+  AUTH_MODE: "{{ .Values.authenticationMode }}"
-  SELF_REGISTRATION: "{{ .Values.adminserver.selfRegistration }}"
+  SELF_REGISTRATION: "{{ .Values.selfRegistration }}"
-  LDAP_URL: "{{ .Values.adminserver.ldap.url }}"
+  LDAP_URL: "{{ .Values.ldap.url }}"
-  LDAP_SEARCH_DN: "{{ .Values.adminserver.ldap.searchDN }}"
+  LDAP_SEARCH_DN: "{{ .Values.ldap.searchDN }}"
-  LDAP_BASE_DN: "{{ .Values.adminserver.ldap.baseDN }}"
+  LDAP_BASE_DN: "{{ .Values.ldap.baseDN }}"
-  LDAP_FILTER: "{{ .Values.adminserver.ldap.filter }}"
+  LDAP_FILTER: "{{ .Values.ldap.filter }}"
-  LDAP_UID: "{{ .Values.adminserver.ldap.uid }}"
+  LDAP_UID: "{{ .Values.ldap.uid }}"
-  LDAP_SCOPE: "{{ .Values.adminserver.ldap.scope }}"
+  LDAP_SCOPE: "{{ .Values.ldap.scope }}"
-  LDAP_TIMEOUT: "{{ .Values.adminserver.ldap.timeout }}"
+  LDAP_TIMEOUT: "{{ .Values.ldap.timeout }}"
-  LDAP_VERIFY_CERT: "{{ .Values.adminserver.ldap.verifyCert }}"
+  LDAP_VERIFY_CERT: "{{ .Values.ldap.verifyCert }}"
-  DATABASE_TYPE: "mysql"
+  DATABASE_TYPE: "postgresql"
   PROJECT_CREATION_RESTRICTION: "everyone"
   VERIFY_REMOTE_CERT: "off"
   MAX_JOB_WORKERS: "3"
@@ -50,10 +46,10 @@ data:
   ADMIRAL_URL: "NA"
   RESET: "false"
   WITH_CLAIR: "{{ .Values.clair.enabled }}"
-  CLAIR_DB_HOST: "{{ .Release.Name }}-postgresql"
+  CLAIR_DB_HOST: "{{ template "harbor.database.host" . }}"
-  CLAIR_DB_PORT: "5432"
+  CLAIR_DB_PORT: "{{ template "harbor.database.port" . }}"
-  CLAIR_DB: "{{ .Values.clair.postgresDatabase }}"
+  CLAIR_DB_USERNAME: "{{ template "harbor.database.username" . }}"
-  CLAIR_DB_USERNAME: "{{ .Values.clair.postgresUser }}"
+  CLAIR_DB: "{{ template "harbor.database.clairDatabase" . }}"
   CLAIR_URL: "http://{{ template "harbor.fullname" . }}-clair:6060"
   UAA_ENDPOINT: ""
   UAA_CLIENTID: ""

contrib/helm/harbor/templates/adminserver/adminserver-secrets.yaml

@@ -8,14 +8,14 @@ metadata:
 type: Opaque
 data:
   secretKey: {{ .Values.secretKey | b64enc | quote }}
-  EMAIL_PWD: {{ .Values.adminserver.emailPwd | b64enc | quote }}
+  EMAIL_PWD: {{ .Values.email.password | b64enc | quote }}
-  HARBOR_ADMIN_PASSWORD: {{ .Values.adminserver.adminPassword | b64enc | quote }}
+  HARBOR_ADMIN_PASSWORD: {{ .Values.harborAdminPassword | b64enc | quote }}
-  MYSQL_PWD: {{ .Values.mysql.pass | b64enc | quote }}
+  POSTGRESQL_PASSWORD: {{ template "harbor.database.password" . }}
   JOBSERVICE_SECRET: {{ .Values.jobservice.secret | b64enc | quote }}
   UI_SECRET: {{ .Values.ui.secret | b64enc | quote }}
-{{- if eq .Values.adminserver.authenticationMode "ldap_auth" }} 
+{{- if eq .Values.authenticationMode "ldap_auth" }} 
-  LDAP_SEARCH_PWD: {{ .Values.adminserver.ldap.searchPwd | b64enc | quote }} 
+  LDAP_SEARCH_PWD: {{ .Values.ldap.searchPassword | b64enc | quote }} 
 {{- end }}
 {{ if .Values.clair.enabled }}
-  CLAIR_DB_PASSWORD: {{ .Values.clair.postgresPassword | b64enc | quote }}
+  CLAIR_DB_PASSWORD: {{ template "harbor.database.password" . }}
 {{ end }}

contrib/helm/harbor/templates/clair/clair-cm.yaml

@@ -12,7 +12,7 @@ data:
       database:
         type: pgsql
         options:
-          source: "postgresql://{{ .Values.clair.postgresUser }}:{{ .Values.clair.postgresPassword }}@{{ .Release.Name }}-postgresql:5432/{{ .Values.clair.postgresDatabase }}?sslmode=disable"
+          source: "{{ template "harbor.database.clair" . }}"
           # Number of elements kept in the cache
           # Values unlikely to change (e.g. namespaces) are cached in order to save prevent needless roundtrips to the database.
           cachesize: 16384

contrib/helm/harbor/templates/database/database-secret.yaml

@@ -0,0 +1,11 @@
+{{- if eq .Values.database.type "internal" -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "{{ template "harbor.fullname" . }}-database"
+  labels:
+{{ include "harbor.labels" . | indent 4 }}
+type: Opaque
+data:
+  POSTGRES_PASSWORD: {{ template "harbor.database.password" . }}
+{{- end -}}

contrib/helm/harbor/templates/database/database-ss.yaml

@@ -0,0 +1,70 @@
+{{- if eq .Values.database.type "internal" -}}
+apiVersion: apps/v1beta2
+kind: StatefulSet
+metadata:
+  name: "{{ template "harbor.fullname" . }}-database"
+  labels:
+{{ include "harbor.labels" . | indent 4 }}
+    component: database
+spec:
+  replicas: 1
+  serviceName: "{{ template "harbor.fullname" . }}-database"
+  selector:
+    matchLabels:
+{{ include "harbor.matchLabels" . | indent 6 }}
+      component: database
+  template:
+    metadata:
+      labels:
+{{ include "harbor.labels" . | indent 8 }}
+        component: database
+    spec:
+      containers:
+      - name: database
+        image: {{ .Values.database.internal.image.repository }}:{{ .Values.database.internal.image.tag }}
+        imagePullPolicy: {{ .Values.database.internal.image.pullPolicy }}
+        resources:
+{{ toYaml .Values.database.internal.resources | indent 10 }}
+        envFrom:
+          - secretRef:
+              name: "{{ template "harbor.fullname" . }}-database"
+        volumeMounts:
+        - name: database-data
+          mountPath: /var/lib/postgresql/data
+      {{- if not .Values.persistence.enabled }}
+      volumes:
+      - name: "database-data"
+        emptyDir: {}
+      {{- end -}}
+    {{- with .Values.database.internal.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.database.internal.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.database.internal.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+  {{- if .Values.persistence.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: "database-data"
+      labels:
+{{ include "harbor.labels" . | indent 8 }}
+    spec:
+      accessModes: [{{ .Values.database.internal.volumes.data.accessMode | quote }}]
+      {{- if .Values.database.internal.volumes.data.storageClass }}
+      {{- if (eq "-" .Values.database.internal.volumes.data.storageClass) }}
+      storageClassName: ""
+      {{- else }}
+      storageClassName: "{{ .Values.database.internal.volumes.data.storageClass }}"
+      {{- end }}
+      {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.database.internal.volumes.data.size | quote }}
+  {{- end -}}
+  {{- end -}}

contrib/helm/harbor/templates/database/database-svc.yaml

@@ -1,12 +1,14 @@
+{{- if eq .Values.database.type "internal" -}}
 apiVersion: v1
 kind: Service
 metadata:
-  name: "{{ template "harbor.fullname" . }}-mysql"
+  name: "{{ template "harbor.fullname" . }}-database"
   labels:
 {{ include "harbor.labels" . | indent 4 }}
 spec:
   ports:
-    - port: 3306
+    - port: 5432
   selector:
 {{ include "harbor.matchLabels" . | indent 4 }}
-    component: mysql
+    component: database
+{{- end -}}

contrib/helm/harbor/templates/ingress/ingress.yaml

@@ -8,12 +8,16 @@ metadata:
   annotations:
 {{ toYaml .Values.ingress.annotations | indent 4 }}
 spec:
-{{ if not .Values.insecureRegistry }}
+{{ if eq .Values.externalProtocol "https" }}
   tls:
   - hosts:
     - "{{ .Values.externalDomain }}"
     - "{{ template "harbor.notaryFQDN" . }}"
+    {{ if eq .Values.ingress.tls.secretName "" }}
     secretName: "{{ template "harbor.fullname" . }}-ingress"
+    {{ else }}
+    secretName: {{ .Values.ingress.tls.secretName }}
+    {{ end }}
 {{ end }}
   rules:
   - host: "{{ .Values.externalDomain }}"
@@ -43,11 +47,15 @@ metadata:
     nginx.ingress.kubernetes.io/rewrite-target: /registryproxy/v2
     ingress.kubernetes.io/rewrite-target: /registryproxy/v2
 spec:
-{{ if not .Values.insecureRegistry }}
+{{ if eq .Values.externalProtocol "https" }}
   tls:
   - hosts:
     - "{{ .Values.externalDomain }}"
+    {{ if eq .Values.ingress.tls.secretName "" }}
     secretName: "{{ template "harbor.fullname" . }}-ingress"
+    {{ else }}
+    secretName: {{ .Values.ingress.tls.secretName }}
+    {{ end }}
 {{ end }}
   rules:
   - host: "{{ .Values.externalDomain }}"
@@ -57,6 +65,4 @@ spec:
         backend:
           serviceName: {{ template "harbor.fullname" . }}-ui
           servicePort: 80
-
-
 {{ end }}

contrib/helm/harbor/templates/ingress/secret.yaml

@@ -1,5 +1,6 @@
-{{ if not .Values.insecureRegistry }}
+{{ if eq .Values.externalProtocol "https" }}
-{{ if .Values.generateCertificates }}
+{{ if .Values.ingress.enabled }}
+{{ if eq .Values.ingress.tls.secretName "" }}
 {{ $ca := genCA "harbor-ca" 3650 }}
 {{ $cert := genSignedCert (include "harbor.certCommonName" .) nil nil 3650 $ca }}
 apiVersion: v1
@@ -15,3 +16,4 @@ data:
   ca.crt: {{ .Values.caCrt | default $ca.Cert | b64enc | quote }}
 {{ end }}
 {{ end }}
+{{ end }}

contrib/helm/harbor/templates/mysql/mysql-secret.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: "{{ template "harbor.fullname" . }}-mysql"
-  labels:
-{{ include "harbor.labels" . | indent 4 }}
-type: Opaque
-data:
-  mysqlRootPassword: {{ .Values.mysql.pass | b64enc | quote }}

contrib/helm/harbor/templates/mysql/mysql-ss.yaml

@@ -1,71 +0,0 @@
-apiVersion: apps/v1beta2
-kind: StatefulSet
-metadata:
-  name: "{{ template "harbor.fullname" . }}-mysql"
-  labels:
-{{ include "harbor.labels" . | indent 4 }}
-    component: mysql
-spec:
-  replicas: 1
-  serviceName: "{{ template "harbor.fullname" . }}-mysql"
-  selector:
-    matchLabels:
-{{ include "harbor.matchLabels" . | indent 6 }}
-      component: mysql
-  template:
-    metadata:
-      labels:
-{{ include "harbor.labels" . | indent 8 }}
-        component: mysql
-    spec:
-      containers:
-      - name: mysql
-        image: {{ .Values.mysql.image.repository }}:{{ .Values.mysql.image.tag }}
-        imagePullPolicy: {{ .Values.mysql.image.pullPolicy }}
-        resources:
-{{ toYaml .Values.mysql.resources | indent 10 }}
-        env:
-          - name: MYSQL_ROOT_PASSWORD
-            valueFrom:
-              secretKeyRef:
-               name: "{{ template "harbor.fullname" . }}-mysql"
-               key: mysqlRootPassword
-        volumeMounts:
-        - name: mysql-data
-          mountPath: /var/lib/mysql
-      {{- if not .Values.persistence.enabled }}
-      volumes:
-      - name: "mysql-data"
-        emptyDir: {}
-      {{- end -}}
-    {{- with .Values.mysql.nodeSelector }}
-      nodeSelector:
-{{ toYaml . | indent 8 }}
-    {{- end }}
-    {{- with .Values.mysql.affinity }}
-      affinity:
-{{ toYaml . | indent 8 }}
-    {{- end }}
-    {{- with .Values.mysql.tolerations }}
-      tolerations:
-{{ toYaml . | indent 8 }}
-    {{- end }}
-  {{- if .Values.persistence.enabled }}
-  volumeClaimTemplates:
-  - metadata:
-      name: "mysql-data"
-      labels:
-{{ include "harbor.labels" . | indent 8 }}
-    spec:
-      accessModes: [{{ .Values.mysql.volumes.data.accessMode | quote }}]
-      {{- if .Values.mysql.volumes.data.storageClass }}
-      {{- if (eq "-" .Values.mysql.volumes.data.storageClass) }}
-      storageClassName: ""
-      {{- else }}
-      storageClassName: "{{ .Values.mysql.volumes.data.storageClass }}"
-      {{- end }}
-      {{- end }}
-      resources:
-        requests:
-          storage: {{ .Values.mysql.volumes.data.size | quote }}
-  {{- end -}}

contrib/helm/harbor/templates/notary/notary-cm.yaml

@@ -1,23 +1,4 @@
 {{ if .Values.notary.enabled }}
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: {{ template "harbor.fullname" . }}-notary-db
-  labels:
-{{ include "harbor.labels" . | indent 4 }}
-    component: notary-db
-data:
-  initial-notaryserver.sql: |
-    CREATE DATABASE IF NOT EXISTS `notaryserver`;
-    CREATE USER "server"@"%" IDENTIFIED BY "{{ .Values.notary.db.password }}";
-    GRANT ALL PRIVILEGES ON `notaryserver`.* TO "server"@"%"
-  initial-notarysigner.sql: |
-    CREATE DATABASE IF NOT EXISTS `notarysigner`;
-    CREATE USER "signer"@"%" IDENTIFIED BY "{{ .Values.notary.db.password }}";
-    GRANT ALL PRIVILEGES ON `notarysigner`.* TO "signer"@"%";
-
----
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -34,7 +15,7 @@ data:
 {{ .Values.notary.signer.tlsCrt | default $cert.Cert | indent 4 }}
   notary-signer.key: |
 {{ .Values.notary.signer.tlsKey | default $cert.Key  | indent 4 }}
-  server-config.json: |
+  server-config.postgres.json: |
     {
       "server": {
         "http_addr": ":4443"
@@ -50,20 +31,20 @@ data:
         "level": "debug"
       },
       "storage": {
-        "backend": "mysql",
+        "backend": "postgres",
-        "db_url": "server:{{ .Values.notary.db.password }}@tcp({{ template "harbor.fullname" . }}-notary-db:3306)/notaryserver?parseTime=True"
+        "db_url": "{{ template "harbor.database.notaryServer" . }}"
       },
       "auth": {
           "type": "token",
           "options": {
-              "realm": "https://{{ template "harbor.externalURL" . }}/service/token",
+              "realm": "{{ template "harbor.externalURL" . }}/service/token",
               "service": "harbor-notary",
               "issuer": "harbor-token-issuer",
               "rootcertbundle": "/root.crt"
           }
       }
     }
-  signer-config.json: |
+  signer-config.postgres.json: |
     {
       "server": {
         "grpc_addr": ":7899",
@@ -74,8 +55,8 @@ data:
         "level": "debug"
       },
       "storage": {
-        "backend": "mysql",
+        "backend": "postgres",
-        "db_url": "signer:{{ .Values.notary.db.password }}@tcp({{ template "harbor.fullname" . }}-notary-db:3306)/notarysigner?parseTime=True",
+        "db_url": "{{ template "harbor.database.notarySigner" . }}",
         "default_alias": "defaultalias"
       }
     }

contrib/helm/harbor/templates/notary/notary-db.yaml

@@ -1,83 +0,0 @@
-{{ if .Values.notary.enabled }}
-apiVersion: apps/v1beta2
-kind: StatefulSet
-metadata:
-  name: {{ template "harbor.fullname" . }}-notary-db
-  labels:
-{{ include "harbor.labels" . | indent 4 }}
-    component: notary-db
-spec:
-  replicas: 1
-  serviceName: "{{ template "harbor.fullname" . }}-notary-db"
-  selector:
-    matchLabels:
-{{ include "harbor.matchLabels" . | indent 6 }}
-      component: notary-db
-  template:
-    metadata:
-      labels:
-{{ include "harbor.labels" . | indent 8 }}
-        component: notary-db
-    spec:
-      initContainers: 
-      - name: move-init-scripts
-        image: alpine:3.6
-        imagePullPolicy: IfNotPresent
-        command: [
-          "sh",
-          "-ce",
-          "ls -l /configmap/docker-entrypoint-initdb.d/* &&
-           cp -L /configmap/docker-entrypoint-initdb.d/* /docker-entrypoint-initdb.d/"
-        ]
-        volumeMounts:
-        - name: notary-db-config
-          mountPath: /configmap/docker-entrypoint-initdb.d
-        - name: notary-db-init
-          mountPath: /docker-entrypoint-initdb.d
-      containers:
-      - name: notary-db
-        image: {{ .Values.notary.db.image.repository }}:{{ .Values.notary.db.image.tag }}
-        imagePullPolicy: {{ .Values.notary.db.image.pullPolicy }}
-        args: ["--innodb_file_per_table"]
-        env:
-          - name: TERM
-            value: "dumb"
-          - name: MYSQL_ALLOW_EMPTY_PASSWORD
-            value: "true"
-        resources:
-{{ toYaml .Values.notary.db.resources | indent 10 }}
-        volumeMounts:
-        - name: notary-db-config
-          mountPath: /configmap/docker-entrypoint-initdb.d
-        - name: notary-db-init
-          mountPath: /docker-entrypoint-initdb.d
-        - name: notary-db-data
-          mountPath: /var/lib/mysql
-      volumes:
-      - name: notary-db-config
-        configMap:
-          name: "{{ template "harbor.fullname" . }}-notary-db"
-      - name: notary-db-init
-        emptyDir: {}
-      {{- if not .Values.persistence.enabled }}
-      - name: notary-db-data
-        emptyDir: {}
-      {{- end -}}
-  {{- if .Values.persistence.enabled }}
-  volumeClaimTemplates:
-  - metadata:
-      name: notary-db-data
-    spec:
-      accessModes: [{{ .Values.notary.db.volumes.data.accessMode | quote }}]
-      {{- if .Values.notary.db.volumes.data.storageClass }}
-      {{- if (eq "-" .Values.notary.db.volumes.data.storageClass) }}
-      storageClassName: ""
-      {{- else }}
-      storageClassName: "{{ .Values.notary.db.volumes.data.storageClass }}"
-      {{- end }}
-      {{- end }}
-      resources:
-        requests:
-          storage: {{ .Values.notary.db.volumes.data.size | quote }}
-  {{- end -}}
-{{ end }}

contrib/helm/harbor/templates/notary/notary-server.yaml

@@ -25,8 +25,10 @@ spec:
         resources:
 {{ toYaml .Values.notary.server.resources | indent 10 }}
         env:
+        - name: MIGRATIONS_PATH
+          value: migrations/server/postgresql
         - name: DB_URL
-          value: "mysql://server:{{ .Values.notary.db.password }}@tcp({{ template "harbor.fullname" . }}-notary-db:3306)/notaryserver?parseTime=True"
+          value: {{ template "harbor.database.notaryServer" . }}
         volumeMounts:
         - name: notary-config
           mountPath: /etc/notary

contrib/helm/harbor/templates/notary/notary-signer.yaml

@@ -25,8 +25,10 @@ spec:
         resources:
 {{ toYaml .Values.notary.signer.resources | indent 10 }}
         env:
+        - name: MIGRATIONS_PATH
+          value: migrations/signer/postgresql
         - name: DB_URL
-          value: "mysql://signer:{{ .Values.notary.db.password }}@tcp({{ template "harbor.fullname" . }}-notary-db:3306)/notarysigner?parseTime=True"
+          value: {{ template "harbor.database.notarySigner" . }}
         - name: NOTARY_SIGNER_DEFAULTALIAS
           value: {{ .Values.notary.signer.env.NOTARY_SIGNER_DEFAULTALIAS }}
         volumeMounts:

contrib/helm/harbor/templates/notary/notary-svc.yaml

@@ -1,19 +1,4 @@
 {{ if .Values.notary.enabled }}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ template "harbor.fullname" . }}-notary-db
-  labels:
-{{ include "harbor.labels" . | indent 4 }}
-spec:
-  ports:
-  - port: 3306
-  selector:
-{{ include "harbor.matchLabels" . | indent 4 }}
-    component: notary-db
-
----
 apiVersion: v1
 kind: Service
 metadata:

contrib/helm/harbor/templates/registry/registry-cm.yaml

@@ -38,7 +38,7 @@ data:
     auth:
       token:
         issuer: harbor-token-issuer
-        realm: "https://{{ template "harbor.externalURL" . }}/service/token"
+        realm: "{{ template "harbor.externalURL" . }}/service/token"
         rootcertbundle: /etc/registry/root.crt
         service: harbor-registry
 

contrib/helm/harbor/templates/ui/ui-dpl.yaml

@@ -50,11 +50,15 @@ spec:
         - name: ui-secrets-private-key
           mountPath: /etc/ui/private_key.pem
           subPath: private_key.pem
-        {{- if and (not .Values.insecureRegistry) .Values.ingress.enabled }}
+        {{- if eq .Values.externalProtocol "https" }}
+        {{- if .Values.ingress.enabled }}
+        {{- if eq .Values.ingress.tls.secretName "" }}
         - name: ca-download
           mountPath: /etc/ui/ca/ca.crt
           subPath: ca.crt
         {{- end }}
+        {{- end }}
+        {{- end }}
         - name: psc
           mountPath: /etc/ui/token
       volumes:
@@ -73,7 +77,9 @@ spec:
           items:
             - key: private_key.pem
               path: private_key.pem
-      {{- if and (not .Values.insecureRegistry) .Values.ingress.enabled }}
+      {{- if eq .Values.externalProtocol "https" }}
+      {{- if .Values.ingress.enabled }}
+      {{- if eq .Values.ingress.tls.secretName "" }}
       - name: ca-download
         secret:
           secretName: "{{ template "harbor.fullname" . }}-ingress"
@@ -81,6 +87,8 @@ spec:
             - key: ca.crt
               path: ca.crt
       {{- end }}
+      {{- end }}
+      {{- end }}
       - name: psc
         emptyDir: {}
     {{- with .Values.ui.nodeSelector }}

contrib/helm/harbor/values.yaml

@@ -1,49 +1,33 @@
-# Configure persisten Volumes per application
-## Applications that require storage have a `volumes` definition which will be used
-## when `persistence.enabled` is set to true.
-## example
-# mysql:
-#   volumes:
-#     data:
-## Persistent Volume Storage Class
-## If defined, storageClassName: <storageClass>
-## If set to "-", storageClassName: "", which disables dynamic provisioning
-## If undefined (the default) or set to null, no storageClassName spec is
-##   set, choosing the default provisioner.  (gp2 on AWS, standard on
-##   GKE, AWS & OpenStack)
-##
-#     storageClass: "-"
-#     accessMode: ReadWriteOnce
-#     size: 1Gi
-
-## Configure resource requests and limits per application
-## ref: http://kubernetes.io/docs/user-guide/compute-resources/
-##
-# mysql:
-#   resources:
-#     requests:
-#       memory: 256Mi
-#       cpu: 100m
-
 persistence:
   enabled: true
-
+externalProtocol: https
-# The tag for Harbor docker images.
+# The FQDN for Harbor service
-harborImageTag: &harbor_image_tag v1.5.0-chart-patch
-
-# The FQDN for Harbor service.
 externalDomain: harbor.my.domain
-# externalPort is the Port for Harbor service, leave empty if the service is to be bound to
+# The Port for Harbor service, leave empty if the service 
-# port 80/443
+# is to be bound to port 80/443
 externalPort: 32700
-# If set to true, you don't need to set tlsCrt/tlsKey/caCrt, but must add
+harborAdminPassword: Harbor12345
-# Harbor FQDN as insecure-registries for your docker client.
+authenticationMode: "db_auth"
-insecureRegistry: false
+selfRegistration: "on"
-generateCertificates: true
+ldap:
-# The TLS certificate for Harbor. The common name of tlsCrt must match the externalDomain above.
+  url: "ldaps://ldapserver"
-tlsCrt:
+  searchDN: ""
-tlsKey:
+  searchPassword: ""
-caCrt:
+  baseDN: ""
+  filter: "(objectClass=person)"
+  uid: "uid"
+  scope: "2"
+  timeout: "5"
+  verifyCert: "True"
+email:
+  host: "smtp.mydomain.com"
+  port: "25"
+  username: "sample_admin@mydomain.com"
+  password: "password"
+  ssl: "false"
+  insecure: "false"
+  from: "admin <sample_admin@mydomain.com>"
+  identity: ""
 
 # The secret key used for encryption. Must be a string of 16 chars.
 secretKey: not-a-secure-key
@@ -54,36 +38,23 @@ ingress:
   enabled: true
   annotations:
     ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
     ingress.kubernetes.io/proxy-body-size: "0"
     nginx.ingress.kubernetes.io/proxy-body-size: "0"
+  tls:
+    # Fill the secretName if you want to use the certificate of 
+    # yourself when Harbor serves with HTTPS. A certificate will 
+    # be generated automatically by the chart if leave it empty
+    secretName: ""
+
+# The tag for Harbor docker images.
+harborImageTag: &harbor_image_tag dev
 
 adminserver:
   image:
     repository: vmware/harbor-adminserver
     tag: *harbor_image_tag
     pullPolicy: IfNotPresent
-  emailHost: "smtp.mydomain.com"
-  emailPort: "25"
-  emailUser: "sample_admin@mydomain.com"
-  emailSsl: "false"
-  emailFrom: "admin <sample_admin@mydomain.com>"
-  emailIdentity: ""
-  emailInsecure: "False"
-  emailPwd: not-a-secure-password
-  adminPassword: Harbor12345
-  authenticationMode: "db_auth"
-  selfRegistration: "on"
-  ldap:
-    url: "ldaps://ldapserver"
-    searchDN: ""
-    searchPassword: ""
-    baseDN: ""
-    filter: "(objectClass=person)"
-    uid: "uid"
-    scope: "2"
-    timeout: "5"
-    verifyCert: "True"
-  ## Persist data to a persistent volume
   volumes:
     config:
       # storageClass: "-"
@@ -97,8 +68,6 @@ adminserver:
   tolerations: []
   affinity: {}
 
-## jobservice
-#
 jobservice:
   image:
     repository: vmware/harbor-jobservice
@@ -114,8 +83,6 @@ jobservice:
   tolerations: []
   affinity: {}
 
-## UI
-#
 ui:
   image:
     repository: vmware/harbor-ui
@@ -182,21 +149,17 @@ ui:
   tolerations: []
   affinity: {}
 
-## MySQL Settings. Currently Harbor does not support an external
+database:
-## MySQL server, only their own image. Until this is fixed, do not
+  # if external database is used, set "type" to "external"
-## Change the settings below.
+  # and fill the connection informations in "external" section
-#
+  type: internal
-mysql:
+  internal:
     image:
       repository: vmware/harbor-db
       tag: *harbor_image_tag
       pullPolicy: IfNotPresent
-  # If left blank will use the included mysql service name.
+    # the superuser password of database
-  host: ~
+    password: "changeit"
-  port: 3306
-  user: "root"
-  pass: "registry"
-  database: "registry"
     volumes:
       data:
         # storageClass: "-"
@@ -209,11 +172,20 @@ mysql:
     nodeSelector: {}
     tolerations: []
     affinity: {}
+  external:
+    host: "192.168.0.1"
+    port: "5432"
+    username: "user"
+    password: "password"
+    coreDatabase: "registry"
+    clairDatabase: "clair"
+    notaryServerDatabase: "notary_server"
+    notarySignerDatabase: "notary_signer"
 
 registry:
   image:
     repository: vmware/registry-photon
-    tag: v2.6.2-v1.5.0-chart-patch
+    tag: dev
     pullPolicy: IfNotPresent
   httpSecret: not-a-secure-secret
   logLevel: info
@@ -284,22 +256,8 @@ clair:
   enabled: true
   image:
     repository: vmware/clair-photon
-    tag: v2.0.1-v1.5.0-chart-patch
+    tag: dev
     pullPolicy: IfNotPresent
-## The following needs to match the credentials
-## in the `postgresql` configuration under the
-## `postgresql` namespace below.
-  postgresPassword: not-a-secure-password
-  postgresUser: clair
-  postgresDatabase: clair
-# resources:
-#  requests:
-#    memory: 256Mi
-#    cpu: 100m
-# pgResources:
-#  requests:
-#    memory: 256Mi
-#    cpu: 100m
   volumes:
     pgData:
       # storageClass: "-"
@@ -313,16 +271,6 @@ clair:
   tolerations: []
   affinity: {}
 
-## Settings for postgresql dependency.
-## see https://github.com/kubernetes/charts/tree/master/stable/postgresql
-## for further configurables.
-postgresql:
-  postgresUser: clair
-  postgresPassword: not-a-secure-password
-  postgresDatabase: clair
-  persistence:
-    enabled: false
-
 ## Settings for redis dependency.
 ## see https://github.com/kubernetes/charts/tree/master/stable/redis
 ## for further configurables.
@@ -342,12 +290,12 @@ notary:
   server:
     image:
       repository: vmware/notary-server-photon
-      tag: v0.5.1-v1.5.0-chart-patch
+      tag: dev
       pullPolicy: IfNotPresent
   signer:
     image:
       repository: vmware/notary-signer-photon
-      tag: v0.5.1-v1.5.0-chart-patch
+      tag: dev
       pullPolicy: IfNotPresent
     env:
       NOTARY_SIGNER_DEFAULTALIAS: defaultalias
@@ -355,21 +303,6 @@ notary:
     caCrt:
     tlsCrt:
     tlsKey:
-  db:
-    image:
-      repository: vmware/mariadb-photon
-      tag: *harbor_image_tag
-      pullPolicy: IfNotPresent
-    password: not-a-secure-password
-    volumes:
-      data:
-        # storageClass: "-"
-        accessMode: ReadWriteOnce
-        size: 1Gi
-    # resources:
-    #  requests:
-    #    memory: 256Mi
-    #    cpu: 100m
   nodeSelector: {}
   tolerations: []
   affinity: {}

docs/compile_guide.md

@@ -50,19 +50,19 @@ You can compile the code by one of the three approaches:
 *  Build, install and bring up Harbor without Notary:
 
    ```sh
-      $ make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.1
+      $ make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.0
    ```
 
 *  Build, install and bring up Harbor with Notary:
 
    ```sh
-      $ make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.1 NOTARYFLAG=true
+      $ make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.0 NOTARYFLAG=true
    ```
 
 *  Build, install and bring up Harbor with Clair:
 
    ```sh
-      $ make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.1 CLAIRFLAG=true
+      $ make install GOBUILDIMAGE=golang:1.9.2 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.4.0 CLAIRFLAG=true
    ```
 
 #### II. Compile code with your own Golang environment, then build Harbor

docs/demo_server.md

@@ -0,0 +1,20 @@
+# Demo Server Guide
+
+**Important!** 
+ - Please note that this demo server is **ONLY** for your trial of Harbor functionalities. 
+ - Please **DO NOT** upload any sensitive images to this server. 
+ - We will **CLEAN AND RESET** the server every **TWO Days**.
+ - You can only experience the non-admin functionalities on this server. Please follow the **[Installation Guide](installation_guide.md)** to set up a Harbor server to try more advanced features.
+ - Please do not push large images(>100MB) as the server has limited storage.
+
+If you encounter any problems during using the demo server, please contact us at <img alt="email" src="img/harbor_email.png" valigin="middle" height="18"/>.
+
+**Usage**
+
+ - 1> The address of the demo server is [https://demo.goharbor.io](https://demo.goharbor.io)
+ - 2> You can register a new user by yourself
+ - 3> Then you can use the account/password created in step 2 to log in 
+ ```
+ docker login demo.goharbor.io
+ ```
+You can refer to [User Guide](user_guide.md) for more details on how to use Harbor.

docs/installation_guide.md

@@ -34,7 +34,7 @@ Harbor is deployed as several Docker containers, and, therefore, can be deployed
 |Port|Protocol|Description|
 |---|---|---|
 |443|HTTPS|Harbor UI and API will accept requests on this port for https protocol|
-|4443|HTTS|Connections to the Docker Content Trust service for Harbor, only needed when Notary is enabled|
+|4443|HTTPS|Connections to the Docker Content Trust service for Harbor, only needed when Notary is enabled|
 |80|HTTP|Harbor UI and API will accept requests on this port for http protocol|
 
 ## Installation Steps

docs/migration_guide.md

@@ -39,7 +39,7 @@ When upgrading your existing Habor instance to a newer version, you may need to
 
 5. Back up database/harbor.cfg to a directory such as `/path/to/backup`. You need to create the directory if it does not exist.  Also, note that the username and password to access the db are provided via environment variable "DB_USR" and "DB_PWD". 
 
-    **NOTE:** Upgrade from harbor 1.2 or older to harbor 1.3 must use `vmware/migratorharbor-db-migrator:1.2`. Because DB engine replaced by MariaDB in harbor 1.3
+    **NOTE:** Upgrade from harbor 1.2 or older to harbor 1.3 must use `vmware/harbor-db-migrator:1.2`. Because DB engine replaced by MariaDB in harbor 1.3
 
     ```
     docker run -it --rm -e DB_USR=root -e DB_PWD={db_pwd} -v ${harbor_db_path}:/var/lib/mysql -v ${harbor_cfg}:/harbor-migration/harbor-cfg/harbor.cfg -v ${backup_path}:/harbor-migration/backup vmware/harbor-migrator:[tag] backup

docs/swagger.yaml

@@ -2325,14 +2325,18 @@ paths:
       summary: Search available ldap groups.
       description: >
         This endpoint searches the available ldap groups based on related
-        configuration parameters. Support searched by input ladp configuration,
+        configuration parameters. support to search by groupname or groupdn.
-        load configuration from the system and specific filter.
       parameters:
         - name: groupname
           in: query
           type: string
           required: false
           description: Ldap group name
+        - name: groupdn
+          in: query
+          type: string
+          required: false
+          description: The LDAP group DN
       tags:
         - Products
       responses:
@@ -2342,6 +2346,10 @@ paths:
             type: array
             items:
               $ref: '#/definitions/UserGroup'
+        "400":
+          description: The Ldap group DN is invalid.
+        '404':
+          description: No ldap group found.
         '500':
           description: Unexpected internal errors.
   /ldap/users/search:
@@ -2372,6 +2380,7 @@ paths:
           description: Only admin has this authority.
         '500':
           description: Unexpected internal errors.
+
   /ldap/users/import:
     post:
       summary: Import selected available ldap users.
@@ -2720,25 +2729,24 @@ definitions:
     type: object
     properties:
       public:
-        type: integer
+        type: string
-        format: int
+        description: The public status of the project. The valid values are "true", "false".
-        description: The public status of the project.
       enable_content_trust:
-        type: boolean
+        type: string
         description: >-
           Whether content trust is enabled or not. If it is enabled, user cann't
-          pull unsigned images from this project.
+          pull unsigned images from this project. The valid values are "true", "false".
       prevent_vulnerable_images_from_running:
-        type: boolean
+        type: string
-        description: Whether prevent the vulnerable images from running.
+        description: Whether prevent the vulnerable images from running. The valid values are "true", "false".
       prevent_vulnerable_images_from_running_severity:
         type: string
         description: >-
           If the vulnerability is high than severity defined here, the images
-          cann't be pulled.
+          cann't be pulled. The valid values are "negligible", "low", "medium", "high", "critical".
       automatically_scan_images_on_push:
-        type: boolean
+        type: string
-        description: Whether scan images automatically when pushing.
+        description: Whether scan images automatically when pushing. The valid values are "true", "false".
   Manifest:
     type: object
     properties:
@@ -2989,9 +2997,7 @@ definitions:
           The replication policy filter kind. The valid values are project,
           repository and tag.
       value:
-        type: 
+        type: string
-          - string
-          - integer
         description: The value of replication policy filter. When creating repository and tag filter, filling it with the pattern as string. When creating label filter, filling it with label ID as integer.
       pattern:
         type: string
@@ -3497,6 +3503,9 @@ definitions:
       ldap_group_search_scope:
         type: integer
         description: The scope to search ldap. '0-LDAP_SCOPE_BASE, 1-LDAP_SCOPE_ONELEVEL, 2-LDAP_SCOPE_SUBTREE'
+      ldap_group_admin_dn:
+        type: string
+        description: Specify the ldap group which have the same privilege with Harbor admin.
       project_creation_restriction:
         type: string
         description: >-
@@ -3653,6 +3662,7 @@ definitions:
       ldap_group_dn:
         type: string
         description: The DN of the LDAP group if group type is 1 (LDAP group).
+
   Resource:
     type: object
     properties:

docs/user_guide.md

@@ -126,11 +126,12 @@ There may be a bit of delay during replication according to the situation of the
 Replication can be configured by creating a rule. Click `NEW REPLICATION RULE` under `Administration->Replications` and fill in the necessary fields. You can choose different image filters and trigger modes according to the different requirements. If there is no endpoint available in the list, you need to create one. Click `SAVE` to create a replication rule for the selected project. If `Replicate existing images immediately` is chosen, the existing images under the project will be replicated to the remote registry immediately.  
 
 #### Image filter
-Two image filters are supported:
+Three image filters are supported:
 * **Repository**: Filter images according to the repository part of image name.
 * **Tag**: Filter images according to the tag part of image name.
+* **Label**: Filter images according to the [labels](#managing-labels). **Notes**: If the labels referenced by a rule are deleted, the rule's status will be set to `Disabled`. You need to edit and update it according to the tips.
 
-Two terms are supported in filter pattern:
+Two terms are supported in the pattern used by repository filter and tag filter:
 * **\***: Matches any sequence of non-separator characters `/`.
 * **?**: Matches any single non-separator character `/`.
 

make/common/templates/adminserver/env

@@ -62,3 +62,5 @@ READ_ONLY=false
 SKIP_RELOAD_ENV_PATTERN=$skip_reload_env_pattern
 RELOAD_KEY=$reload_key
 CHART_SERVICE_URL=$chart_service_url
+LDAP_GROUP_ADMIN_DN=$ldap_group_admin_dn
+REGISTRY_CONTROLLER_URL=$registry_controller_url

make/common/templates/nginx/nginx.http.conf

@@ -12,11 +12,6 @@ http {
   # this is necessary for us to be able to disable request buffering in all cases
   proxy_http_version 1.1;
 
-
-  upstream registry {
-    server registry:5000;
-  }
-
   upstream ui {
     server ui:8080;
   }

make/common/templates/nginx/nginx.https.conf

@@ -13,10 +13,6 @@ http {
   # this is necessary for us to be able to disable request buffering in all cases
   proxy_http_version 1.1;
 
-  upstream registry {
-    server registry:5000;
-  }
-
   upstream ui {
     server ui:8080;
   }

make/common/templates/registry/config.yml

@@ -5,13 +5,16 @@ log:
     service: registry
 storage:
   cache:
-    layerinfo: inmemory
+    layerinfo: redis
   $storage_provider_info
   maintenance:
     uploadpurging:
       enabled: false
   delete:
     enabled: true
+redis:
+  addr: $redis_url
+  db: 1
 http:
   addr: :5000
   secret: placeholder

make/common/templates/registry/config_ha.yml

@@ -14,15 +14,7 @@ storage:
     enabled: true
 redis:
   addr: $redis_url
-  db: 0
+  db: 1
-  dialtimeout: 10ms
-  readtimeout: 10ms
-  writetimeout: 10ms
-  pool:
-    maxidle: 16
-    maxactive: 64
-    idletimeout: 300s
-
 http:
   addr: :5000
   secret: placeholder

make/common/templates/registryctl/config.yml

@@ -0,0 +1,8 @@
+---
+protocol: "http"
+port: 8080
+log_level: "INFO"
+
+#https_config:
+#  cert: "server.crt"
+#  key: "server.key"

make/common/templates/registryctl/env

@@ -0,0 +1,3 @@
+UI_SECRET=$ui_secret
+JOBSERVICE_SECRET=$jobservice_secret
+

make/common/templates/ui/env

@@ -6,3 +6,4 @@ GODEBUG=netdns=cgo
 ADMINSERVER_URL=$adminserver_url
 UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem
 _REDIS_URL=$redis_url
+SYNC_REGISTRY=false

make/dev/nodeclarity/entrypoint.sh

@@ -34,7 +34,7 @@ sed -i 's/* as//g' src/app/shared/gauge/gauge.component.js
 cp ./dist/build.min.js ../ui/static/
 
 cp -r ./src/i18n/ ../ui/static/
-cp ./src/styles.scss ../ui/static/
+cp ./src/styles.css ../ui/static/
 cp -r ./src/images/ ../ui/static/
 cp ./src/setting.json ../ui/static/
 

make/dev/registryctl/Dockerfile

@@ -0,0 +1,12 @@
+FROM golang:1.9.2
+
+MAINTAINER wangyan@vmware.com
+
+COPY . /go/src/github.com/vmware/harbor
+
+WORKDIR /go/src/github.com/vmware/harbor/src/registryctl
+
+RUN go build -a -o /go/bin/harbor_registryctl \
+    && chmod u+x /go/bin/harbor_registryctl 
+WORKDIR /go/bin/
+ENTRYPOINT ["/go/bin/harbor_registryctl"]

make/docker-compose.clair.tpl

@@ -22,7 +22,7 @@ services:
     container_name: clair
     image: vmware/clair-photon:__clair_version__
     restart: always
-    cpu_quota: 150000
+    cpu_quota: 50000
     depends_on:
       - postgresql
     volumes:

make/docker-compose.tpl

@@ -22,8 +22,6 @@ services:
       - harbor
     environment:
       - GODEBUG=netdns=cgo
-    command:
-      ["serve", "/etc/registry/config.yml"]
     depends_on:
       - log
     logging:
@@ -31,6 +29,27 @@ services:
       options:  
         syslog-address: "tcp://127.0.0.1:1514"
         tag: "registry"
+  registryctl:
+    image: vmware/harbor-registryctl:__version__
+    container_name: registryctl
+    env_file:
+      - ./common/config/registryctl/env
+    restart: always
+    volumes:
+      - /data/registry:/storage:z
+      - ./common/config/registry/:/etc/registry/:z
+      - ./common/config/registryctl/config.yml:/etc/registryctl/config.yml:z
+    networks:
+      - harbor
+    environment:
+      - GODEBUG=netdns=cgo
+    depends_on:
+      - log
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "registryctl"
   postgresql:
     image: vmware/harbor-db:__version__
     container_name: harbor-db
@@ -116,7 +135,7 @@ services:
     container_name: redis
     restart: always
     volumes:
-      - /data/redis:/data
+      - /data/redis:/var/lib/redis
     networks:
       - harbor
     depends_on:

make/migrations/postgresql/0001_initial_schema.up.sql

@@ -341,5 +341,5 @@ CREATE TABLE IF NOT EXISTS alembic_version (
     version_num varchar(32) NOT NULL
 );
 
-insert into alembic_version values ('1.5.0');
+insert into alembic_version values ('1.6.0');
 

make/photon/Makefile

@@ -71,6 +71,10 @@ DOCKERFILEPATH_REG=$(DOCKERFILEPATH)/registry
 DOCKERFILENAME_REG=Dockerfile
 DOCKERIMAGENAME_REG=vmware/registry-photon
 
+DOCKERFILEPATH_REGISTRYCTL=$(DOCKERFILEPATH)/registryctl
+DOCKERFILENAME_REGISTRYCTL=Dockerfile
+DOCKERIMAGENAME_REGISTRYCTL=vmware/harbor-registryctl
+
 DOCKERFILEPATH_NOTARY=$(DOCKERFILEPATH)/notary
 DOCKERFILENAME_NOTARYSIGNER=signer.Dockerfile
 DOCKERIMAGENAME_NOTARYSIGNER=vmware/notary-signer-photon
@@ -177,6 +181,11 @@ _build_registry:
 	fi
 	@echo "building registry container for photon..."
 	@cd $(DOCKERFILEPATH_REG) && chmod 655 $(DOCKERFILEPATH_REG)/binary/registry && $(DOCKERBUILD) -f $(DOCKERFILEPATH_REG)/$(DOCKERFILENAME_REG) -t $(DOCKERIMAGENAME_REG):$(REGISTRYVERSION)-$(VERSIONTAG) .
+	@echo "Done."
+
+_build_registryctl:		
+	@echo "building registry controller for photon..."
+	$(DOCKERBUILD) -f $(DOCKERFILEPATH_REGISTRYCTL)/$(DOCKERFILENAME_REGISTRYCTL) -t $(DOCKERIMAGENAME_REGISTRYCTL):$(VERSIONTAG) .
 	@rm -rf $(DOCKERFILEPATH_REG)/binary
 	@echo "Done."
 
@@ -187,14 +196,14 @@ _build_redis:
 
 _build_migrator:
 	@echo "building db migrator container for photon..."
-	@cd $(DOCKERFILEPATH_MIGRATOR) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_MIGRATOR)/$(DOCKERFILENAME_MIGRATOR) -t $(DOCKERIMAGENAME_MIGRATOR):$(VERSIONTAG) .
+	@cd $(DOCKERFILEPATH_MIGRATOR) && $(DOCKERBUILD) -f $(DOCKERFILEPATH_MIGRATOR)/$(DOCKERFILENAME_MIGRATOR) -t $(DOCKERIMAGENAME_MIGRATOR):$(MIGRATORVERSION) .
 	@echo "Done."
 
 define _get_binary
 	$(WGET) --timeout 30 --no-check-certificate $1 -O $2
 endef
 
-build: _build_db _build_adminiserver _build_ui _build_jobservice _build_log _build_nginx _build_registry _build_notary _build_clair _build_redis _build_migrator _build_chart_server
+build: _build_db _build_adminiserver _build_ui _build_jobservice _build_log _build_nginx _build_registry _build_registryctl _build_notary _build_clair _build_redis _build_migrator _build_chart_server
 
 cleanimage:
 	@echo "cleaning image for photon..."

make/photon/adminserver/Dockerfile

@@ -7,7 +7,7 @@ RUN tdnf erase vim -y \
     && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
     && mkdir /harbor/
 COPY ./make/dev/adminserver/harbor_adminserver ./make/photon/adminserver/start.sh /harbor/
-#There is a race condition that both ui and adminserver may initialize schema
+#As UI will be blocked until adminserver is ready, let adminserver do the initialise work for DB
 COPY ./make/migrations /harbor/migrations
 
 HEALTHCHECK CMD curl --fail -s http://127.0.0.1:8080/api/ping || exit 1 

make/photon/clair/Dockerfile

@@ -5,10 +5,10 @@ RUN tdnf distro-sync -y \
     && tdnf erase vim -y \
     && tdnf install -y git shadow sudo bzr rpm xz python-xml >>/dev/null\
     && tdnf clean all \
-    && mkdir /clair2.0.1/ \
+    && mkdir /clair/ \
     && groupadd -r -g 10000 clair \
     && useradd --no-log-init -m -r -g 10000 -u 10000 clair
-COPY ./binary/clair /clair2.0.1/
+COPY ./binary/clair /clair/
 COPY docker-entrypoint.sh /docker-entrypoint.sh
 COPY dumb-init /dumb-init 
 
@@ -16,8 +16,8 @@ VOLUME /config
 
 EXPOSE 6060 6061
 
-RUN chown -R 10000:10000 /clair2.0.1 \
+RUN chown -R 10000:10000 /clair \
-    && chmod u+x /clair2.0.1/clair \
+    && chmod u+x /clair/clair \
     && chmod u+x /docker-entrypoint.sh \
     && chmod +x /dumb-init
 

make/photon/clair/Dockerfile.binary

@@ -1,4 +1,4 @@
-FROM golang:1.7.3
+FROM golang:1.9.2
 
 ADD .   /go/src/github.com/coreos/clair/
 WORKDIR /go/src/github.com/coreos/clair/

make/photon/clair/docker-entrypoint.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -e
-sudo -E -H -u \#10000 sh -c "/dumb-init -- /clair2.0.1/clair -config /etc/clair/config.yaml"
+sudo -E -H -u \#10000 sh -c "/dumb-init -- /clair/clair -config /etc/clair/config.yaml"
 set +e

make/photon/redis/Dockerfile

@@ -1,12 +1,10 @@
 FROM vmware/photon:1.0
 
 RUN tdnf distro-sync -y \
-    && tdnf install -y redis sudo \
+    && tdnf install -y redis sudo 
-    && mkdir /data \
-    && chown redis:redis /data
 
-VOLUME /data
+VOLUME /var/lib/redis
-WORKDIR /data
+WORKDIR /var/lib/redis
 COPY docker-entrypoint.sh /usr/bin/
 COPY redis.conf /etc/redis.conf
 RUN chmod +x /usr/bin/docker-entrypoint.sh \

make/photon/registryctl/Dockerfile

@@ -0,0 +1,25 @@
+FROM vmware/photon:1.0
+
+MAINTAINER wangyan@vmware.com
+
+RUN tdnf distro-sync -y || echo \
+    && tdnf erase vim -y \
+    && tdnf install sudo -y >> /dev/null\
+    && tdnf clean all \
+    && groupadd -r -g 10000 harbor && useradd --no-log-init -r -g 10000 -u 10000 harbor \
+    && mkdir -p /etc/registry
+
+COPY ./make/photon/registry/binary/registry /usr/bin
+COPY ./make/photon/registryctl/start.sh /harbor/
+COPY ./make/dev/registryctl/harbor_registryctl /harbor/
+
+RUN chmod u+x /harbor/harbor_registryctl \
+    && chmod u+x /usr/bin/registry \
+    && chmod u+x /harbor/start.sh
+
+HEALTHCHECK CMD curl --fail -s http://127.0.0.1:8080/api/health || exit 1 
+
+VOLUME ["/var/lib/registry"]
+WORKDIR /harbor/
+
+ENTRYPOINT ["/harbor/start.sh"]

make/photon/registryctl/start.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+set -e
+
+# The directory /var/lib/registry is within the container, and used to store image in CI testing.
+# So for now we need to chown to it to avoid failure in CI.
+if [ -d /var/lib/registry ]; then
+    chown 10000:10000 -R /var/lib/registry
+fi
+
+if [ -d /storage ]; then
+    if ! stat -c '%u:%g' /storage | grep -q '10000:10000' ; then
+        # 10000 is the id of harbor user/group.
+        # Usually NFS Server does not allow changing owner of the export directory,
+        # so need to skip this step and requires NFS Server admin to set its owner to 10000.
+        chown 10000:10000 -R /storage
+    fi
+fi
+
+sudo -E -u \#10000 "/harbor/harbor_registryctl" "-c" "/etc/registryctl/config.yml"

make/photon/ui/Dockerfile

@@ -11,8 +11,6 @@ HEALTHCHECK CMD curl --fail -s http://127.0.0.1:8080/api/ping || exit 1
 COPY ./make/dev/ui/harbor_ui ./src/favicon.ico ./make/photon/ui/start.sh ./UIVERSION /harbor/
 COPY ./src/ui/views /harbor/views
 COPY ./src/ui/static /harbor/static
-#There is a race condition that both ui and adminserver may initialize schema
-COPY ./make/migrations /harbor/migrations
 
 RUN chmod u+x /harbor/start.sh /harbor/harbor_ui
 WORKDIR /harbor/

make/prepare

@@ -306,6 +306,7 @@ ui_certificates_dir =  prep_conf_dir(ui_config_dir,"certificates")
 db_config_dir = prep_conf_dir(config_dir, "db")
 job_config_dir = prep_conf_dir(config_dir, "jobservice")
 registry_config_dir = prep_conf_dir(config_dir, "registry")
+registryctl_config_dir = prep_conf_dir(config_dir, "registryctl")
 nginx_config_dir = prep_conf_dir (config_dir, "nginx")
 nginx_conf_d = prep_conf_dir(nginx_config_dir, "conf.d")
 log_config_dir = prep_conf_dir (config_dir, "log")
@@ -316,6 +317,8 @@ ui_conf = os.path.join(config_dir, "ui", "app.conf")
 ui_cert_dir = os.path.join(config_dir, "ui", "certificates")
 jobservice_conf = os.path.join(config_dir, "jobservice", "config.yml")
 registry_conf = os.path.join(config_dir, "registry", "config.yml")
+registryctl_conf_env = os.path.join(config_dir, "registryctl", "env")
+registryctl_conf_yml = os.path.join(config_dir, "registryctl", "config.yml")
 db_conf_env = os.path.join(config_dir, "db", "env")
 job_conf_env = os.path.join(config_dir, "jobservice", "env")
 nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf")
@@ -323,6 +326,7 @@ cert_dir = os.path.join(config_dir, "nginx", "cert")
 log_rotate_config = os.path.join(config_dir, "log", "logrotate.conf") 
 adminserver_url = "http://adminserver:8080"
 registry_url = "http://registry:5000"
+registry_controller_url = "http://registryctl:8080"
 ui_url = "http://ui:8080"
 token_service_url = "http://ui:8080/service/token"
 
@@ -356,6 +360,8 @@ else:
 #Use reload_key to avoid reload config after restart harbor
 reload_key = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6)) if reload_config == "true" else ""
 
+ldap_group_admin_dn = rcp.get("configuration", "ldap_group_admin_dn") if rcp.has_option("configuration", "ldap_group_admin_dn") else ""
+
 render(os.path.join(templates_dir, "adminserver", "env"),
         adminserver_conf_env,
         reload_config=reload_config,
@@ -376,6 +382,7 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         ldap_group_filter=ldap_group_filter,
         ldap_group_gid=ldap_group_gid,
         ldap_group_scope=ldap_group_scope,
+        ldap_group_admin_dn=ldap_group_admin_dn,
         db_password=db_password,
         db_host=db_host,
         db_user=db_user,
@@ -415,6 +422,7 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         reload_key=reload_key,
         skip_reload_env_pattern=skip_reload_env_pattern,
         chart_service_url=chart_service_url
+        registry_controller_url = registry_controller_url
 	)
 
 render(os.path.join(templates_dir, "ui", "env"), 
@@ -425,7 +433,8 @@ render(os.path.join(templates_dir, "ui", "env"),
         adminserver_url = adminserver_url
         )
 
-registry_config_file = "config_ha.yml" if args.ha_mode else "config.yml"
+registry_config_file_ha = "config_ha.yml"
+registry_config_file = "config.yml"
 if storage_provider_name == "filesystem":
     if not storage_provider_config:
         storage_provider_config = "rootdirectory: /storage"
@@ -436,6 +445,13 @@ storage_provider_conf_list = [storage_provider_name + ':']
 for c in storage_provider_config.split(","):
     storage_provider_conf_list.append(c.strip())
 storage_provider_info = ('\n' + ' ' * 4).join(storage_provider_conf_list)
+render(os.path.join(templates_dir, "registry", registry_config_file_ha),
+    registry_conf,
+    storage_provider_info=storage_provider_info,
+    public_url=public_url,
+    ui_url=ui_url,
+    redis_url=redis_url)
+
 render(os.path.join(templates_dir, "registry", registry_config_file),
     registry_conf,
     storage_provider_info=storage_provider_info,
@@ -463,7 +479,13 @@ render(os.path.join(templates_dir, "log", "logrotate.conf"),
         log_rotate_count=log_rotate_count,
 		log_rotate_size=log_rotate_size)
 
+render(os.path.join(templates_dir, "registryctl", "env"),
+        registryctl_conf_env,
+        jobservice_secret=jobservice_secret,
+        ui_secret=ui_secret)
+
 shutil.copyfile(os.path.join(templates_dir, "ui", "app.conf"), ui_conf)
+shutil.copyfile(os.path.join(templates_dir, "registryctl", "config.yml"), registryctl_conf_yml)
 print("Generated configuration file: %s" % ui_conf)
 
 if auth_mode == "uaa_auth":
@@ -525,7 +547,7 @@ def openssl_installed():
 
 if customize_crt == 'on' and openssl_installed():
     shell_stat = subprocess.check_call(["which", "openssl"], stdout=FNULL, stderr=subprocess.STDOUT)
-    empty_subj = "/C=/ST=/L=/O=/CN=/"
+    empty_subj = "/"
     private_key_pem = os.path.join(config_dir, "ui", "private_key.pem")
     root_crt = os.path.join(config_dir, "registry", "root.crt")
     create_root_cert(empty_subj, key_path=private_key_pem, cert_path=root_crt)

partners.md

@@ -15,6 +15,7 @@ Below are some of the partners and users of Harbor projects. If you wish to be l
 <a href="http://www.hydsoft.com/" border="0" target="_blank"><img alt="HYDSoft" src="docs/img/hydsoft.png" height="50"></a>&nbsp; &nbsp; &nbsp; &nbsp;
 <a href="http://www.cloud-star.com.cn/" border="0" target="_blank"><img alt="CloudStar" src="docs/img/cloudstar.png" height="50"></a>&nbsp; &nbsp; &nbsp; &nbsp;
 <a href="http://www.beyondsoft.com/" border="0" target="_blank"><img alt="BeyondSoft" src="docs/img/beyondsoft.png" height="50"></a>&nbsp; &nbsp; &nbsp; &nbsp;
+<a href="http://www.chinamobileltd.com/" border="0" target="_blank"><img alt="ChinaMobile" src="docs/img/china-mobile.png" height="50"></a>&nbsp; &nbsp; &nbsp; &nbsp;
 
 ## Partners
 <a href="https://www.caicloud.io" target="_blank" border="0"><img alt="CaiCloud" src="docs/img/caicloud.png" height="50"></a>&nbsp; &nbsp; &nbsp; &nbsp;

src/adminserver/api/cfg.go

@@ -31,7 +31,7 @@ func ListCfgs(w http.ResponseWriter, r *http.Request) {
 		handleInternalServerError(w)
 		return
 	}
-
+	systemcfg.AddMissedKey(cfg)
 	if err = writeJSON(w, cfg); err != nil {
 		log.Errorf("failed to write response: %v", err)
 		return
@@ -52,7 +52,6 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) {
 		handleBadRequestError(w, err.Error())
 		return
 	}
-
 	if err = systemcfg.CfgStore.Write(m); err != nil {
 		log.Errorf("failed to update system configurations: %v", err)
 		handleInternalServerError(w)
@@ -68,7 +67,6 @@ func ResetCfgs(w http.ResponseWriter, r *http.Request) {
 		handleInternalServerError(w)
 		return
 	}
-
 	if err := systemcfg.CfgStore.Write(cfgs); err != nil {
 		log.Errorf("failed to write system configurations to storage: %v", err)
 		handleInternalServerError(w)

src/adminserver/systemcfg/systemcfg.go

@@ -30,6 +30,7 @@ import (
 	comcfg "github.com/vmware/harbor/src/common/config"
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils"
 	"github.com/vmware/harbor/src/common/utils/log"
 )
 
@@ -164,6 +165,7 @@ var (
 			parse: parseStringToBool,
 		},
 		common.ReloadKey:        "RELOAD_KEY",
+		common.LdapGroupAdminDn: "LDAP_GROUP_ADMIN_DN",
 	}
 
 	// configurations need read from environment variables
@@ -244,34 +246,43 @@ func parseStringToBool(str string) (interface{}, error) {
 // Init system configurations. If env RESET is set or configurations
 // read from storage driver is null, load all configurations from env
 func Init() (err error) {
-	if err = initCfgStore(); err != nil {
+	//init database
+	envCfgs := map[string]interface{}{}
+	if err := LoadFromEnv(envCfgs, true); err != nil {
 		return err
 	}
-	cfgs := map[string]interface{}{}
+	db := GetDatabaseFromCfg(envCfgs)
+	//Initialize the schema, then register the DB.
+	if err := dao.UpgradeSchema(db); err != nil {
+		return err
+	}
+	if err := dao.InitDatabase(db); err != nil {
+		return err
+	}
+	if err := initCfgStore(); err != nil {
+		return err
+	}
+
 	//Use reload key to avoid reset customed setting after restart
 	curCfgs, err := CfgStore.Read()
 	if err != nil {
 		return err
 	}
-	loadAll := isLoadAll(curCfgs[common.ReloadKey])
+	loadAll := isLoadAll(curCfgs)
-	if !loadAll {
+	if curCfgs == nil {
-		cfgs = curCfgs
+		curCfgs = map[string]interface{}{}
-		if cfgs == nil {
-			log.Info("configurations read from storage driver are null, will load them from environment variables")
-			loadAll = true
-			cfgs = map[string]interface{}{}
 	}
-	}
+	//restart: only repeatload envs will be load
-
+	//reload_config: all envs will be reload except the skiped envs
-	if err = LoadFromEnv(cfgs, loadAll); err != nil {
+	if err = LoadFromEnv(curCfgs, loadAll); err != nil {
 		return err
 	}
-
+	AddMissedKey(curCfgs)
-	return CfgStore.Write(cfgs)
+	return CfgStore.Write(curCfgs)
 }
 
-func isLoadAll(curReloadKey interface{}) bool {
+func isLoadAll(cfg map[string]interface{}) bool {
-	return strings.EqualFold(os.Getenv("RESET"), "true") && os.Getenv("RELOAD_KEY") != curReloadKey
+	return cfg == nil || strings.EqualFold(os.Getenv("RESET"), "true") && os.Getenv("RELOAD_KEY") != cfg[common.ReloadKey]
 }
 
 func initCfgStore() (err error) {
@@ -287,16 +298,6 @@ func initCfgStore() (err error) {
 	log.Infof("the path of json configuration storage: %s", path)
 
 	if drivertype == common.CfgDriverDB {
-		//init database
-		cfgs := map[string]interface{}{}
-		if err = LoadFromEnv(cfgs, true); err != nil {
-			return err
-		}
-		cfgdb := GetDatabaseFromCfg(cfgs)
-		//Initialize the schema.
-		if err = dao.InitDatabase(cfgdb, true); err != nil {
-			return err
-		}
 		CfgStore, err = database.NewCfgStore()
 		if err != nil {
 			return err
@@ -322,7 +323,6 @@ func initCfgStore() (err error) {
 				// only used when migrating harbor release before v1.3
 				// after v1.3 there is always a db configuration before migrate.
 				validLdapScope(jsonconfig, true)
-
 				err = CfgStore.Write(jsonconfig)
 				if err != nil {
 					log.Error("Failed to update old configuration to database")
@@ -412,11 +412,11 @@ func GetDatabaseFromCfg(cfg map[string]interface{}) *models.Database {
 	database := &models.Database{}
 	database.Type = cfg[common.DatabaseType].(string)
 	postgresql := &models.PostGreSQL{}
-	postgresql.Host = cfg[common.PostGreSQLHOST].(string)
+	postgresql.Host = utils.SafeCastString(cfg[common.PostGreSQLHOST])
-	postgresql.Port = int(cfg[common.PostGreSQLPort].(int))
+	postgresql.Port = int(utils.SafeCastInt(cfg[common.PostGreSQLPort]))
-	postgresql.Username = cfg[common.PostGreSQLUsername].(string)
+	postgresql.Username = utils.SafeCastString(cfg[common.PostGreSQLUsername])
-	postgresql.Password = cfg[common.PostGreSQLPassword].(string)
+	postgresql.Password = utils.SafeCastString(cfg[common.PostGreSQLPassword])
-	postgresql.Database = cfg[common.PostGreSQLDatabase].(string)
+	postgresql.Database = utils.SafeCastString(cfg[common.PostGreSQLDatabase])
 	database.PostGreSQL = postgresql
 	return database
 }
@@ -442,3 +442,26 @@ func validLdapScope(cfg map[string]interface{}, isMigrate bool) {
 	cfg[ldapScopeKey] = ldapScope
 
 }
+
+//AddMissedKey ... If the configure key is missing in the cfg map, add default value to it
+func AddMissedKey(cfg map[string]interface{}) {
+
+	for k, v := range common.HarborStringKeysMap {
+		if _, exist := cfg[k]; !exist {
+			cfg[k] = v
+		}
+	}
+
+	for k, v := range common.HarborNumKeysMap {
+		if _, exist := cfg[k]; !exist {
+			cfg[k] = v
+		}
+	}
+
+	for k, v := range common.HarborBoolKeysMap {
+		if _, exist := cfg[k]; !exist {
+			cfg[k] = v
+		}
+	}
+
+}

src/adminserver/systemcfg/systemcfg_test.go

@@ -135,8 +135,10 @@ func TestIsLoadAll(t *testing.T) {
 	if err := os.Setenv("RESET", "True"); err != nil {
 		t.Fatalf("failed to set env: %v", err)
 	}
-	assert.False(t, isLoadAll("123456"))
+	cfg1 := map[string]interface{}{common.ReloadKey: "123456"}
-	assert.True(t, isLoadAll("654321"))
+	cfg2 := map[string]interface{}{common.ReloadKey: "654321"}
+	assert.False(t, isLoadAll(cfg1))
+	assert.True(t, isLoadAll(cfg2))
 }
 
 func TestLoadFromEnvWithReloadConfigInvalidSkipPattern(t *testing.T) {
@@ -258,3 +260,31 @@ func TestValidLdapScope(t *testing.T) {
 	}
 
 }
+func Test_AddMissingKey(t *testing.T) {
+
+	cfg := map[string]interface{}{
+		common.LDAPURL:        "sampleurl",
+		common.EmailPort:      555,
+		common.LDAPVerifyCert: true,
+	}
+
+	type args struct {
+		cfg map[string]interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{"Add default value", args{cfg}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			AddMissedKey(tt.args.cfg)
+		})
+	}
+
+	if _, ok := cfg[common.LDAPBaseDN]; !ok {
+		t.Errorf("Can not found default value for %v", common.LDAPBaseDN)
+	}
+
+}

src/common/const.go

@@ -108,4 +108,90 @@ const (
 	DefaultNotaryEndpoint             = "http://notary-server:4443"
 	LdapGroupType                     = 1
 	ReloadKey                         = "reload_key"
+	LdapGroupAdminDn                  = "ldap_group_admin_dn"
+	DefaultRegistryControllerEndpoint = "http://registryctl:8080"
+)
+
+// Shared variable, not allowed to modify
+var (
+	// the keys of configurations which user can modify in PUT method and user can
+	// get in GET method
+	HarborValidKeys = []string{
+		AUTHMode,
+		SelfRegistration,
+		LDAPURL,
+		LDAPSearchDN,
+		LDAPSearchPwd,
+		LDAPBaseDN,
+		LDAPUID,
+		LDAPFilter,
+		LDAPScope,
+		LDAPTimeout,
+		LDAPVerifyCert,
+		LDAPGroupAttributeName,
+		LDAPGroupBaseDN,
+		LDAPGroupSearchFilter,
+		LDAPGroupSearchScope,
+		EmailHost,
+		EmailPort,
+		EmailUsername,
+		EmailPassword,
+		EmailFrom,
+		EmailSSL,
+		EmailIdentity,
+		EmailInsecure,
+		ProjectCreationRestriction,
+		TokenExpiration,
+		ScanAllPolicy,
+		UAAClientID,
+		UAAClientSecret,
+		UAAEndpoint,
+		UAAVerifyCert,
+		ReadOnly,
+	}
+
+	//value is default value
+	HarborStringKeysMap = map[string]string{
+		AUTHMode:                   "db_auth",
+		LDAPURL:                    "",
+		LDAPSearchDN:               "",
+		LDAPSearchPwd:              "",
+		LDAPBaseDN:                 "",
+		LDAPUID:                    "",
+		LDAPFilter:                 "",
+		LDAPGroupAttributeName:     "",
+		LDAPGroupBaseDN:            "",
+		LDAPGroupSearchFilter:      "",
+		EmailHost:                  "smtp.mydomain.com",
+		EmailUsername:              "sample_admin@mydomain.com",
+		EmailPassword:              "abc",
+		EmailFrom:                  "admin <sample_admin@mydomain.com>",
+		EmailIdentity:              "",
+		ProjectCreationRestriction: ProCrtRestrEveryone,
+		UAAClientID:                "",
+		UAAEndpoint:                "",
+	}
+
+	HarborNumKeysMap = map[string]int{
+		EmailPort:            25,
+		LDAPScope:            2,
+		LDAPTimeout:          5,
+		LDAPGroupSearchScope: 2,
+		TokenExpiration:      30,
+	}
+
+	HarborBoolKeysMap = map[string]bool{
+		EmailSSL:         false,
+		EmailInsecure:    false,
+		SelfRegistration: true,
+		LDAPVerifyCert:   true,
+		UAAVerifyCert:    true,
+		ReadOnly:         false,
+	}
+
+	HarborPasswordKeys = []string{
+		EmailPassword,
+		LDAPSearchPwd,
+		UAAClientSecret,
+	}
 )

src/common/dao/base.go

@@ -46,7 +46,6 @@ type Database interface {
 
 // InitClairDB ...
 func InitClairDB(clairDB *models.PostGreSQL) error {
-	//Except for password other information will not be configurable, so keep it hard coded for 1.2.0.
 	p := &pgsql{
 		host:     clairDB.Host,
 		port:     strconv.Itoa(clairDB.Port),
@@ -62,25 +61,26 @@ func InitClairDB(clairDB *models.PostGreSQL) error {
 	return nil
 }
 
-// InitDatabase initializes the database, there's an optional parm as a flag
+// UpgradeSchema will call the internal migrator to upgrade schema based on the setting of database.
-// to indicate whether it should initialize the schema.
+func UpgradeSchema(database *models.Database) error {
-func InitDatabase(database *models.Database, initSchema ...bool) error {
+	db, err := getDatabase(database)
+	if err != nil {
+		return err
+	}
+	return db.UpgradeSchema()
+}
+
+// InitDatabase registers the database
+func InitDatabase(database *models.Database) error {
 	db, err := getDatabase(database)
 	if err != nil {
 		return err
 	}
 
-	log.Infof("initializing database: %s", db.String())
+	log.Infof("Registering database: %s", db.String())
 	if err := db.Register(); err != nil {
 		return err
 	}
-	if len(initSchema) > 0 && initSchema[0] {
-		err := db.UpgradeSchema()
-		if err != nil {
-			return err
-		}
-	}
-
 	version, err := GetSchemaVersion()
 	if err != nil {
 		return err
@@ -90,7 +90,7 @@ func InitDatabase(database *models.Database, initSchema ...bool) error {
 			SchemaVersion, version.Version)
 	}
 
-	log.Info("initialize database completed")
+	log.Info("Register database completed")
 	return nil
 }
 

src/common/dao/version.go

@@ -20,7 +20,7 @@ import (
 
 const (
 	// SchemaVersion is the version of database schema
-	SchemaVersion = "1.5.0"
+	SchemaVersion = "1.6.0"
 )
 
 // GetSchemaVersion return the version of database schema

src/common/job/const.go

@@ -9,6 +9,8 @@ const (
 	ImageDelete = "IMAGE_DELETE"
 	// ImageReplicate : the name of image replicate job in job service
 	ImageReplicate = "IMAGE_REPLICATE"
+	// ImageGC the name of image garbage collection job in job service
+	ImageGC = "IMAGE_GC"
 
 	//JobKindGeneric : Kind of generic job
 	JobKindGeneric = "Generic"

src/common/models/ldap.go

@@ -33,6 +33,7 @@ type LdapGroupConf struct {
 	LdapGroupFilter        string `json:"ldap_group_filter,omitempty"`
 	LdapGroupNameAttribute string `json:"ldap_group_name_attribute,omitempty"`
 	LdapGroupSearchScope   int    `json:"ldap_group_search_scope"`
+	LdapGroupAdminDN       string `json:"ldap_group_admin_dn,omitempty"`
 }
 
 // LdapUser ...

src/common/registryctl/client.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package registryctl
+
+import (
+	"os"
+
+	"github.com/vmware/harbor/src/common"
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/registryctl/client"
+)
+
+var (
+	// RegistryCtlClient is a client for registry controller
+	RegistryCtlClient client.Client
+)
+
+// Init ...
+func Init() {
+	initRegistryCtlClient()
+}
+
+func initRegistryCtlClient() {
+	registryCtlURL := os.Getenv("REGISTRY_CONTROLLER_URL")
+	if len(registryCtlURL) == 0 {
+		registryCtlURL = common.DefaultRegistryControllerEndpoint
+	}
+
+	log.Infof("initializing client for reigstry %s ...", registryCtlURL)
+	cfg := &client.Config{
+		Secret: os.Getenv("JOBSERVICE_SECRET"),
+	}
+	RegistryCtlClient = client.NewClient(registryCtlURL, cfg)
+}

src/common/utils/test/registryctl.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"time"
+)
+
+// GCResult ...
+type GCResult struct {
+	Status    bool      `json:"status"`
+	Msg       string    `json:"msg"`
+	StartTime time.Time `json:"starttime"`
+	EndTime   time.Time `json:"endtime"`
+}
+
+// NewRegistryCtl returns a mock registry server
+func NewRegistryCtl(config map[string]interface{}) (*httptest.Server, error) {
+	m := []*RequestHandlerMapping{}
+
+	gcr := GCResult{true, "hello-world", time.Now(), time.Now()}
+	b, err := json.Marshal(gcr)
+	if err != nil {
+		return nil, err
+	}
+
+	resp := &Response{
+		StatusCode: http.StatusOK,
+		Body:       b,
+	}
+
+	m = append(m, &RequestHandlerMapping{
+		Method:  "GET",
+		Pattern: "/api/health",
+		Handler: Handler(&Response{
+			StatusCode: http.StatusOK,
+		}),
+	})
+
+	m = append(m, &RequestHandlerMapping{
+		Method:  "POST",
+		Pattern: "/api/registry/gc",
+		Handler: Handler(resp),
+	})
+
+	return NewServer(m...), nil
+}

src/common/utils/utils.go

@@ -167,3 +167,35 @@ func ParseProjectIDOrName(value interface{}) (int64, string, error) {
 	}
 	return id, name, nil
 }
+
+//SafeCastString -- cast a object to string saftely
+func SafeCastString(value interface{}) string {
+	if result, ok := value.(string); ok {
+		return result
+	}
+	return ""
+}
+
+//SafeCastInt --
+func SafeCastInt(value interface{}) int {
+	if result, ok := value.(int); ok {
+		return result
+	}
+	return 0
+}
+
+//SafeCastBool --
+func SafeCastBool(value interface{}) bool {
+	if result, ok := value.(bool); ok {
+		return result
+	}
+	return false
+}
+
+//SafeCastFloat64 --
+func SafeCastFloat64(value interface{}) float64 {
+	if result, ok := value.(float64); ok {
+		return result
+	}
+	return 0
+}

src/common/utils/utils_test.go

@@ -248,3 +248,91 @@ func TestConvertMapToStruct(t *testing.T) {
 		}
 	}
 }
+
+func TestSafeCastString(t *testing.T) {
+	type args struct {
+		value interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"nil value", args{nil}, ""},
+		{"normal string", args{"sample"}, "sample"},
+		{"wrong type", args{12}, ""},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := SafeCastString(tt.args.value); got != tt.want {
+				t.Errorf("SafeCastString() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSafeCastBool(t *testing.T) {
+	type args struct {
+		value interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{"nil value", args{nil}, false},
+		{"normal bool", args{true}, true},
+		{"wrong type", args{"true"}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := SafeCastBool(tt.args.value); got != tt.want {
+				t.Errorf("SafeCastBool() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSafeCastInt(t *testing.T) {
+	type args struct {
+		value interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want int
+	}{
+		{"nil value", args{nil}, 0},
+		{"normal int", args{1234}, 1234},
+		{"wrong type", args{"sample"}, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := SafeCastInt(tt.args.value); got != tt.want {
+				t.Errorf("SafeCastInt() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestSafeCastFloat64(t *testing.T) {
+	type args struct {
+		value interface{}
+	}
+	tests := []struct {
+		name string
+		args args
+		want float64
+	}{
+		{"nil value", args{nil}, 0},
+		{"normal float64", args{12.34}, 12.34},
+		{"wrong type", args{false}, 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := SafeCastFloat64(tt.args.value); got != tt.want {
+				t.Errorf("SafeCastFloat64() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

src/jobservice/job/impl/gc/job.go

@@ -0,0 +1,70 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package gc
+
+import (
+	"github.com/vmware/harbor/src/common/registryctl"
+	"github.com/vmware/harbor/src/jobservice/env"
+	"github.com/vmware/harbor/src/jobservice/logger"
+	"github.com/vmware/harbor/src/registryctl/client"
+)
+
+// GarbageCollector is the struct to run registry's garbage collection
+type GarbageCollector struct {
+	registryCtlClient client.Client
+	logger            logger.Interface
+}
+
+// MaxFails implements the interface in job/Interface
+func (gc *GarbageCollector) MaxFails() uint {
+	return 1
+}
+
+// ShouldRetry implements the interface in job/Interface
+func (gc *GarbageCollector) ShouldRetry() bool {
+	return false
+}
+
+// Validate implements the interface in job/Interface
+func (gc *GarbageCollector) Validate(params map[string]interface{}) error {
+	return nil
+}
+
+// Run implements the interface in job/Interface
+func (gc *GarbageCollector) Run(ctx env.JobContext, params map[string]interface{}) error {
+	if err := gc.init(ctx); err != nil {
+		return err
+	}
+	if err := gc.registryCtlClient.Health(); err != nil {
+		gc.logger.Errorf("failed to start gc as regsitry controller is unreachable: %v", err)
+		return err
+	}
+	gc.logger.Infof("start to run gc in job.")
+	gcr, err := gc.registryCtlClient.StartGC()
+	if err != nil {
+		gc.logger.Errorf("failed to get gc result: %v", err)
+		return err
+	}
+	gc.logger.Infof("GC results: status: %t, message: %s, start: %s, end: %s.", gcr.Status, gcr.Msg, gcr.StartTime, gcr.EndTime)
+	gc.logger.Infof("success to run gc in job.")
+	return nil
+}
+
+func (gc *GarbageCollector) init(ctx env.JobContext) error {
+	registryctl.Init()
+	gc.registryCtlClient = registryctl.RegistryCtlClient
+	gc.logger = ctx.GetLogger()
+	return nil
+}

src/jobservice/runtime/bootstrap.go

@@ -17,6 +17,7 @@ import (
 	"github.com/vmware/harbor/src/jobservice/core"
 	"github.com/vmware/harbor/src/jobservice/env"
 	"github.com/vmware/harbor/src/jobservice/job/impl"
+	"github.com/vmware/harbor/src/jobservice/job/impl/gc"
 	"github.com/vmware/harbor/src/jobservice/job/impl/replication"
 	"github.com/vmware/harbor/src/jobservice/job/impl/scan"
 	"github.com/vmware/harbor/src/jobservice/logger"
@@ -186,6 +187,7 @@ func (bs *Bootstrap) loadAndRunRedisWorkerPool(ctx *env.Context, cfg *config.Con
 			job.ImageTransfer:  (*replication.Transfer)(nil),
 			job.ImageDelete:    (*replication.Deleter)(nil),
 			job.ImageReplicate: (*replication.Replicator)(nil),
+			job.ImageGC:        (*gc.GarbageCollector)(nil),
 		}); err != nil {
 		//exit
 		return nil, err

src/registryctl/api/base.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+func handleInternalServerError(w http.ResponseWriter) {
+	http.Error(w, http.StatusText(http.StatusInternalServerError),
+		http.StatusInternalServerError)
+}
+
+func handleUnauthorized(w http.ResponseWriter) {
+	http.Error(w, http.StatusText(http.StatusUnauthorized),
+		http.StatusUnauthorized)
+}
+
+// response status code will be written automatically if there is an error
+func writeJSON(w http.ResponseWriter, v interface{}) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		handleInternalServerError(w)
+		return err
+	}
+
+	if _, err = w.Write(b); err != nil {
+		return err
+	}
+	return nil
+}

src/registryctl/api/base_test.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestHandleInternalServerError(t *testing.T) {
+	w := httptest.NewRecorder()
+	handleInternalServerError(w)
+
+	if w.Code != http.StatusInternalServerError {
+		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusInternalServerError)
+	}
+
+}

src/registryctl/api/health.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"net/http"
+
+	"github.com/vmware/harbor/src/common/utils/log"
+)
+
+// Health ...
+func Health(w http.ResponseWriter, r *http.Request) {
+	if err := writeJSON(w, "healthy"); err != nil {
+		log.Errorf("Failed to write response: %v", err)
+		return
+	}
+}

src/registryctl/api/health_test.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHealth(t *testing.T) {
+	w := httptest.NewRecorder()
+	req, _ := http.NewRequest("GET", "", nil)
+	Health(w, req)
+	assert.Equal(t, http.StatusOK, w.Code)
+	result, _ := ioutil.ReadAll(w.Body)
+	assert.Equal(t, "\"healthy\"", string(result))
+}

src/registryctl/api/registry.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"bytes"
+	"net/http"
+	"time"
+
+	"os/exec"
+
+	"github.com/vmware/harbor/src/common/utils/log"
+)
+
+const (
+	regConf = "/etc/registry/config.yml"
+)
+
+// GCResult ...
+type GCResult struct {
+	Status    bool      `json:"status"`
+	Msg       string    `json:"msg"`
+	StartTime time.Time `json:"starttime"`
+	EndTime   time.Time `json:"endtime"`
+}
+
+// StartGC ...
+func StartGC(w http.ResponseWriter, r *http.Request) {
+	cmd := exec.Command("/bin/bash", "-c", "registry garbage-collect "+regConf)
+	var outBuf, errBuf bytes.Buffer
+	cmd.Stdout = &outBuf
+	cmd.Stderr = &errBuf
+
+	start := time.Now()
+	if err := cmd.Run(); err != nil {
+		log.Errorf("Fail to execute GC: %v, command err: %s", err, errBuf.String())
+		handleInternalServerError(w)
+		return
+	}
+
+	gcr := GCResult{true, outBuf.String(), start, time.Now()}
+	if err := writeJSON(w, gcr); err != nil {
+		log.Errorf("failed to write response: %v", err)
+		return
+	}
+}

src/registryctl/auth/auth.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"errors"
+	"net/http"
+)
+
+var (
+	// ErrInvalidCredential is returned when the auth token does not authenticate correctly.
+	ErrInvalidCredential = errors.New("invalid authorization credential")
+)
+
+// AuthenticationHandler is an interface for authorizing a request
+type AuthenticationHandler interface {
+
+	// AuthorizeRequest ...
+	AuthorizeRequest(req *http.Request) error
+}

src/registryctl/auth/secret.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+
+	"github.com/vmware/harbor/src/common/secret"
+)
+
+//HarborSecret is the prefix of the value of Authorization header.
+const HarborSecret = secret.HeaderPrefix
+
+var (
+	// ErrNoSecret ...
+	ErrNoSecret = errors.New("no secret auth credentials")
+)
+
+type secretHandler struct {
+	secrets map[string]string
+}
+
+// NewSecretHandler creaters a new authentiation handler which adds
+// basic authentication credentials to a request.
+func NewSecretHandler(secrets map[string]string) AuthenticationHandler {
+	return &secretHandler{
+		secrets: secrets,
+	}
+}
+
+func (s *secretHandler) AuthorizeRequest(req *http.Request) error {
+	if len(s.secrets) == 0 || req == nil {
+		return ErrNoSecret
+	}
+
+	auth := req.Header.Get("Authorization")
+	if !strings.HasPrefix(auth, HarborSecret) {
+		return ErrInvalidCredential
+	}
+	secInReq := strings.TrimPrefix(auth, HarborSecret)
+
+	for _, v := range s.secrets {
+		if secInReq == v {
+			return nil
+		}
+	}
+
+	return ErrInvalidCredential
+}

src/registryctl/auth/secret_test.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	commonsecret "github.com/vmware/harbor/src/common/secret"
+)
+
+func TestAuthorizeRequestInvalid(t *testing.T) {
+	secret := "correct"
+	req, err := http.NewRequest("", "", nil)
+	if err != nil {
+		t.Fatalf("failed to create request: %v", err)
+	}
+	_ = commonsecret.AddToRequest(req, secret)
+
+	authenticator := NewSecretHandler(map[string]string{"secret1": "incorrect"})
+	err = authenticator.AuthorizeRequest(req)
+	assert.Equal(t, err, ErrInvalidCredential)
+
+}
+
+func TestAuthorizeRequestValid(t *testing.T) {
+	secret := "correct"
+	req, err := http.NewRequest("", "", nil)
+	if err != nil {
+		t.Fatalf("failed to create request: %v", err)
+	}
+	_ = commonsecret.AddToRequest(req, secret)
+
+	authenticator := NewSecretHandler(map[string]string{"secret1": "correct"})
+	err = authenticator.AuthorizeRequest(req)
+	assert.Nil(t, err)
+
+}

src/registryctl/client/client.go

@@ -0,0 +1,102 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	common_http "github.com/vmware/harbor/src/common/http"
+	"github.com/vmware/harbor/src/common/http/modifier/auth"
+	"github.com/vmware/harbor/src/common/utils"
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/registryctl/api"
+)
+
+// Client defines methods that an Regsitry client should implement
+type Client interface {
+	// Health tests the connection with registry server
+	Health() error
+	// StartGC enable the gc of registry server
+	StartGC() (*api.GCResult, error)
+}
+
+type client struct {
+	baseURL string
+	client  *common_http.Client
+}
+
+// Config contains configurations needed for client
+type Config struct {
+	Secret string
+}
+
+// NewClient return an instance of Registry client
+func NewClient(baseURL string, cfg *Config) Client {
+	baseURL = strings.TrimRight(baseURL, "/")
+	if !strings.Contains(baseURL, "://") {
+		baseURL = "http://" + baseURL
+	}
+	client := &client{
+		baseURL: baseURL,
+	}
+	if cfg != nil {
+		authorizer := auth.NewSecretAuthorizer(cfg.Secret)
+		client.client = common_http.NewClient(nil, authorizer)
+	}
+	return client
+}
+
+// Health ...
+func (c *client) Health() error {
+	addr := strings.Split(c.baseURL, "://")[1]
+	if !strings.Contains(addr, ":") {
+		addr = addr + ":80"
+	}
+	return utils.TestTCPConn(addr, 60, 2)
+}
+
+// StartGC ...
+func (c *client) StartGC() (*api.GCResult, error) {
+	url := c.baseURL + "/api/registry/gc"
+	gcr := &api.GCResult{}
+
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		log.Errorf("Failed to start gc: %d", resp.StatusCode)
+		return nil, fmt.Errorf("Failed to start GC: %d", resp.StatusCode)
+	}
+	if err := json.Unmarshal(data, gcr); err != nil {
+		return nil, err
+	}
+
+	return gcr, nil
+}

src/registryctl/client/client_test.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package client
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/common/utils/test"
+)
+
+var c Client
+
+func TestMain(m *testing.M) {
+
+	server, err := test.NewRegistryCtl(nil)
+	if err != nil {
+		fmt.Printf("failed to create regsitry: %v", err)
+		os.Exit(1)
+	}
+
+	c = NewClient(server.URL, &Config{})
+
+	os.Exit(m.Run())
+}
+
+func TesHealth(t *testing.T) {
+	err := c.Health()
+	assert.Nil(t, err)
+}
+
+func TesStartGC(t *testing.T) {
+	gcr, err := c.StartGC()
+	assert.NotNil(t, err)
+	assert.Equal(t, gcr.Msg, "hello-world")
+	assert.Equal(t, gcr.Status, true)
+}

src/registryctl/config/config.go

@@ -0,0 +1,103 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"io/ioutil"
+	"os"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+//DefaultConfig ...
+var DefaultConfig = &Configuration{}
+
+//Configuration loads the configuration of registry controller.
+type Configuration struct {
+	Protocol    string `yaml:"protocol"`
+	Port        string `yaml:"port"`
+	LogLevel    string `yaml:"log_level"`
+	HTTPSConfig struct {
+		Cert string `yaml:"cert"`
+		Key  string `yaml:"key"`
+	} `yaml:"https_config,omitempty"`
+}
+
+//Load the configuration options from the specified yaml file.
+func (c *Configuration) Load(yamlFilePath string, detectEnv bool) error {
+	if len(yamlFilePath) != 0 {
+		//Try to load from file first
+		data, err := ioutil.ReadFile(yamlFilePath)
+		if err != nil {
+			return err
+		}
+		if err = yaml.Unmarshal(data, c); err != nil {
+			return err
+		}
+	}
+
+	if detectEnv {
+		c.loadEnvs()
+	}
+
+	return nil
+}
+
+//GetLogLevel returns the log level
+func GetLogLevel() string {
+	return DefaultConfig.LogLevel
+}
+
+//GetJobAuthSecret get the auth secret from the env
+func GetJobAuthSecret() string {
+	return os.Getenv("JOBSERVICE_SECRET")
+}
+
+//GetUIAuthSecret get the auth secret of UI side
+func GetUIAuthSecret() string {
+	return os.Getenv("UI_SECRET")
+}
+
+//loadEnvs Load env variables
+func (c *Configuration) loadEnvs() {
+	prot := os.Getenv("REGISTRYCTL_PROTOCOL")
+	if len(prot) != 0 {
+		c.Protocol = prot
+	}
+
+	p := os.Getenv("PORT")
+	if len(p) != 0 {
+		c.Port = p
+	}
+
+	//Only when protocol is https
+	if c.Protocol == "HTTPS" {
+		cert := os.Getenv("REGISTRYCTL_HTTPS_CERT")
+		if len(cert) != 0 {
+			c.HTTPSConfig.Cert = cert
+		}
+
+		certKey := os.Getenv("REGISTRYCTL_HTTPS_KEY")
+		if len(certKey) != 0 {
+			c.HTTPSConfig.Key = certKey
+		}
+	}
+
+	loggerLevel := os.Getenv("LOG_LEVEL")
+	if len(loggerLevel) != 0 {
+		c.LogLevel = loggerLevel
+	}
+
+}

src/registryctl/config/config_test.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigDoesNotExists(t *testing.T) {
+	cfg := &Configuration{}
+	err := cfg.Load("./config.not-existing.yaml", false)
+	assert.NotNil(t, err)
+}
+
+func TestConfigLoadingWithEnv(t *testing.T) {
+	os.Setenv("REGISTRYCTL_PROTOCOL", "https")
+	os.Setenv("PORT", "1000")
+	os.Setenv("LOG_LEVEL", "DEBUG")
+
+	cfg := &Configuration{}
+	err := cfg.Load("../config_test.yml", true)
+	assert.Nil(t, err)
+	assert.Equal(t, "https", cfg.Protocol)
+	assert.Equal(t, "1000", cfg.Port)
+	assert.Equal(t, "DEBUG", cfg.LogLevel)
+}
+
+func TestConfigLoadingWithYml(t *testing.T) {
+	cfg := &Configuration{}
+	err := cfg.Load("../config_test.yml", false)
+	assert.Nil(t, err)
+	assert.Equal(t, "http", cfg.Protocol)
+	assert.Equal(t, "1234", cfg.Port)
+	assert.Equal(t, "ERROR", cfg.LogLevel)
+}
+
+func TestGetLogLevel(t *testing.T) {
+	err := DefaultConfig.Load("../config_test.yml", false)
+	assert.Nil(t, err)
+	assert.Equal(t, "ERROR", GetLogLevel())
+}
+
+func TestGetJobAuthSecret(t *testing.T) {
+	os.Setenv("JOBSERVICE_SECRET", "test_job_secret")
+	assert.Equal(t, "test_job_secret", GetJobAuthSecret())
+}
+
+func TestGetUIAuthSecret(t *testing.T) {
+	os.Setenv("UI_SECRET", "test_ui_secret")
+	assert.Equal(t, "test_ui_secret", GetUIAuthSecret())
+}

src/registryctl/config_test.yml

@@ -0,0 +1,8 @@
+---
+protocol: "http"
+port: 1234
+log_level: "ERROR"
+
+https_config:
+  cert: "server.crt"
+  key: "server.key"

src/registryctl/handlers/handler.go

@@ -0,0 +1,82 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"net/http"
+	"os"
+
+	gorilla_handlers "github.com/gorilla/handlers"
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/registryctl/auth"
+)
+
+// NewHandlerChain returns a gorilla router which is wrapped by  authenticate handler
+// and logging handler
+func NewHandlerChain() http.Handler {
+	h := newRouter()
+	secrets := map[string]string{
+		"jobSecret": os.Getenv("JOBSERVICE_SECRET"),
+	}
+	insecureAPIs := map[string]bool{
+		"/api/health": true,
+	}
+	h = newAuthHandler(auth.NewSecretHandler(secrets), h, insecureAPIs)
+	h = gorilla_handlers.LoggingHandler(os.Stdout, h)
+	return h
+}
+
+type authHandler struct {
+	authenticator auth.AuthenticationHandler
+	handler       http.Handler
+	insecureAPIs  map[string]bool
+}
+
+func newAuthHandler(authenticator auth.AuthenticationHandler, handler http.Handler, insecureAPIs map[string]bool) http.Handler {
+	return &authHandler{
+		authenticator: authenticator,
+		handler:       handler,
+		insecureAPIs:  insecureAPIs,
+	}
+}
+
+func (a *authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if a.authenticator == nil {
+		log.Errorf("No authenticator found in regsitry controller.")
+		http.Error(w, http.StatusText(http.StatusInternalServerError),
+			http.StatusInternalServerError)
+		return
+	}
+
+	if a.insecureAPIs != nil && a.insecureAPIs[r.URL.Path] {
+		if a.handler != nil {
+			a.handler.ServeHTTP(w, r)
+		}
+		return
+	}
+
+	err := a.authenticator.AuthorizeRequest(r)
+	if err != nil {
+		log.Errorf("failed to authenticate request: %v", err)
+		http.Error(w, http.StatusText(http.StatusUnauthorized),
+			http.StatusUnauthorized)
+		return
+	}
+
+	if a.handler != nil {
+		a.handler.ServeHTTP(w, r)
+	}
+	return
+}

src/registryctl/handlers/handler_test.go

@@ -0,0 +1,69 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/registryctl/auth"
+)
+
+type fakeAuthenticator struct {
+	err error
+}
+
+func (f *fakeAuthenticator) AuthorizeRequest(req *http.Request) error {
+	return f.err
+}
+
+type fakeHandler struct {
+	responseCode int
+}
+
+func (f *fakeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(f.responseCode)
+}
+
+func TestNewAuthHandler(t *testing.T) {
+	cases := []struct {
+		authenticator auth.AuthenticationHandler
+		handler       http.Handler
+		insecureAPIs  map[string]bool
+		responseCode  int
+		requestURL    string
+	}{
+		{nil, nil, nil, http.StatusInternalServerError, "http://localhost/good"},
+		{&fakeAuthenticator{err: nil}, nil, nil, http.StatusOK, "http://localhost/hello"},
+		{&fakeAuthenticator{err: errors.New("error")}, nil, nil, http.StatusUnauthorized, "http://localhost/hello"},
+		{&fakeAuthenticator{err: nil}, &fakeHandler{http.StatusNotFound}, nil, http.StatusNotFound, "http://localhost/notexsit"}, {&fakeAuthenticator{err: nil}, &fakeHandler{http.StatusOK}, map[string]bool{"/api/insecure": true}, http.StatusOK, "http://localhost/api/insecure"},
+	}
+
+	for _, c := range cases {
+		handler := newAuthHandler(c.authenticator, c.handler, c.insecureAPIs)
+		w := httptest.NewRecorder()
+		r := httptest.NewRequest("GET", c.requestURL, nil)
+		handler.ServeHTTP(w, r)
+		assert.Equal(t, c.responseCode, w.Code, "unexpected response code")
+	}
+	handler := NewHandlerChain()
+	w := httptest.NewRecorder()
+	r := httptest.NewRequest("GET", "http://localhost/api/health", nil)
+	handler.ServeHTTP(w, r)
+
+}

src/registryctl/handlers/router.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/vmware/harbor/src/registryctl/api"
+)
+
+func newRouter() http.Handler {
+	r := mux.NewRouter()
+	r.HandleFunc("/api/registry/gc", api.StartGC).Methods("POST")
+	r.HandleFunc("/api/health", api.Health).Methods("GET")
+	return r
+}

src/registryctl/main.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"net/http"
+
+	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/registryctl/config"
+	"github.com/vmware/harbor/src/registryctl/handlers"
+)
+
+// RegistryCtl for registry controller
+type RegistryCtl struct {
+	ServerConf config.Configuration
+	Handler    http.Handler
+}
+
+// Start the registry controller
+func (s *RegistryCtl) Start() {
+	regCtl := &http.Server{
+		Addr:    ":" + s.ServerConf.Port,
+		Handler: s.Handler,
+	}
+
+	if s.ServerConf.Protocol == "HTTPS" {
+		tlsCfg := &tls.Config{
+			MinVersion:               tls.VersionTLS12,
+			CurvePreferences:         []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256},
+			PreferServerCipherSuites: true,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_256_GCM_SHA384,
+				tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+			},
+		}
+
+		regCtl.TLSConfig = tlsCfg
+		regCtl.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler), 0)
+	}
+
+	var err error
+	if s.ServerConf.Protocol == "HTTPS" {
+		err = regCtl.ListenAndServeTLS(s.ServerConf.HTTPSConfig.Cert, s.ServerConf.HTTPSConfig.Key)
+	} else {
+		err = regCtl.ListenAndServe()
+	}
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return
+}
+
+func main() {
+
+	configPath := flag.String("c", "", "Specify the yaml config file path")
+	flag.Parse()
+
+	if configPath == nil || len(*configPath) == 0 {
+		flag.Usage()
+		log.Fatal("Config file should be specified")
+	}
+
+	if err := config.DefaultConfig.Load(*configPath, true); err != nil {
+		log.Fatalf("Failed to load configurations with error: %s\n", err)
+	}
+
+	regCtl := &RegistryCtl{
+		ServerConf: *config.DefaultConfig,
+		Handler:    handlers.NewHandlerChain(),
+	}
+
+	regCtl.Start()
+}

src/ui/api/config.go

@@ -26,88 +26,6 @@ import (
 	"github.com/vmware/harbor/src/ui/config"
 )
 
-var (
-	// the keys of configurations which user can modify in PUT method and user can
-	// get in GET method
-	validKeys = []string{
-		common.AUTHMode,
-		common.SelfRegistration,
-		common.LDAPURL,
-		common.LDAPSearchDN,
-		common.LDAPSearchPwd,
-		common.LDAPBaseDN,
-		common.LDAPUID,
-		common.LDAPFilter,
-		common.LDAPScope,
-		common.LDAPTimeout,
-		common.LDAPVerifyCert,
-		common.LDAPGroupAttributeName,
-		common.LDAPGroupBaseDN,
-		common.LDAPGroupSearchFilter,
-		common.LDAPGroupSearchScope,
-		common.EmailHost,
-		common.EmailPort,
-		common.EmailUsername,
-		common.EmailPassword,
-		common.EmailFrom,
-		common.EmailSSL,
-		common.EmailIdentity,
-		common.EmailInsecure,
-		common.ProjectCreationRestriction,
-		common.TokenExpiration,
-		common.ScanAllPolicy,
-		common.UAAClientID,
-		common.UAAClientSecret,
-		common.UAAEndpoint,
-		common.UAAVerifyCert,
-		common.ReadOnly,
-	}
-
-	stringKeys = []string{
-		common.AUTHMode,
-		common.LDAPURL,
-		common.LDAPSearchDN,
-		common.LDAPSearchPwd,
-		common.LDAPBaseDN,
-		common.LDAPUID,
-		common.LDAPFilter,
-		common.LDAPGroupAttributeName,
-		common.LDAPGroupBaseDN,
-		common.LDAPGroupSearchFilter,
-		common.EmailHost,
-		common.EmailUsername,
-		common.EmailPassword,
-		common.EmailFrom,
-		common.EmailIdentity,
-		common.ProjectCreationRestriction,
-		common.UAAClientID,
-		common.UAAEndpoint,
-	}
-
-	numKeys = []string{
-		common.EmailPort,
-		common.LDAPScope,
-		common.LDAPTimeout,
-		common.LDAPGroupSearchScope,
-		common.TokenExpiration,
-	}
-
-	boolKeys = []string{
-		common.EmailSSL,
-		common.EmailInsecure,
-		common.SelfRegistration,
-		common.LDAPVerifyCert,
-		common.UAAVerifyCert,
-		common.ReadOnly,
-	}
-
-	passwordKeys = []string{
-		common.EmailPassword,
-		common.LDAPSearchPwd,
-		common.UAAClientSecret,
-	}
-)
-
 // ConfigAPI ...
 type ConfigAPI struct {
 	BaseController
@@ -140,7 +58,7 @@ func (c *ConfigAPI) Get() {
 	}
 
 	cfgs := map[string]interface{}{}
-	for _, k := range validKeys {
+	for _, k := range common.HarborValidKeys {
 		if v, ok := configs[k]; ok {
 			cfgs[k] = v
 		}
@@ -162,7 +80,7 @@ func (c *ConfigAPI) Put() {
 	c.DecodeJSONReq(&m)
 
 	cfg := map[string]interface{}{}
-	for _, k := range validKeys {
+	for _, k := range common.HarborValidKeys {
 		if v, ok := m[k]; ok {
 			cfg[k] = v
 		}
@@ -205,7 +123,7 @@ func (c *ConfigAPI) Reset() {
 
 func validateCfg(c map[string]interface{}) (bool, error) {
 	strMap := map[string]string{}
-	for _, k := range stringKeys {
+	for k := range common.HarborStringKeysMap {
 		if _, ok := c[k]; !ok {
 			continue
 		}
@@ -215,7 +133,7 @@ func validateCfg(c map[string]interface{}) (bool, error) {
 		strMap[k] = c[k].(string)
 	}
 	numMap := map[string]int{}
-	for _, k := range numKeys {
+	for k := range common.HarborNumKeysMap {
 		if _, ok := c[k]; !ok {
 			continue
 		}
@@ -225,7 +143,7 @@ func validateCfg(c map[string]interface{}) (bool, error) {
 		numMap[k] = int(c[k].(float64))
 	}
 	boolMap := map[string]bool{}
-	for _, k := range boolKeys {
+	for k := range common.HarborBoolKeysMap {
 		if _, ok := c[k]; !ok {
 			continue
 		}
@@ -327,7 +245,7 @@ func validateCfg(c map[string]interface{}) (bool, error) {
 func convertForGet(cfg map[string]interface{}) (map[string]*value, error) {
 	result := map[string]*value{}
 
-	for _, k := range passwordKeys {
+	for _, k := range common.HarborPasswordKeys {
 		if _, ok := cfg[k]; ok {
 			delete(cfg, k)
 		}

src/ui/api/harborapi_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/vmware/harbor/src/common/dao"
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils"
+	ldapUtils "github.com/vmware/harbor/src/common/utils/ldap"
 	"github.com/vmware/harbor/src/ui/config"
 	"github.com/vmware/harbor/src/ui/filter"
 	"github.com/vmware/harbor/tests/apitests/apilib"
@@ -77,6 +78,25 @@ type usrInfo struct {
 }
 
 func init() {
+	ldapConfig := models.LdapConf{
+		LdapURL:               "ldap://127.0.0.1:389",
+		LdapSearchDn:          "cn=admin,dc=example,dc=com",
+		LdapSearchPassword:    "admin",
+		LdapBaseDn:            "dc=example,dc=com",
+		LdapUID:               "cn",
+		LdapScope:             2,
+		LdapConnectionTimeout: 5,
+	}
+	ldapGroupConfig := models.LdapGroupConf{
+		LdapGroupBaseDN:        "ou=groups,dc=example,dc=com",
+		LdapGroupFilter:        "objectclass=groupOfNames",
+		LdapGroupSearchScope:   2,
+		LdapGroupNameAttribute: "cn",
+	}
+	ldapTestConfig, err := ldapUtils.CreateWithAllConfig(ldapConfig, ldapGroupConfig)
+	if err != nil {
+		log.Fatalf("failed to initialize configurations: %v", err)
+	}
 	if err := config.Init(); err != nil {
 		log.Fatalf("failed to initialize configurations: %v", err)
 	}
@@ -134,7 +154,10 @@ func init() {
 	beego.Router("/api/systeminfo", &SystemInfoAPI{}, "get:GetGeneralInfo")
 	beego.Router("/api/systeminfo/volumes", &SystemInfoAPI{}, "get:GetVolumeInfo")
 	beego.Router("/api/systeminfo/getcert", &SystemInfoAPI{}, "get:GetCert")
-	beego.Router("/api/ldap/ping", &LdapAPI{}, "post:Ping")
+	beego.Router("/api/ldap/ping", &LdapAPI{ldapConfig: ldapTestConfig, useTestConfig: true}, "post:Ping")
+	beego.Router("/api/ldap/users/search", &LdapAPI{ldapConfig: ldapTestConfig, useTestConfig: true}, "get:Search")
+	beego.Router("/api/ldap/groups/search", &LdapAPI{ldapConfig: ldapTestConfig, useTestConfig: true}, "get:SearchGroup")
+	beego.Router("/api/ldap/users/import", &LdapAPI{ldapConfig: ldapTestConfig, useTestConfig: true}, "post:ImportUser")
 	beego.Router("/api/configurations", &ConfigAPI{})
 	beego.Router("/api/configurations/reset", &ConfigAPI{}, "post:Reset")
 	beego.Router("/api/email/ping", &EmailAPI{}, "post:Ping")

src/ui/api/ldap.go

@@ -21,11 +21,15 @@ import (
 	ldapUtils "github.com/vmware/harbor/src/common/utils/ldap"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/auth"
+
+	goldap "gopkg.in/ldap.v2"
 )
 
 // LdapAPI handles requesst to /api/ldap/ping /api/ldap/user/search /api/ldap/user/import
 type LdapAPI struct {
 	BaseController
+	ldapConfig    *ldapUtils.Session
+	useTestConfig bool // Only used for unit test
 }
 
 const (
@@ -47,6 +51,14 @@ func (l *LdapAPI) Prepare() {
 		l.HandleForbidden(l.SecurityCtx.GetUsername())
 		return
 	}
+	if l.useTestConfig {
+		ldapCfg, err := ldapUtils.LoadSystemLdapConfig()
+		if err != nil {
+			l.HandleInternalServerError(fmt.Sprintf("Can't load system configuration, error: %v", err))
+			return
+		}
+		l.ldapConfig = ldapCfg
+	}
 }
 
 // Ping ...
@@ -55,16 +67,11 @@ func (l *LdapAPI) Ping() {
 		LdapConnectionTimeout: 5,
 	}
 	var err error
-	var ldapSession *ldapUtils.Session
 
 	l.Ctx.Input.CopyBody(1 << 32)
 
 	if string(l.Ctx.Input.RequestBody) == "" {
-		ldapSession, err = ldapUtils.LoadSystemLdapConfig()
+		ldapSession := *l.ldapConfig
-		if err != nil {
-			l.HandleInternalServerError(fmt.Sprintf("Can't load system configuration, error: %v", err))
-			return
-		}
 		err = ldapSession.ConnectionTest()
 	} else {
 		l.DecodeJSONReqAndValidate(&ldapConfs)
@@ -81,7 +88,7 @@ func (l *LdapAPI) Ping() {
 func (l *LdapAPI) Search() {
 	var err error
 	var ldapUsers []models.LdapUser
-	ldapSession, err := ldapUtils.LoadSystemLdapConfig()
+	ldapSession := *l.ldapConfig
 	if err = ldapSession.Open(); err != nil {
 		l.HandleInternalServerError(fmt.Sprintf("Can't Open LDAP session, error: %v", err))
 		return
@@ -106,11 +113,10 @@ func (l *LdapAPI) Search() {
 func (l *LdapAPI) ImportUser() {
 	var ldapImportUsers models.LdapImportUser
 	var ldapFailedImportUsers []models.LdapFailedImportUser
-	var ldapConfs models.LdapConf
 
 	l.DecodeJSONReqAndValidate(&ldapImportUsers)
 
-	ldapFailedImportUsers, err := importUsers(ldapConfs, ldapImportUsers.LdapUIDList)
+	ldapFailedImportUsers, err := importUsers(ldapImportUsers.LdapUIDList, l.ldapConfig)
 
 	if err != nil {
 		l.HandleInternalServerError(fmt.Sprintf("LDAP import user fail, error: %v", err))
@@ -127,17 +133,12 @@ func (l *LdapAPI) ImportUser() {
 
 }
 
-func importUsers(ldapConfs models.LdapConf, ldapImportUsers []string) ([]models.LdapFailedImportUser, error) {
+func importUsers(ldapImportUsers []string, ldapConfig *ldapUtils.Session) ([]models.LdapFailedImportUser, error) {
 	var failedImportUser []models.LdapFailedImportUser
 	var u models.LdapFailedImportUser
 
-	ldapSession, err := ldapUtils.LoadSystemLdapConfig()
+	ldapSession := *ldapConfig
-	if err != nil {
+	if err := ldapSession.Open(); err != nil {
-		log.Errorf("Can't load system configuration, error: %v", err)
-		return nil, err
-	}
-
-	if err = ldapSession.Open(); err != nil {
 		log.Errorf("Can't connect to LDAP, error: %v", err)
 	}
 	defer ldapSession.Close()
@@ -194,19 +195,37 @@ func importUsers(ldapConfs models.LdapConf, ldapImportUsers []string) ([]models.
 
 // SearchGroup ... Search LDAP by groupname
 func (l *LdapAPI) SearchGroup() {
+	var ldapGroups []models.LdapGroup
+	var err error
 	searchName := l.GetString("groupname")
-	ldapSession, err := ldapUtils.LoadSystemLdapConfig()
+	groupDN := l.GetString("groupdn")
-	if err != nil {
+	ldapSession := *l.ldapConfig
-		l.HandleInternalServerError(fmt.Sprintf("Can't get LDAP system config, error: %v", err))
-		return
-	}
 	ldapSession.Open()
 	defer ldapSession.Close()
-	ldapGroups, err := ldapSession.SearchGroupByName(searchName)
+
+	//Search LDAP group by groupName or group DN
+	if len(searchName) > 0 {
+		ldapGroups, err = ldapSession.SearchGroupByName(searchName)
 		if err != nil {
 			l.HandleInternalServerError(fmt.Sprintf("Can't search LDAP group by name, error: %v", err))
 			return
 		}
+	} else if len(groupDN) > 0 {
+		if _, err := goldap.ParseDN(groupDN); err != nil {
+			l.HandleBadRequest(fmt.Sprintf("Invalid DN: %v", err))
+			return
+		}
+		ldapGroups, err = ldapSession.SearchGroupByDN(groupDN)
+		if err != nil {
+			//OpenLDAP usually return an error if DN is not found
+			l.HandleNotFound(fmt.Sprintf("Search LDAP group fail, error: %v", err))
+			return
+		}
+	}
+	if len(ldapGroups) == 0 {
+		l.HandleNotFound("No ldap group found")
+		return
+	}
 	l.Data["json"] = ldapGroups
 	l.ServeJSON()
 }

src/ui/api/ldap_test.go

@@ -0,0 +1,136 @@
+package api
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/vmware/harbor/src/common/models"
+)
+
+func TestLDAPPing(t *testing.T) {
+	cases := []*codeCheckingCase{
+		&codeCheckingCase{
+			request: &testingRequest{
+				method: http.MethodPost,
+				url:    "/api/ldap/ping",
+			},
+			code: http.StatusUnauthorized,
+		},
+		&codeCheckingCase{
+			request: &testingRequest{
+				method:     http.MethodPost,
+				url:        "/api/ldap/ping",
+				credential: admin,
+			},
+			code: http.StatusOK,
+		},
+		&codeCheckingCase{
+			request: &testingRequest{
+				method: http.MethodPost,
+				url:    "/api/ldap/ping",
+				bodyJSON: &models.LdapConf{
+					LdapURL:               "ldap://127.0.0.1:389",
+					LdapSearchDn:          "cn=admin,dc=example,dc=com",
+					LdapSearchPassword:    "admin",
+					LdapBaseDn:            "dc=example,dc=com",
+					LdapUID:               "cn",
+					LdapScope:             2,
+					LdapConnectionTimeout: 5,
+				},
+				credential: admin,
+			},
+			code: http.StatusOK,
+		},
+	}
+	runCodeCheckingCases(t, cases...)
+}
+
+func TestLDAPUserSearch(t *testing.T) {
+	cases := []*codeCheckingCase{
+		&codeCheckingCase{
+			request: &testingRequest{
+				method: http.MethodGet,
+				url:    "/api/ldap/users/search?username=mike",
+			},
+			code: http.StatusUnauthorized,
+		},
+		&codeCheckingCase{
+			request: &testingRequest{
+				method:     http.MethodGet,
+				url:        "/api/ldap/users/search?username=mike",
+				credential: admin,
+			},
+			code: http.StatusOK,
+		},
+	}
+	runCodeCheckingCases(t, cases...)
+}
+
+func TestLDAPGroupSearch(t *testing.T) {
+	cases := []*codeCheckingCase{
+		&codeCheckingCase{
+			request: &testingRequest{
+				method: http.MethodGet,
+				url:    "/api/ldap/groups/search?groupname=harbor_users",
+			},
+			code: http.StatusUnauthorized,
+		},
+		&codeCheckingCase{
+			request: &testingRequest{
+				method:     http.MethodGet,
+				url:        "/api/ldap/groups/search?groupname=harbor_users",
+				credential: admin,
+			},
+			code: http.StatusOK,
+		},
+	}
+	runCodeCheckingCases(t, cases...)
+}
+
+func TestLDAPGroupSearchWithDN(t *testing.T) {
+	cases := []*codeCheckingCase{
+		&codeCheckingCase{
+			request: &testingRequest{
+				method: http.MethodGet,
+				url:    "/api/ldap/groups/search?groupdn=cn=harbor_users,ou=groups,dc=example,dc=com",
+			},
+			code: http.StatusUnauthorized,
+		},
+		&codeCheckingCase{
+			request: &testingRequest{
+				method:     http.MethodGet,
+				url:        "/api/ldap/groups/search?groupname=cn=harbor_users,ou=groups,dc=example,dc=com",
+				credential: admin,
+			},
+			code: http.StatusOK,
+		},
+	}
+	runCodeCheckingCases(t, cases...)
+}
+
+func TestLDAPImportUser(t *testing.T) {
+	cases := []*codeCheckingCase{
+		&codeCheckingCase{
+			request: &testingRequest{
+				method: http.MethodPost,
+				url:    "/api/ldap/users/import",
+				bodyJSON: &models.LdapImportUser{
+					LdapUIDList: []string{"mike", "mike02"},
+				},
+			},
+			code: http.StatusUnauthorized,
+		},
+		&codeCheckingCase{
+			request: &testingRequest{
+				method: http.MethodPost,
+				url:    "/api/ldap/users/import",
+				bodyJSON: &models.LdapImportUser{
+					LdapUIDList: []string{"mike", "mike02"},
+				},
+				credential: admin,
+			},
+			code: http.StatusOK,
+		},
+	}
+	runCodeCheckingCases(t, cases...)
+}

src/ui/api/systeminfo.go

@@ -167,17 +167,17 @@ func (sia *SystemInfoAPI) GetGeneralInfo() {
 	_, caStatErr := os.Stat(defaultRootCert)
 	harborVersion := sia.getVersion()
 	info := GeneralInfo{
-		AdmiralEndpoint:             cfg[common.AdmiralEndpoint].(string),
+		AdmiralEndpoint:             utils.SafeCastString(cfg[common.AdmiralEndpoint]),
 		WithAdmiral:                 config.WithAdmiral(),
 		WithNotary:                  config.WithNotary(),
 		WithClair:                   config.WithClair(),
-		AuthMode:                    cfg[common.AUTHMode].(string),
+		AuthMode:                    utils.SafeCastString(cfg[common.AUTHMode]),
-		ProjectCreationRestrict:     cfg[common.ProjectCreationRestriction].(string),
+		ProjectCreationRestrict:     utils.SafeCastString(cfg[common.ProjectCreationRestriction]),
-		SelfRegistration:            cfg[common.SelfRegistration].(bool),
+		SelfRegistration:            utils.SafeCastBool(cfg[common.SelfRegistration]),
 		RegistryURL:                 registryURL,
 		HasCARoot:                   caStatErr == nil,
 		HarborVersion:               harborVersion,
-		RegistryStorageProviderName: cfg[common.RegistryStorageProviderName].(string),
+		RegistryStorageProviderName: utils.SafeCastString(cfg[common.RegistryStorageProviderName]),
 		ReadOnly:                    config.ReadOnly(),
 	}
 	if info.WithClair {