Merge branch 'master' into metadata_description

.drone.yml

@@ -48,16 +48,17 @@ pipeline:
     commands:
       - du -ks harbor-offline-installer-*.tgz | awk '{print $1 / 1024}' | { read x; echo $x MB; }
       - mkdir -p bundle
-      - mkdir -p latest
+      - mkdir -p pks-bundle
-      - echo $(git describe --tags) > latest/version 
+      - echo $(git describe --tags) > pks-bundle/version 
       - cp harbor-offline-installer-*.tgz bundle
-      - if [ ${DRONE_BRANCH} = "master" ]; then cp harbor-offline-installer-*.tgz latest/harbor-offline-installer-latest-master.tgz; else cp harbor-offline-installer-*.tgz latest/harbor-offline-installer-latest-release.tgz; fi
+      - if [ ${DRONE_BRANCH} = "master" ]; then cp harbor-offline-installer-*.tgz pks-bundle/harbor-offline-installer-latest-master.tgz; fi
+      - if ( echo ${DRONE_BRANCH} | grep "pks*" ); then cp harbor-offline-installer-*.tgz pks-bundle/harbor-offline-installer-latest-pks.tgz; fi
       - ls -la bundle
-      - ls -la latest
+      - ls -la pks-bundle
     when:
       repo: vmware/harbor
       event: [ push, tag ]
-      branch: [ master, release-*, refs/tags/* ]
+      branch: [ master, release-*, pks-*, refs/tags/* ]
       status: success
 
   notify-slack:
@@ -99,10 +100,10 @@ pipeline:
       branch: [ release-*, refs/tags/* ]
       status: success
 
-  publish-gcs-latest:
+  publish-gcs-pks-builds:
     image: maplain/drone-gcs:latest
     pull: true
-    source: latest
+    source: pks-bundle
     target: harbor-ci-pipeline-store/latest
     acl:
       - allUsers:READER
@@ -110,7 +111,7 @@ pipeline:
     when:
       repo: vmware/harbor
       event: [ push, tag ]
-      branch: [ master, release-*, refs/tags/* ]
+      branch: [ master, pks-*, refs/tags/* ]
       status: success
 
   trigger:

.drone.yml.sig

@@ -1 +1 @@
-eyJhbGciOiJIUzI1NiJ9.IyBIYXJib3IgZHJvbmUuCi0tLQp3b3Jrc3BhY2U6CiAgYmFzZTogL2Ryb25lCiAgcGF0aDogc3JjL2dpdGh1Yi5jb20vdm13YXJlL2hhcmJvcgoKcGlwZWxpbmU6CiAgY2xvbmU6CiAgICBpbWFnZTogcGx1Z2lucy9naXQKICAgIHRhZ3M6IHRydWUKICAgIHJlY3Vyc2l2ZTogZmFsc2UKCiAgaW50ZWdyYXRpb24tdGVzdC1vbi1wcjoKICAgIGltYWdlOiB2bXdhcmUvaGFyYm9yLWUyZS1lbmdpbmU6MS4zOAogICAgcHVsbDogdHJ1ZQogICAgcHJpdmlsZWdlZDogdHJ1ZQogICAgZW52aXJvbm1lbnQ6CiAgICAgIEJJTjogYmluCiAgICAgIEdPUEFUSDogL2Ryb25lCiAgICAgIFNIRUxMOiAvYmluL2Jhc2gKICAgICAgTE9HX1RFTVBfRElSOiBpbnN0YWxsLWxvZ3MKICAgICAgR0lUSFVCX0FVVE9NQVRJT05fQVBJX0tFWTogICR7R0lUSFVCX0FVVE9NQVRJT05fQVBJX0tFWX0KICAgICAgRFJPTkVfU0VSVkVSOiAgJHtEUk9ORV9TRVJWRVJ9CiAgICAgIERST05FX1RPS0VOOiAgJHtEUk9ORV9UT0tFTl9JTlRFfQogICAgICBIQVJCT1JfQURNSU46ICR7SEFSQk9SX0FETUlOfQogICAgICBIQVJCT1JfUEFTU1dPUkQ6ICR7SEFSQk9SX1BBU1NXT1JEfQogICAgICBHU19QUk9KRUNUX0lEOiAke0dTX1BST0pFQ1RfSUR9CiAgICAgIEdTX0NMSUVOVF9FTUFJTDogJHtHU19DTElFTlRfRU1BSUx9CiAgICAgIEdTX1BSSVZBVEVfS0VZOiAke0dTX1BSSVZBVEVfS0VZfQogICAgICBET01BSU46ICR7Q0lfRE9NQUlOfQogICAgICBNQUlMX1BXRDogJHtNQUlMX1BXRH0KICAgICAgTlBNX1VTRVJOQU1FOiAke05QTV9VU0VSTkFNRX0KICAgICAgTlBNX1BBU1NXT1JEOiAke05QTV9QQVNTV09SRH0KICAgIGNvbW1hbmRzOgogICAgICAtIHRlc3RzL2ludGVncmF0aW9uLnNoCiAgICB3aGVuOgogICAgICBzdGF0dXM6IHN1Y2Nlc3MKCiAgYnVuZGxlOgogICAgaW1hZ2U6IHZtd2FyZS9oYXJib3ItZTJlLWVuZ2luZToxLjM4CiAgICBwdWxsOiB0cnVlCiAgICBwcml2aWxlZ2VkOiB0cnVlCiAgICBlbnZpcm9ubWVudDoKICAgICAgQklOOiBiaW4KICAgICAgR09QQVRIOiAvZHJvbmUKICAgICAgU0hFTEw6IC9iaW4vYmFzaAogICAgICBCVUlMRF9OVU1CRVI6ICR7RFJPTkVfQlVJTERfTlVNQkVSfQogICAgY29tbWFuZHM6CiAgICAgIC0gZHUgLWtzIGhhcmJvci1vZmZsaW5lLWluc3RhbGxlci0qLnRneiB8IGF3ayAne3ByaW50ICQxIC8gMTAyNH0nIHwgeyByZWFkIHg7IGVjaG8gJHggTUI7IH0KICAgICAgLSBta2RpciAtcCBidW5kbGUKICAgICAgLSBta2RpciAtcCBsYXRlc3QKICAgICAgLSBlY2hvICQoZ2l0IGRlc2NyaWJlIC0tdGFncykgPiBsYXRlc3QvdmVyc2lvbiAKICAgICAgLSBjcCBoYXJib3Itb2ZmbGluZS1pbnN0YWxsZXItKi50Z3ogYnVuZGxlCiAgICAgIC0gaWYgWyAke0RST05FX0JSQU5DSH0gPSAibWFzdGVyIiBdOyB0aGVuIGNwIGhhcmJvci1vZmZsaW5lLWluc3RhbGxlci0qLnRneiBsYXRlc3QvaGFyYm9yLW9mZmxpbmUtaW5zdGFsbGVyLWxhdGVzdC1tYXN0ZXIudGd6OyBlbHNlIGNwIGhhcmJvci1vZmZsaW5lLWluc3RhbGxlci0qLnRneiBsYXRlc3QvaGFyYm9yLW9mZmxpbmUtaW5zdGFsbGVyLWxhdGVzdC1yZWxlYXNlLnRnejsgZmkKICAgICAgLSBscyAtbGEgYnVuZGxlCiAgICAgIC0gbHMgLWxhIGxhdGVzdAogICAgd2hlbjoKICAgICAgcmVwbzogdm13YXJlL2hhcmJvcgogICAgICBldmVudDogWyBwdXNoLCB0YWcgXQogICAgICBicmFuY2g6IFsgbWFzdGVyLCByZWxlYXNlLSosIHJlZnMvdGFncy8qIF0KICAgICAgc3RhdHVzOiBzdWNjZXNzCgogIG5vdGlmeS1zbGFjazoKICAgIGltYWdlOiBwbHVnaW5zL3NsYWNrCiAgICB3ZWJob29rOiAke1NMQUNLX1VSTH0KICAgIHVzZXJuYW1lOiBkcm9uZQogICAgdGVtcGxhdGU6ID4KICAgICAgYnVpbGQgaHR0cHM6Ly9jaS52Y25hLmlvL3Ztd2FyZS9oYXJib3Ive3sgYnVpbGQubnVtYmVyIH19IGZpbmlzaGVkIHdpdGggYSB7eyBidWlsZC5zdGF0dXMgfX0gc3RhdHVzLiBQbGVhc2UgZmluZCBsb2dzIGF0IGh0dHBzOi8vc3RvcmFnZS5nb29nbGVhcGlzLmNvbS9oYXJib3ItY2ktbG9ncy9pbnRlZ3JhdGlvbl9sb2dzX3t7IGJ1aWxkLm51bWJlciB9fV97eyBidWlsZC5jb21taXQgfX0udGFyLmd6CiAgICB3aGVuOgogICAgICByZXBvOiB2bXdhcmUvaGFyYm9yCiAgICAgIGJyYW5jaDogWyBtYXN0ZXIsIHJlbGVhc2UtKiwgcmVmcy90YWdzLyogXQogICAgICBzdGF0dXM6IFsgZmFpbHVyZSwgc3VjY2VzcyBdCgogIHB1Ymxpc2gtZ2NzLWJ1aWxkczoKICAgIGltYWdlOiBtYXBsYWluL2Ryb25lLWdjczpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIHNvdXJjZTogYnVuZGxlCiAgICB0YXJnZXQ6IGhhcmJvci1idWlsZHMKICAgIGFjbDoKICAgICAgLSBhbGxVc2VyczpSRUFERVIKICAgIGNhY2hlX2NvbnRyb2w6IHB1YmxpYyxtYXgtYWdlPTM2MDAKICAgIHdoZW46CiAgICAgIHJlcG86IHZtd2FyZS9oYXJib3IKICAgICAgZXZlbnQ6IFsgcHVzaCwgdGFnIF0KICAgICAgYnJhbmNoOiBbIG1hc3RlciwgcmVsZWFzZS0qIF0KICAgICAgc3RhdHVzOiBzdWNjZXNzCgogIHB1Ymxpc2gtZ2NzLXJlbGVhc2VzOgogICAgaW1hZ2U6IG1hcGxhaW4vZHJvbmUtZ2NzOmxhdGVzdAogICAgcHVsbDogdHJ1ZQogICAgc291cmNlOiBidW5kbGUKICAgIHRhcmdldDogaGFyYm9yLXJlbGVhc2VzCiAgICBhY2w6CiAgICAgIC0gYWxsVXNlcnM6UkVBREVSCiAgICBjYWNoZV9jb250cm9sOiBwdWJsaWMsbWF4LWFnZT0zNjAwCiAgICB3aGVuOgogICAgICByZXBvOiB2bXdhcmUvaGFyYm9yCiAgICAgIGV2ZW50OiBbIHB1c2gsIHRhZyBdCiAgICAgIGJyYW5jaDogWyByZWxlYXNlLSosIHJlZnMvdGFncy8qIF0KICAgICAgc3RhdHVzOiBzdWNjZXNzCgogIHB1Ymxpc2gtZ2NzLWxhdGVzdDoKICAgIGltYWdlOiBtYXBsYWluL2Ryb25lLWdjczpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIHNvdXJjZTogbGF0ZXN0CiAgICB0YXJnZXQ6IGhhcmJvci1jaS1waXBlbGluZS1zdG9yZS9sYXRlc3QKICAgIGFjbDoKICAgICAgLSBhbGxVc2VyczpSRUFERVIKICAgIGNhY2hlX2NvbnRyb2w6IHB1YmxpYyxtYXgtYWdlPTM2MDAKICAgIHdoZW46CiAgICAgIHJlcG86IHZtd2FyZS9oYXJib3IKICAgICAgZXZlbnQ6IFsgcHVzaCwgdGFnIF0KICAgICAgYnJhbmNoOiBbIG1hc3RlciwgcmVsZWFzZS0qLCByZWZzL3RhZ3MvKiBdCiAgICAgIHN0YXR1czogc3VjY2VzcwoKICB0cmlnZ2VyOgogICAgaW1hZ2U6IHBsdWdpbnMvZG93bnN0cmVhbQogICAgc2VydmVyOiBodHRwczovL2NpLnZjbmEuaW8KICAgIHRva2VuOiAke0RPV05TVFJFQU1fVE9LRU59CiAgICBmb3JrOiB0cnVlCiAgICByZXBvc2l0b3JpZXM6CiAgICAgICAtIHZtd2FyZS92aWMtcHJvZHVjdAogICAgd2hlbjoKICAgICAgcmVwbzogdm13YXJlL2hhcmJvcgogICAgICBldmVudDogWyBwdXNoLCB0YWcgXQogICAgICBicmFuY2g6IFsgbWFzdGVyLCByZWxlYXNlLSosIHJlZnMvdGFncy8qIF0KICAgICAgc3RhdHVzOiBzdWNjZXNzCg.LfFhLULxrTbdmiIzQU-tzCQ9WZRcUZrq4nX3eBJS6JQ
+eyJhbGciOiJIUzI1NiJ9.IyBIYXJib3IgZHJvbmUuCi0tLQp3b3Jrc3BhY2U6CiAgYmFzZTogL2Ryb25lCiAgcGF0aDogc3JjL2dpdGh1Yi5jb20vdm13YXJlL2hhcmJvcgoKcGlwZWxpbmU6CiAgY2xvbmU6CiAgICBpbWFnZTogcGx1Z2lucy9naXQKICAgIHRhZ3M6IHRydWUKICAgIHJlY3Vyc2l2ZTogZmFsc2UKCiAgaW50ZWdyYXRpb24tdGVzdC1vbi1wcjoKICAgIGltYWdlOiB2bXdhcmUvaGFyYm9yLWUyZS1lbmdpbmU6MS4zOAogICAgcHVsbDogdHJ1ZQogICAgcHJpdmlsZWdlZDogdHJ1ZQogICAgZW52aXJvbm1lbnQ6CiAgICAgIEJJTjogYmluCiAgICAgIEdPUEFUSDogL2Ryb25lCiAgICAgIFNIRUxMOiAvYmluL2Jhc2gKICAgICAgTE9HX1RFTVBfRElSOiBpbnN0YWxsLWxvZ3MKICAgICAgR0lUSFVCX0FVVE9NQVRJT05fQVBJX0tFWTogICR7R0lUSFVCX0FVVE9NQVRJT05fQVBJX0tFWX0KICAgICAgRFJPTkVfU0VSVkVSOiAgJHtEUk9ORV9TRVJWRVJ9CiAgICAgIERST05FX1RPS0VOOiAgJHtEUk9ORV9UT0tFTl9JTlRFfQogICAgICBIQVJCT1JfQURNSU46ICR7SEFSQk9SX0FETUlOfQogICAgICBIQVJCT1JfUEFTU1dPUkQ6ICR7SEFSQk9SX1BBU1NXT1JEfQogICAgICBHU19QUk9KRUNUX0lEOiAke0dTX1BST0pFQ1RfSUR9CiAgICAgIEdTX0NMSUVOVF9FTUFJTDogJHtHU19DTElFTlRfRU1BSUx9CiAgICAgIEdTX1BSSVZBVEVfS0VZOiAke0dTX1BSSVZBVEVfS0VZfQogICAgICBET01BSU46ICR7Q0lfRE9NQUlOfQogICAgICBNQUlMX1BXRDogJHtNQUlMX1BXRH0KICAgICAgTlBNX1VTRVJOQU1FOiAke05QTV9VU0VSTkFNRX0KICAgICAgTlBNX1BBU1NXT1JEOiAke05QTV9QQVNTV09SRH0KICAgIGNvbW1hbmRzOgogICAgICAtIHRlc3RzL2ludGVncmF0aW9uLnNoCiAgICB3aGVuOgogICAgICBzdGF0dXM6IHN1Y2Nlc3MKCiAgYnVuZGxlOgogICAgaW1hZ2U6IHZtd2FyZS9oYXJib3ItZTJlLWVuZ2luZToxLjM4CiAgICBwdWxsOiB0cnVlCiAgICBwcml2aWxlZ2VkOiB0cnVlCiAgICBlbnZpcm9ubWVudDoKICAgICAgQklOOiBiaW4KICAgICAgR09QQVRIOiAvZHJvbmUKICAgICAgU0hFTEw6IC9iaW4vYmFzaAogICAgICBCVUlMRF9OVU1CRVI6ICR7RFJPTkVfQlVJTERfTlVNQkVSfQogICAgY29tbWFuZHM6CiAgICAgIC0gZHUgLWtzIGhhcmJvci1vZmZsaW5lLWluc3RhbGxlci0qLnRneiB8IGF3ayAne3ByaW50ICQxIC8gMTAyNH0nIHwgeyByZWFkIHg7IGVjaG8gJHggTUI7IH0KICAgICAgLSBta2RpciAtcCBidW5kbGUKICAgICAgLSBta2RpciAtcCBwa3MtYnVuZGxlCiAgICAgIC0gZWNobyAkKGdpdCBkZXNjcmliZSAtLXRhZ3MpID4gcGtzLWJ1bmRsZS92ZXJzaW9uIAogICAgICAtIGNwIGhhcmJvci1vZmZsaW5lLWluc3RhbGxlci0qLnRneiBidW5kbGUKICAgICAgLSBpZiBbICR7RFJPTkVfQlJBTkNIfSA9ICJtYXN0ZXIiIF07IHRoZW4gY3AgaGFyYm9yLW9mZmxpbmUtaW5zdGFsbGVyLSoudGd6IHBrcy1idW5kbGUvaGFyYm9yLW9mZmxpbmUtaW5zdGFsbGVyLWxhdGVzdC1tYXN0ZXIudGd6OyBmaQogICAgICAtIGlmICggZWNobyAke0RST05FX0JSQU5DSH0gfCBncmVwICJwa3MqIiApOyB0aGVuIGNwIGhhcmJvci1vZmZsaW5lLWluc3RhbGxlci0qLnRneiBwa3MtYnVuZGxlL2hhcmJvci1vZmZsaW5lLWluc3RhbGxlci1sYXRlc3QtcGtzLnRnejsgZmkKICAgICAgLSBscyAtbGEgYnVuZGxlCiAgICAgIC0gbHMgLWxhIHBrcy1idW5kbGUKICAgIHdoZW46CiAgICAgIHJlcG86IHZtd2FyZS9oYXJib3IKICAgICAgZXZlbnQ6IFsgcHVzaCwgdGFnIF0KICAgICAgYnJhbmNoOiBbIG1hc3RlciwgcmVsZWFzZS0qLCBwa3MtKiwgcmVmcy90YWdzLyogXQogICAgICBzdGF0dXM6IHN1Y2Nlc3MKCiAgbm90aWZ5LXNsYWNrOgogICAgaW1hZ2U6IHBsdWdpbnMvc2xhY2sKICAgIHdlYmhvb2s6ICR7U0xBQ0tfVVJMfQogICAgdXNlcm5hbWU6IGRyb25lCiAgICB0ZW1wbGF0ZTogPgogICAgICBidWlsZCBodHRwczovL2NpLnZjbmEuaW8vdm13YXJlL2hhcmJvci97eyBidWlsZC5udW1iZXIgfX0gZmluaXNoZWQgd2l0aCBhIHt7IGJ1aWxkLnN0YXR1cyB9fSBzdGF0dXMuIFBsZWFzZSBmaW5kIGxvZ3MgYXQgaHR0cHM6Ly9zdG9yYWdlLmdvb2dsZWFwaXMuY29tL2hhcmJvci1jaS1sb2dzL2ludGVncmF0aW9uX2xvZ3Nfe3sgYnVpbGQubnVtYmVyIH19X3t7IGJ1aWxkLmNvbW1pdCB9fS50YXIuZ3oKICAgIHdoZW46CiAgICAgIHJlcG86IHZtd2FyZS9oYXJib3IKICAgICAgYnJhbmNoOiBbIG1hc3RlciwgcmVsZWFzZS0qLCByZWZzL3RhZ3MvKiBdCiAgICAgIHN0YXR1czogWyBmYWlsdXJlLCBzdWNjZXNzIF0KCiAgcHVibGlzaC1nY3MtYnVpbGRzOgogICAgaW1hZ2U6IG1hcGxhaW4vZHJvbmUtZ2NzOmxhdGVzdAogICAgcHVsbDogdHJ1ZQogICAgc291cmNlOiBidW5kbGUKICAgIHRhcmdldDogaGFyYm9yLWJ1aWxkcwogICAgYWNsOgogICAgICAtIGFsbFVzZXJzOlJFQURFUgogICAgY2FjaGVfY29udHJvbDogcHVibGljLG1heC1hZ2U9MzYwMAogICAgd2hlbjoKICAgICAgcmVwbzogdm13YXJlL2hhcmJvcgogICAgICBldmVudDogWyBwdXNoLCB0YWcgXQogICAgICBicmFuY2g6IFsgbWFzdGVyLCByZWxlYXNlLSogXQogICAgICBzdGF0dXM6IHN1Y2Nlc3MKCiAgcHVibGlzaC1nY3MtcmVsZWFzZXM6CiAgICBpbWFnZTogbWFwbGFpbi9kcm9uZS1nY3M6bGF0ZXN0CiAgICBwdWxsOiB0cnVlCiAgICBzb3VyY2U6IGJ1bmRsZQogICAgdGFyZ2V0OiBoYXJib3ItcmVsZWFzZXMKICAgIGFjbDoKICAgICAgLSBhbGxVc2VyczpSRUFERVIKICAgIGNhY2hlX2NvbnRyb2w6IHB1YmxpYyxtYXgtYWdlPTM2MDAKICAgIHdoZW46CiAgICAgIHJlcG86IHZtd2FyZS9oYXJib3IKICAgICAgZXZlbnQ6IFsgcHVzaCwgdGFnIF0KICAgICAgYnJhbmNoOiBbIHJlbGVhc2UtKiwgcmVmcy90YWdzLyogXQogICAgICBzdGF0dXM6IHN1Y2Nlc3MKCiAgcHVibGlzaC1nY3MtcGtzLWJ1aWxkczoKICAgIGltYWdlOiBtYXBsYWluL2Ryb25lLWdjczpsYXRlc3QKICAgIHB1bGw6IHRydWUKICAgIHNvdXJjZTogcGtzLWJ1bmRsZQogICAgdGFyZ2V0OiBoYXJib3ItY2ktcGlwZWxpbmUtc3RvcmUvbGF0ZXN0CiAgICBhY2w6CiAgICAgIC0gYWxsVXNlcnM6UkVBREVSCiAgICBjYWNoZV9jb250cm9sOiBwdWJsaWMsbWF4LWFnZT0zNjAwCiAgICB3aGVuOgogICAgICByZXBvOiB2bXdhcmUvaGFyYm9yCiAgICAgIGV2ZW50OiBbIHB1c2gsIHRhZyBdCiAgICAgIGJyYW5jaDogWyBtYXN0ZXIsIHBrcy0qLCByZWZzL3RhZ3MvKiBdCiAgICAgIHN0YXR1czogc3VjY2VzcwoKICB0cmlnZ2VyOgogICAgaW1hZ2U6IHBsdWdpbnMvZG93bnN0cmVhbQogICAgc2VydmVyOiBodHRwczovL2NpLnZjbmEuaW8KICAgIHRva2VuOiAke0RPV05TVFJFQU1fVE9LRU59CiAgICBmb3JrOiB0cnVlCiAgICByZXBvc2l0b3JpZXM6CiAgICAgICAtIHZtd2FyZS92aWMtcHJvZHVjdAogICAgd2hlbjoKICAgICAgcmVwbzogdm13YXJlL2hhcmJvcgogICAgICBldmVudDogWyBwdXNoLCB0YWcgXQogICAgICBicmFuY2g6IFsgbWFzdGVyLCByZWxlYXNlLSosIHJlZnMvdGFncy8qIF0KICAgICAgc3RhdHVzOiBzdWNjZXNzCg.TRzg0jvokGI8PBccqkW4foVBX_1uGzFUhTRaPzMFaeY

.travis.yml

@@ -98,6 +98,7 @@ script:
   - goveralls -coverprofile=profile.cov -service=travis-ci
   - docker-compose -f make/docker-compose.test.yml down
   - sudo rm -rf /data/config/*
+  - sudo rm -rf /data/database/*
   - ls /data/cert
   - sudo make install GOBUILDIMAGE=golang:1.7.3 COMPILETAG=compile_golangimage CLARITYIMAGE=vmware/harbor-clarity-ui-builder:1.2.7 NOTARYFLAG=true CLAIRFLAG=true
   - sleep 10  

LICENSE

@@ -88,7 +88,7 @@ limitations under the License.
 
 =======================================================================
 
-Admiral version 1.3.0 includes a number of components with separate 
+Harbor version 1.3.0 includes a number of components with separate 
 copyright notices and license terms.  This product does not necessarily 
 use all the open source components referred to below. Your use of the 
 source code for these components is subject to the terms and conditions 
@@ -103,7 +103,7 @@ further if you wish to review the copyright notice(s) and the full text
 of the license associated with each component.
 
 
-PART 1. VIRTUAL APPLIANCE: OPERATING SYSTEM
+PART 1. VIRTUAL APPLIANCE: OPERATING SYSTEM LAYER
 
 
 SECTION 1: BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES
@@ -355,7 +355,7 @@ APPENDIX. Standard License Files
    
 
 
-====================PART 1. VIRTUAL APPLIANCE: OPERATING SYSTEM ====================
+====================PART 1. VIRTUAL APPLIANCE: OPERATING SYSTEM LAYER ====================
 
 
 --------------- SECTION 1:  BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES ----------
@@ -25884,7 +25884,6 @@ THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 
 ===========================================================================
 
-
 To the extent any open source components are licensed under the GPL
 and/or LGPL, or other similar licenses that require the source code
 and/or modifications to source code to be made available, you may obtain
@@ -25896,4 +25895,4 @@ All such requests should clearly specify: OPEN SOURCE FILES REQUEST, Attention G
 mail a copy of the Source Files to you on a CD or equivalent physical medium. This offer to obtain a copy of the Source
 Files is valid for three years from the date you acquired this software product.
 
-[HARBOR130GAVS112117]
+[HARBOR130GAVS120817]

Makefile

@@ -230,13 +230,14 @@ PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(GITTAGVERSION).tgz \
 				  $(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \
 				  $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
-				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME)
+				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
+				  $(HARBORPKG)/ha
 PACKAGE_ONLINE_PARA=-zcvf harbor-online-installer-$(GITTAGVERSION).tgz \
 		          $(HARBORPKG)/common/templates $(HARBORPKG)/prepare \
 				  $(HARBORPKG)/LICENSE $(HARBORPKG)/NOTICE \
 				  $(HARBORPKG)/upgrade $(HARBORPKG)/harbor_1_1_0_template \
 				  $(HARBORPKG)/install.sh $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
-				  $(HARBORPKG)/harbor.cfg
+				  $(HARBORPKG)/harbor.cfg $(HARBORPKG)/ha
 DOCKERCOMPOSE_LIST=-f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
 
 ifeq ($(NOTARYFLAG), true)
@@ -327,7 +328,9 @@ build: build_$(BASEIMAGE)
 modify_composefile:
 	@echo "preparing docker-compose file..."
 	@cp $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
+	@cp $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSETPLFILENAME) $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
 	@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSEFILENAME)
+	@$(SEDCMD) -i 's/__version__/$(VERSIONTAG)/g' $(DOCKERCOMPOSEFILEPATH)/ha/$(DOCKERCOMPOSEFILENAME)
 
 modify_sourcefiles:
 	@echo "change mode of source files."
@@ -345,6 +348,8 @@ package_online: modify_composefile
 	@if [ -n "$(REGISTRYSERVER)" ] ; then \
 		$(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \
 		$(HARBORPKG)/docker-compose.yml ; \
+		$(SEDCMD) -i 's/image\: vmware/image\: $(REGISTRYSERVER)\/$(REGISTRYPROJECTNAME)/' \
+		$(HARBORPKG)/ha/docker-compose.yml ; \
 	fi
 	@cp LICENSE $(HARBORPKG)/LICENSE
 	@cp NOTICE $(HARBORPKG)/NOTICE
@@ -363,6 +368,7 @@ package_offline: compile build modify_sourcefiles modify_composefile
 	@cp NOTICE $(HARBORPKG)/NOTICE
 	@cp tools/migration/migration_cfg/upgrade $(HARBORPKG)/upgrade
 	@cp tools/migration/migration_cfg/harbor_1_1_0_template $(HARBORPKG)/harbor_1_1_0_template
+	@cp $(HARBORPKG)/common/db/registry.sql $(HARBORPKG)/ha/
 
 	@echo "pulling nginx and registry..."
 	@$(DOCKERPULL) vmware/registry:$(REGISTRYVERSION)

docs/user_guide.md

@@ -2,20 +2,23 @@
 ## Overview  
 This guide walks you through the fundamentals of using Harbor. You'll learn how to use Harbor to:  
 
-* Manage your projects.
+* [Manage your projects.](#managing-projects)
-* Manage members of a project.
+* [Manage members of a project.](#managing-members-of-a-project)
-* Replicate projects to a remote registry.
+* [Replicate projects to a remote registry.](#replicationg-images)
-* Search projects and repositories.
+* [Search projects and repositories.](#searching-projects-and-repositories)
-* Manage Harbor system if you are the system administrator:
+* [Manage Harbor system if you are the system administrator:](#administrator-options)
-  * Manage users.
+  * [Manage users.](#managing-user)
-  * Manage destinations.
+  * [Manage destinations.](#managing-endpoint)
-  * Manage replication policies.
+  * [Manage replication policies.](#managing-replication)
-  * Manage configuration.
+  * [Manage authentication.](#managing-authentication)
-* Pull and push images using Docker client.
+  * [Manage project creation.](#managing-project-creation)
-* Delete repositories and images.
+  * [Manage self-registration.](#managing-self-registration)
-* Content trust.  
+  * [Manage email settings.](#managing-email-settings)
-* Vulnerability scanning via Clair.
+* [Pull and push images using Docker client.](#pulling-and-pushing-images-using-docker-client)
-* Pull image from Harbor in Kubernetes.
+* [Delete repositories and images.](#deleting-repositories)
+* [Content trust.  ](#content-trust)
+* [Vulnerability scanning via Clair.](#vulnerability-scaning-via-clair)
+* [Pull image from Harbor in Kubernetes.](#pull-image-from-harbor-in-kubernetes)
 
 ## Role Based Access Control(RBAC)  
 

make/common/db/registry.sql

@@ -221,9 +221,11 @@ UNIQUE(namespace)
 );
 
 create table properties (
+ id int NOT NULL AUTO_INCREMENT,
  k varchar(64) NOT NULL,
  v varchar(128) NOT NULL,
- primary key (k)
+ PRIMARY KEY(id),
+ UNIQUE (k)
  );
 
 CREATE TABLE IF NOT EXISTS `alembic_version` (

make/common/db/registry_sqlite.sql

@@ -212,9 +212,10 @@ UNIQUE(namespace)
 );
 
 create table properties (
+ id INTEGER PRIMARY KEY,
  k varchar(64) NOT NULL,
  v varchar(128) NOT NULL,
- primary key (k)
+ UNIQUE(k)
  );
 
 create table alembic_version (

make/common/templates/registry/config_ha.yml

@@ -0,0 +1,45 @@
+version: 0.1
+log:
+  level: debug
+  fields:
+    service: registry
+storage:
+    cache:
+        layerinfo: redis
+    Place_holder_for_Storage_configureation
+    maintenance:
+        uploadpurging:
+            enabled: false
+    delete:
+        enabled: true
+redis:
+  addr: $redis_url
+  db: 0
+  dialtimeout: 10ms
+  readtimeout: 10ms
+  writetimeout: 10ms
+  pool:
+    maxidle: 16
+    maxactive: 64
+    idletimeout: 300s
+
+http:
+    addr: :5000
+    secret: placeholder
+    debug:
+        addr: localhost:5001
+auth:
+  token:
+    issuer: harbor-token-issuer
+    realm: $ui_url/service/token
+    rootcertbundle: /etc/registry/root.crt
+    service: harbor-registry
+
+notifications:
+  endpoints:
+      - name: harbor
+        disabled: false
+        url: http://ui:8080/service/notifications
+        timeout: 3000ms
+        threshold: 5
+        backoff: 1s

make/common/templates/ui/env

@@ -5,3 +5,4 @@ JOBSERVICE_SECRET=$jobservice_secret
 GODEBUG=netdns=cgo
 ADMINSERVER_URL=http://adminserver:8080
 UAA_CA_ROOT=/etc/ui/certificates/uaa_ca.pem
+_REDIS_URL=$redis_url

make/ha/docker-compose.tpl

@@ -0,0 +1,121 @@
+version: '2'
+services:
+  log:
+    image: vmware/harbor-log:__version__
+    container_name: harbor-log 
+    restart: always
+    volumes:
+      - /var/log/harbor/:/var/log/docker/:z
+      - ./common/config/log/:/etc/logrotate.d/:z
+    ports:
+      - 127.0.0.1:1514:10514
+    networks:
+      - harbor
+  registry:
+    image: vmware/registry:2.6.2-photon
+    container_name: registry
+    restart: always
+    volumes:
+      - /data/registry:/storage:z
+      - ./common/config/registry/:/etc/registry/:z
+    networks:
+      - harbor
+    environment:
+      - GODEBUG=netdns=cgo
+    command:
+      ["serve", "/etc/registry/config.yml"]
+    depends_on:
+      - log
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "registry"
+  adminserver:
+    image: vmware/harbor-adminserver:__version__
+    container_name: harbor-adminserver
+    env_file:
+      - ./common/config/adminserver/env
+    restart: always
+    volumes:
+      - /data/config/:/etc/adminserver/config/:z
+      - /data/secretkey:/etc/adminserver/key:z
+      - /data/:/data/:z
+    networks:
+      - harbor
+    depends_on:
+      - log
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "adminserver"
+  ui:
+    image: vmware/harbor-ui:__version__
+    container_name: harbor-ui
+    env_file:
+      - ./common/config/ui/env
+    restart: always
+    volumes:
+      - ./common/config/ui/app.conf:/etc/ui/app.conf:z
+      - ./common/config/ui/private_key.pem:/etc/ui/private_key.pem:z
+      - ./common/config/ui/certificates/:/etc/ui/certifates/
+      - /data/secretkey:/etc/ui/key:z
+      - /data/ca_download/:/etc/ui/ca/:z
+      - /data/psc/:/etc/ui/token/:z
+    networks:
+      - harbor
+    depends_on:
+      - log
+      - adminserver
+      - registry
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "ui"
+  jobservice:
+    image: vmware/harbor-jobservice:__version__
+    container_name: harbor-jobservice
+    env_file:
+      - ./common/config/jobservice/env
+    restart: always
+    volumes:
+      - /data/job_logs:/var/log/jobs:z
+      - ./common/config/jobservice/app.conf:/etc/jobservice/app.conf:z
+      - /data/secretkey:/etc/jobservice/key:z
+    networks:
+      - harbor
+    depends_on:
+      - ui
+      - adminserver
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "jobservice"
+  proxy:
+    image: vmware/nginx-photon:1.11.13
+    container_name: nginx
+    restart: always
+    volumes:
+      - ./common/config/nginx:/etc/nginx:z
+    networks:
+      - harbor
+    ports:
+      - 80:80
+      - 443:443
+      - 4443:4443
+    depends_on:
+      - registry
+      - ui
+      - log
+    logging:
+      driver: "syslog"
+      options:  
+        syslog-address: "tcp://127.0.0.1:1514"
+        tag: "proxy"
+networks:
+  harbor:
+    external: false
+

make/ha/sample/active_active/keepalived_active_active.conf

@@ -0,0 +1,80 @@
+global_defs {
+  router_id haborlb
+}
+vrrp_sync_groups VG1 {
+  group {
+    VI_1
+  }
+}
+#Please change to ens160 to the interface name on you loadbalancer hosts.
+vrrp_instance VI_1 {
+  interface ens160
+
+  track_interface {
+    ens160
+  }
+
+  state MASTER
+  virtual_router_id 51
+  priority 10
+
+  virtual_ipaddress {
+    VIP/32
+  }
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass d0cker
+  }
+
+}
+#Please change VIP, harbor_node1_ip, harbor_node2_ip to real ip address
+virtual_server VIP 80 {
+  delay_loop 15
+  lb_algo rr
+  lb_kind DR
+  protocol TCP
+  nat_mask 255.255.255.0
+  persistence_timeout 10
+
+  real_server harbor_node1_ip 80 {
+    weight 10
+    TCP_CHECK {
+     connect_timeout 3
+     connect_port 80
+    }
+  }
+
+  real_server harbor_node2_ip 80 {
+    weight 10
+    TCP_CHECK {
+      connect_timeout 3
+      connect_port 80
+    }
+  }
+}
+#Please uncomment the follow when harbor running under https
+#virtual_server VIP 443 {
+#  delay_loop 15
+#  lb_algo rr
+#  lb_kind DR
+#  protocol TCP
+#  nat_mask 255.255.255.0
+#  persistence_timeout 10
+#
+#  real_server harbor_node1_ip 443 {
+#    weight 10
+#    TCP_CHECK {
+#     connect_timeout 3
+#     connect_port 443
+#    }
+#  }
+#
+#  real_server harbor_node2_ip 443 {
+#    weight 10
+#    TCP_CHECK {
+#      connect_timeout 3
+#      connect_port 443
+#    }
+#  }
+#}

make/ha/sample/active_standby/check_harbor.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+http_code = `curl -s -o /dev/null -w "%{http_code}" 127.0.0.1`
+if [ $http_code == 200 ] || [ $http_code == 301 ] ; then
+    exit 0
+else
+    exit 1
+fi

make/ha/sample/active_standby/keepalived_active_standby.conf

@@ -0,0 +1,36 @@
+global_defs {
+  router_id haborlb
+}
+vrrp_script check_harbor {
+  script "/usr/local/bin/check_harbor.sh"
+  interval 15
+  fail 5
+  rise 2
+}
+vrrp_sync_groups VG1 {
+  group {
+    VI_1
+  }
+}
+#Please change to ens160 to the interface name on you loadbalancer hosts.
+vrrp_instance VI_1 {
+  interface ens160
+
+  track_interface {
+    ens160
+  }
+
+  state MASTER
+  virtual_router_id 51
+  priority 10
+
+  virtual_ipaddress {
+    VIP/32
+  }
+  advert_int 1
+  authentication {
+    auth_type PASS
+    auth_pass d0cker
+  }
+
+}

make/harbor.cfg

@@ -116,6 +116,8 @@ db_port = 3306
 
 #The user name of mysql database
 db_user = root
+#The redis server address
+redis_url =
 #************************END INITIAL PROPERTIES************************
 #The following attributes only need to be set when auth mode is uaa_auth
 uaa_endpoint = uaa.mydomain.org

make/install.sh

@@ -58,7 +58,8 @@ item=0
 with_notary=$false
 # clair is not enabled by default
 with_clair=$false
-
+# HA mode is not enabled by default
+harbor_ha=$false
 while [ $# -gt 0 ]; do
         case $1 in
             --help)
@@ -68,6 +69,8 @@ while [ $# -gt 0 ]; do
             with_notary=true;;
             --with-clair)
             with_clair=true;;
+            --ha)
+            harbor_ha=true;;
             *)
             note "$usage"
             exit 1;;
@@ -158,24 +161,28 @@ then
 	sed "s/^hostname = .*/hostname = $host/g" -i ./harbor.cfg
 fi
 prepare_para=
-if [ $with_notary ]
+if [ $with_notary ] && [ ! $harbor_ha ]
 then
 	prepare_para="${prepare_para} --with-notary"
 fi
-if [ $with_clair ]
+if [ $with_clair ] && [ ! $harbor_ha ]
 then
 	prepare_para="${prepare_para} --with-clair"
 fi
+if [ $harbor_ha ]
+then
+    prepare_para="${prepare_para} --ha"
+fi
 ./prepare $prepare_para
 echo ""
 
 h2 "[Step $item]: checking existing instance of Harbor ..."; let item+=1
 docker_compose_list='-f docker-compose.yml'
-if [ $with_notary ]
+if [ $with_notary ] && [ ! $harbor_ha ]
 then
 	docker_compose_list="${docker_compose_list} -f docker-compose.notary.yml"
 fi
-if [ $with_clair ]
+if [ $with_clair ] && [ ! $harbor_ha ]
 then
 	docker_compose_list="${docker_compose_list} -f docker-compose.clair.yml"
 fi
@@ -188,6 +195,11 @@ fi
 echo ""
 
 h2 "[Step $item]: starting Harbor ..."
+if [ $harbor_ha ]
+then
+    mv docker-compose.yml docker-compose.yml.bak 
+    cp ha/docker-compose.yml docker-compose.yml
+fi
 docker-compose $docker_compose_list up -d
 
 protocol=http

make/prepare

@@ -20,6 +20,27 @@ if sys.version_info[:3][0] == 3:
     import io as StringIO
 
 def validate(conf, args): 
+    if args.ha_mode:
+        db_host = rcp.get("configuration", "db_host")
+        if db_host == "mysql":
+            raise Exception("Error: In HA mode, db_host in harbor.cfg needs to point to an external DB address")
+        registry_config_path = os.path.join(templates_dir,"registry","config_ha.yml")
+        if check_storage_config(registry_config_path):
+            raise Exception("Error: In HA model shared storage configuration is required registry, refer HA installation guide for detail.")
+        redis_url = rcp.get("configuration", "redis_url")
+        if redis_url is None or len(redis_url) < 1:
+            raise Exception("Error: In HA mode redis is required redis_url need to point to an redis cluster")
+        if args.notary_mode or args.clair_mode:
+            raise Exception("Error: HA mode doesn't support clair and notary currently")
+        cert_path = rcp.get("configuration", "ssl_cert")
+        cert_key_path = rcp.get("configuration", "ssl_cert_key")
+        shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path))
+        shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path))
+        if os.path.isfile(shared_cert_key):
+            shutil.copy2(shared_cert_key, cert_key_path)
+        if os.path.isfile(shared_cert_path):
+            shutil.copy2(shared_cert_path, cert_path)
+
     protocol = rcp.get("configuration", "ui_url_protocol")
     if protocol != "https" and args.notary_mode:
         raise Exception("Error: the protocol must be https when Harbor is deployed with Notary")
@@ -39,6 +60,63 @@ def validate(conf, args):
     if project_creation != "everyone" and project_creation != "adminonly":
         raise Exception("Error invalid value for project_creation_restriction: %s" % project_creation)
 
+def prepare_ha(conf, args):
+    #files under ha folder will have high prority
+    protocol = rcp.get("configuration", "ui_url_protocol")
+    if protocol == "https":
+        #copy nginx certificate
+        cert_path = rcp.get("configuration", "ssl_cert")
+        cert_key_path = rcp.get("configuration", "ssl_cert_key")
+        shared_cert_key = os.path.join(base_dir, "ha", os.path.basename(cert_key_path))
+        shared_cert_path = os.path.join(base_dir, "ha", os.path.basename(cert_path))
+        if os.path.isfile(shared_cert_key):
+            shutil.copy2(shared_cert_key, cert_key_path)
+        else: 
+            if os.path.isfile(cert_key_path):
+                shutil.copy2(cert_key_path, shared_cert_key)
+        if os.path.isfile(shared_cert_path):
+            shutil.copy2(shared_cert_path, cert_path)
+        else:
+            if os.path.isfile(cert_path):
+                shutil.copy2(cert_path, shared_cert_path)
+        #check if ca exsit
+        cert_ca_path = "/data/ca_download/ca.crt"
+        shared_ca_path = os.path.join(base_dir, "ha", os.path.basename(cert_ca_path))
+        if os.path.isfile(shared_ca_path):
+            shutil.copy2(shared_ca_path, cert_ca_path)
+        else:
+            if os.path.isfile(cert_ca_path):
+                shutil.copy2(cert_ca_path, shared_ca_path)
+    #check root.crt and priviate_key.pem
+    private_key_pem = os.path.join(config_dir, "ui", "private_key.pem")
+    root_crt = os.path.join(config_dir, "registry", "root.crt")
+    shared_private_key_pem = os.path.join(base_dir, "ha", "private_key.pem")
+    shared_root_crt = os.path.join(base_dir, "ha", "root.crt")
+    if os.path.isfile(shared_private_key_pem):
+        shutil.copy2(shared_private_key_pem, private_key_pem)
+    else:
+        if os.path.isfile(private_key_pem):
+            shutil.copy2(private_key_pem, shared_private_key_pem)
+    if os.path.isfile(shared_root_crt):
+        shutil.copy2(shared_root_crt, root_crt)
+    else:
+        if os.path.isfile(root_crt):
+            shutil.copy2(root_crt, shared_root_crt)
+    #secretkey
+    shared_secret_key = os.path.join(base_dir, "ha", "secretkey")
+    secretkey_path = rcp.get("configuration", "secretkey_path") 
+    secret_key = os.path.join(secretkey_path, "secretkey")
+    if os.path.isfile(shared_secret_key):
+        shutil.copy2(shared_secret_key, secret_key)
+    else:
+        if os.path.isfile(secret_key):
+            shutil.copy2(secret_key, shared_secret_key)
+
+def check_storage_config(path):
+    if 'Place_holder_for_Storage_configureation' in open(path).read():
+        return True
+    return False
+
 def get_secret_key(path):
     secret_key = _get_secret(path, "secretkey") 
     if len(secret_key) != 16:
@@ -57,12 +135,12 @@ def _get_secret(folder, filename, length=16):
             print("loaded secret from file: %s" % key_file)
         return key
     if not os.path.isdir(folder):
-        os.makedirs(folder, mode=0600)
+        os.makedirs(folder, mode=0o600)
     key = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(length))  
     with open(key_file, 'w') as f:
         f.write(key)
         print("Generated and saved secret to file: %s" % key_file)
-    os.chmod(key_file, 0600)
+    os.chmod(key_file, 0o600)
     return key
 
 def prep_conf_dir(root, name):
@@ -96,6 +174,7 @@ parser = argparse.ArgumentParser()
 parser.add_argument('--conf', dest='cfgfile', default=base_dir+'/harbor.cfg',type=str,help="the path of Harbor configuration file")
 parser.add_argument('--with-notary', dest='notary_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with notary")
 parser.add_argument('--with-clair', dest='clair_mode', default=False, action='store_true', help="the Harbor instance is to be deployed with clair")
+parser.add_argument('--ha', dest='ha_mode', default=False, action='store_true', help="the Harbor instance is to be deployed in HA mode")
 args = parser.parse_args()
 
 delfile(config_dir)
@@ -106,7 +185,6 @@ conf.write(open(args.cfgfile).read())
 conf.seek(0, os.SEEK_SET)
 rcp = ConfigParser.RawConfigParser()
 rcp.readfp(conf)
-
 validate(rcp, args)
 
 hostname = rcp.get("configuration", "hostname")
@@ -141,8 +219,8 @@ ldap_scope = rcp.get("configuration", "ldap_scope")
 ldap_timeout = rcp.get("configuration", "ldap_timeout")
 db_password = rcp.get("configuration", "db_password")
 db_host = rcp.get("configuration", "db_host")
-db_port = rcp.get("configuration", "db_port")
 db_user = rcp.get("configuration", "db_user")
+db_port = rcp.get("configuration", "db_port")
 self_registration = rcp.get("configuration", "self_registration")
 if protocol == "https":
     cert_path = rcp.get("configuration", "ssl_cert")
@@ -161,9 +239,15 @@ uaa_endpoint = rcp.get("configuration", "uaa_endpoint")
 uaa_clientid = rcp.get("configuration", "uaa_clientid")
 uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret")
 uaa_ca_root = rcp.get("configuration", "uaa_ca_root")
+
 secret_key = get_secret_key(secretkey_path)
 log_rotate_count = rcp.get("configuration", "log_rotate_count")
 log_rotate_size = rcp.get("configuration", "log_rotate_size")
+
+if rcp.has_option("configuration", "redis_url"):
+    redis_url = rcp.get("configuration", "redis_url")
+else:
+    redis_url = ""
 ########
 
 ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))  
@@ -229,8 +313,8 @@ render(os.path.join(templates_dir, "adminserver", "env"),
         ldap_timeout=ldap_timeout,
         db_password=db_password,
         db_host=db_host,
-        db_port=db_port,
         db_user=db_user,
+        db_port=db_port,
         email_host=email_host,
         email_port=email_port,
         email_usr=email_usr,
@@ -258,8 +342,15 @@ render(os.path.join(templates_dir, "ui", "env"),
         ui_conf_env, 
         ui_secret=ui_secret,
         jobservice_secret=jobservice_secret,
+        redis_url = redis_url
         )
-
+if args.ha_mode:
+    render(os.path.join(templates_dir, "registry",
+		"config_ha.yml"),
+                registry_conf,
+                ui_url=ui_url,
+                redis_url=redis_url)
+else:
     render(os.path.join(templates_dir, "registry",
                 "config.yml"),
                 registry_conf,
@@ -338,8 +429,8 @@ if customize_crt == 'on' and openssl_installed():
     private_key_pem = os.path.join(config_dir, "ui", "private_key.pem")
     root_crt = os.path.join(config_dir, "registry", "root.crt")
     create_root_cert(empty_subj, key_path=private_key_pem, cert_path=root_crt)
-    os.chmod(private_key_pem, 0600)
+    os.chmod(private_key_pem, 0o600)
-    os.chmod(root_crt, 0600)
+    os.chmod(root_crt, 0o600)
 else:
     print("Copied configuration file: %s" % ui_config_dir + "private_key.pem")
     shutil.copyfile(os.path.join(templates_dir, "ui", "private_key.pem"), os.path.join(ui_config_dir, "private_key.pem"))
@@ -367,9 +458,9 @@ if args.notary_mode:
             create_root_cert(ca_subj, key_path=signer_ca_key, cert_path=signer_ca_cert)
             create_cert(cert_subj, signer_ca_key, signer_ca_cert, key_path=signer_key_path, cert_path=signer_cert_path)
             print("Copying certs for notary signer")
-	    os.chmod(signer_cert_path, 0600)
+            os.chmod(signer_cert_path, 0o600)
-	    os.chmod(signer_key_path, 0600)
+            os.chmod(signer_key_path, 0o600)
-	    os.chmod(signer_ca_cert, 0600)
+            os.chmod(signer_ca_cert, 0o600)
             shutil.copy2(signer_cert_path, notary_config_dir)
             shutil.copy2(signer_key_path, notary_config_dir)
             shutil.copy2(signer_ca_cert, notary_config_dir)
@@ -413,6 +504,9 @@ if args.clair_mode:
     clair_conf = os.path.join(clair_config_dir, "config.yaml")
     render(os.path.join(clair_temp_dir, "config.yaml"), clair_conf, password = pg_password)
 
+if args.ha_mode:
+    prepare_ha(rcp, args)
+
 FNULL.close()
 print("The configuration files are ready, please use docker-compose to start the service.")
 

src/adminserver/systemcfg/store/database/driver_db.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package database
+
+import (
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/adminserver/systemcfg/store"
+	"github.com/vmware/harbor/src/common"
+	"fmt"
+	"strconv"
+)
+
+const (
+	name = "database"
+	)
+var(
+	numKeys = map[string]bool{
+		common.EmailPort:true,
+		common.LDAPScope:true,
+		common.LDAPTimeout:true,
+		common.TokenExpiration:true,
+		common.MySQLPort:true,
+		common.MaxJobWorkers:true,
+		common.CfgExpiration:true,
+		}
+	boolKeys = map[string]bool{
+		common.WithClair:true,
+		common.WithNotary:true,
+		common.SelfRegistration:true,
+		common.EmailSSL:true,
+		common.EmailInsecure:true,
+		common.LDAPVerifyCert:true,
+	}
+	)
+type cfgStore struct {
+	name string
+}
+
+// Name The name of the driver
+func (c *cfgStore) Name() string {
+	return name
+}
+
+// NewCfgStore New a cfg store for database driver
+func NewCfgStore()  (store.Driver, error){
+	return &cfgStore{
+		name: name,
+	}, nil
+}
+// Read configuration from database
+func (c *cfgStore) Read() (map[string]interface{}, error) {
+	configEntries,error := dao.GetConfigEntries()
+	if error != nil {
+		return nil, error
+	}
+	return WrapperConfig(configEntries)
+}
+
+// WrapperConfig Wrapper the configuration
+func WrapperConfig (configEntries []*models.ConfigEntry) (map[string]interface{}, error) {
+	config := make(map[string]interface{})
+	for _,entry := range configEntries{
+		if numKeys[entry.Key]{
+			strvalue, err := strconv.Atoi(entry.Value)
+			if err != nil {
+				return nil, err
+			}
+			config[entry.Key] = float64(strvalue)
+		}else if boolKeys[entry.Key] {
+			strvalue, err := strconv.ParseBool(entry.Value)
+			if err != nil {
+				return nil, err
+			}
+			config[entry.Key]=strvalue
+		}else{
+			config[entry.Key] = entry.Value
+		}
+
+	}
+	return config, nil
+}
+// Write save configuration to database
+func (c *cfgStore) Write(config map[string]interface{}) error {
+	configEntries ,_:= TranslateConfig(config)
+	return dao.SaveConfigEntries(configEntries)
+}
+
+// TranslateConfig Translate configuration from int, bool, float64 to string
+func TranslateConfig(config map[string]interface{}) ([]models.ConfigEntry,error) {
+	var configEntries []models.ConfigEntry
+	for k, v := range config {
+		var entry = new(models.ConfigEntry)
+		entry.Key = k
+		switch v.(type) {
+		case string:
+			entry.Value=v.(string)
+		case int:
+			entry.Value=strconv.Itoa(v.(int))
+		case bool:
+			entry.Value=strconv.FormatBool(v.(bool))
+		case float64:
+			entry.Value=strconv.Itoa(int(v.(float64)))
+		default:
+			return nil, fmt.Errorf("unknown type %v", v)
+		}
+		configEntries = append(configEntries,*entry)
+	}
+	return configEntries,nil
+}

src/adminserver/systemcfg/store/database/driver_db_test.go

@@ -0,0 +1,74 @@
+package database
+
+import (
+	"testing"
+    "github.com/stretchr/testify/assert"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common"
+)
+
+func TestCfgStore_Name(t *testing.T) {
+	driver,err := NewCfgStore()
+	if err != nil {
+		t.Fatalf("Failed to create db configuration store %v", err)
+	}
+	assert.Equal(t, name, driver.Name())
+}
+
+func TestWrapperConfig(t *testing.T) {
+	cfg:=[]*models.ConfigEntry{
+		{
+			Key:common.CfgExpiration,
+			Value:"500",
+		},
+		{
+			Key:common.WithNotary,
+			Value:"true",
+		},
+		{
+			Key:common.MySQLHost,
+			Value:"192.168.1.210",
+		},
+	}
+	result,err := WrapperConfig(cfg)
+	if err != nil {
+		t.Fatalf("Failed to wrapper config %v", err)
+	}
+	withNotary,_ := result[common.WithNotary].(bool)
+	assert.Equal(t,true, withNotary)
+
+	mysqlhost, ok := result[common.MySQLHost].(string)
+	assert.True(t, ok)
+	assert.Equal(t, "192.168.1.210", mysqlhost)
+
+	expiration, ok := result[common.CfgExpiration].(float64)
+
+	assert.True(t, ok)
+	assert.Equal(t, float64(500), expiration)
+}
+
+func TestTranslateConfig(t *testing.T) {
+	config := map[string]interface{}{}
+	config[common.MySQLHost]="192.168.1.210"
+
+	entries,err := TranslateConfig(config)
+	if err != nil {
+		t.Fatalf("Failed to translate configuration %v", err)
+	}
+	assert.Equal(t, "192.168.1.210",entries[0].Value)
+	config =make(map[string]interface{})
+	config[common.WithNotary]=true
+	entries,err = TranslateConfig(config)
+	if err != nil {
+		t.Fatalf("Failed to translate configuration %v", err)
+	}
+	assert.Equal(t, "true", entries[0].Value)
+
+	config =make(map[string]interface{})
+	config[common.CfgExpiration]=float64(500)
+	entries,err = TranslateConfig(config)
+	if err != nil {
+		t.Fatalf("Failed to translate configuration %v", err)
+	}
+	assert.Equal(t, "500", entries[0].Value)
+}

src/adminserver/systemcfg/systemcfg.go

@@ -23,10 +23,13 @@ import (
 	enpt "github.com/vmware/harbor/src/adminserver/systemcfg/encrypt"
 	"github.com/vmware/harbor/src/adminserver/systemcfg/store"
 	"github.com/vmware/harbor/src/adminserver/systemcfg/store/encrypt"
-	"github.com/vmware/harbor/src/adminserver/systemcfg/store/json"
 	"github.com/vmware/harbor/src/common"
 	comcfg "github.com/vmware/harbor/src/common/config"
 	"github.com/vmware/harbor/src/common/utils/log"
+	"github.com/vmware/harbor/src/adminserver/systemcfg/store/database"
+	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/dao"
+	"github.com/vmware/harbor/src/adminserver/systemcfg/store/json"
 )
 
 const (
@@ -215,17 +218,61 @@ func Init() (err error) {
 }
 
 func initCfgStore() (err error) {
+
+	drivertype := os.Getenv("CFG_DRIVER")
+	if len(drivertype) == 0 {
+		drivertype = common.CfgDriverDB
+	}
 	path := os.Getenv("JSON_CFG_STORE_PATH")
 	if len(path) == 0 {
 		path = defaultJSONCfgStorePath
 	}
 	log.Infof("the path of json configuration storage: %s", path)
 
+	if drivertype == common.CfgDriverDB {
+		//init database
+		cfgs := map[string]interface{}{}
+		if err = LoadFromEnv(cfgs, true); err != nil {
+			return err
+		}
+		cfgdb := GetDatabaseFromCfg(cfgs)
+		if err = dao.InitDatabase(cfgdb); err != nil {
+			return err
+		}
+		CfgStore, err = database.NewCfgStore()
+		if err != nil {
+			return err
+		}
+		//migration check: if no data in the db , then will try to load from path
+		m, err := CfgStore.Read()
+		if err != nil {
+			return err
+		}
+		if m == nil || len(m) == 0 {
+			if _, err := os.Stat(path); err == nil {
+				jsondriver, err := json.NewCfgStore(path)
+				if err != nil {
+					log.Errorf("Failed to migrate configuration from %s", path)
+					return err
+				}
+				jsonconfig, err := jsondriver.Read()
+				if err != nil {
+					log.Errorf("Failed to read old configuration from %s", path)
+					return err
+				}
+				err = CfgStore.Write(jsonconfig)
+				if err != nil {
+					log.Error("Failed to update old configuration to dattabase")
+					return err
+				}
+			}
+		}
+	} else {
 		CfgStore, err = json.NewCfgStore(path)
 		if err != nil {
-		return
+			return err
+		}
 	}
-
 	kp := os.Getenv("KEY_PATH")
 	if len(kp) == 0 {
 		kp = defaultKeyPath
@@ -278,3 +325,20 @@ func LoadFromEnv(cfgs map[string]interface{}, all bool) error {
 
 	return nil
 }
+
+// GetDatabaseFromCfg Create database object from config
+func GetDatabaseFromCfg(cfg map[string]interface{}) (*models.Database){
+	database := &models.Database{}
+	database.Type = cfg[common.DatabaseType].(string)
+	mysql := &models.MySQL{}
+	mysql.Host = cfg[common.MySQLHost].(string)
+	mysql.Port = int(cfg[common.MySQLPort].(int))
+	mysql.Username = cfg[common.MySQLUsername].(string)
+	mysql.Password = cfg[common.MySQLPassword].(string)
+	mysql.Database = cfg[common.MySQLDatabase].(string)
+	database.MySQL = mysql
+	sqlite := &models.SQLite{}
+	sqlite.File = cfg[common.SQLiteFile].(string)
+	database.SQLite = sqlite
+	return database
+}

src/adminserver/systemcfg/systemcfg_test.go

@@ -62,6 +62,9 @@ func TestParseStringToBool(t *testing.T) {
 func TestInitCfgStore(t *testing.T) {
 	os.Clearenv()
 	path := "/tmp/config.json"
+	if err := os.Setenv("CFG_DRIVER", "json"); err != nil {
+		t.Fatalf("failed to set env: %v", err)
+	}
 	if err := os.Setenv("JSON_CFG_STORE_PATH", path); err != nil {
 		t.Fatalf("failed to set env: %v", err)
 	}
@@ -122,3 +125,19 @@ func TestLoadFromEnv(t *testing.T) {
 	assert.Equal(t, "ldap_url", cfgs[common.LDAPURL])
 	assert.Equal(t, true, cfgs[common.LDAPVerifyCert])
 }
+
+func TestGetDatabaseFromCfg(t *testing.T) {
+	cfg  :=map[string]interface{} {
+		common.DatabaseType:"mysql",
+		common.MySQLDatabase:"registry",
+		common.MySQLHost:"127.0.0.1",
+		common.MySQLPort:3306,
+		common.MySQLPassword:"1234",
+		common.MySQLUsername:"root",
+		common.SQLiteFile:"/tmp/sqlite.db",
+	}
+
+	database := GetDatabaseFromCfg(cfg)
+
+	assert.Equal(t,"mysql",database.Type)
+}

src/common/api/base.go

@@ -121,7 +121,8 @@ func (b *BaseAPI) RenderError(code int, text string) {
 func (b *BaseAPI) DecodeJSONReq(v interface{}) {
 	err := json.Unmarshal(b.Ctx.Input.CopyBody(1<<32), v)
 	if err != nil {
-		log.Errorf("Error while decoding the json request, error: %v", err)
+		log.Errorf("Error while decoding the json request, error: %v, %v",
+			err, string(b.Ctx.Input.CopyBody(1<<32)[:]))
 		b.CustomAbort(http.StatusBadRequest, "Invalid json request")
 	}
 }

src/common/const.go

@@ -73,6 +73,7 @@ const (
 	UAAEndpoint                = "uaa_endpoint"
 	UAAClientID                = "uaa_client_id"
 	UAAClientSecret            = "uaa_client_secret"
-
 	DefaultClairEndpoint = "http://clair:6060"
+	CfgDriverDB                = "db"
+	CfgDriverJSON              = "json"
 )

src/common/dao/config.go

@@ -29,3 +29,42 @@ func AuthModeCanBeModified() (bool, error) {
 	// admin and anonymous
 	return c == 2, nil
 }
+
+// GetConfigEntries Get configuration from database
+func GetConfigEntries() ([]*models.ConfigEntry, error) {
+	o := GetOrmer()
+	var p []*models.ConfigEntry
+	sql:="select * from properties"
+	n,err := o.Raw(sql,[]interface{}{}).QueryRows(&p)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if n == 0 {
+		return nil, nil
+	}
+	return p,nil
+}
+
+// SaveConfigEntries Save configuration to database.
+func SaveConfigEntries(entries []models.ConfigEntry) error{
+	o := GetOrmer()
+	tempEntry:=models.ConfigEntry{}
+	for _, entry := range entries{
+		tempEntry.Key = entry.Key
+		tempEntry.Value = entry.Value
+		created, _, error := o.ReadOrCreate(&tempEntry,"k")
+		if error != nil {
+			return error
+		}
+		if !created {
+			entry.ID = tempEntry.ID
+			_ ,err := o.Update(&entry,"v")
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

src/common/dao/dao_test.go

@@ -20,7 +20,7 @@ import (
 	"time"
 
 	"github.com/astaxie/beego/orm"
-	//"github.com/vmware/harbor/src/common/config"
+	"github.com/vmware/harbor/src/common"
 	"github.com/stretchr/testify/assert"
 	"github.com/vmware/harbor/src/common/models"
 	"github.com/vmware/harbor/src/common/utils"
@@ -1631,3 +1631,103 @@ func TestGetScanJobsByStatus(t *testing.T) {
 	assert.Equal(1, len(r2))
 	assert.Equal(sj1.Repository, r2[0].Repository)
 }
+
+
+func TestSaveConfigEntries(t *testing.T)  {
+	configEntries :=[]models.ConfigEntry{
+		{
+			Key:"teststringkey",
+			Value:"192.168.111.211",
+		},
+		{
+			Key:"testboolkey",
+			Value:"true",
+		},
+		{
+			Key:"testnumberkey",
+			Value:"5",
+		},
+		{
+			Key:common.CfgDriverDB,
+			Value:"db",
+		},
+	}
+	err := SaveConfigEntries(configEntries)
+	if err != nil {
+		t.Fatalf("failed to save configuration to database %v", err)
+	}
+	readEntries, err:=GetConfigEntries()
+	if err !=nil {
+		t.Fatalf("Failed to get configuration from database %v", err)
+	}
+	findItem:=0
+	for _,entry:= range readEntries{
+		switch entry.Key {
+		case "teststringkey":
+			if "192.168.111.211" == entry.Value {
+				findItem++
+			}
+		case "testnumberkey":
+			if "5" == entry.Value {
+				findItem++
+			}
+		case "testboolkey":
+			if "true" == entry.Value {
+				findItem++
+			}
+		default:
+		}
+	}
+	if findItem !=3 {
+		t.Fatalf("Should update 3 configuration but only update %d", findItem)
+	}
+
+	configEntries =[]models.ConfigEntry{
+		{
+			Key:"teststringkey",
+			Value:"192.168.111.215",
+		},
+		{
+			Key:"testboolkey",
+			Value:"false",
+		},
+		{
+			Key:"testnumberkey",
+			Value:"7",
+		},
+		{
+			Key:common.CfgDriverDB,
+			Value:"db",
+		},
+	}
+	err = SaveConfigEntries(configEntries)
+	if err != nil {
+		t.Fatalf("failed to save configuration to database %v", err)
+	}
+	readEntries, err=GetConfigEntries()
+	if err !=nil {
+		t.Fatalf("Failed to get configuration from database %v", err)
+	}
+	findItem=0
+	for _,entry:= range readEntries{
+		switch entry.Key {
+		case "teststringkey":
+			if "192.168.111.215" == entry.Value {
+				findItem++
+			}
+		case "testnumberkey":
+			if "7" == entry.Value {
+				findItem++
+			}
+		case "testboolkey":
+			if "false" == entry.Value {
+				findItem++
+			}
+		default:
+		}
+	}
+	if findItem !=3 {
+		t.Fatalf("Should update 3 configuration but only update %d", findItem)
+	}
+
+}

src/common/models/base.go

@@ -30,5 +30,6 @@ func init() {
 		new(RepoRecord),
 		new(ImgScanOverview),
 		new(ClairVulnTimestamp),
-		new(ProjectMetadata))
+		new(ProjectMetadata),
+	    new(ConfigEntry))
 }

src/common/models/config.go

@@ -98,3 +98,14 @@ type SystemCfg struct {
 	CfgExpiration              int             `json:"cfg_expiration"`
 }
 */
+
+// ConfigEntry ...
+type ConfigEntry struct {
+	ID   int64    `orm:"pk;auto;column(id)" json:"-"`
+	Key string `orm:"column(k)" json:"k"`
+	Value string `orm:"column(v)" json:"v"`
+}
+// TableName ...
+func (ce *ConfigEntry)TableName() string {
+	return "properties"
+}

src/ui_ng/lib/src/create-edit-endpoint/create-edit-endpoint.component.ts

@@ -262,10 +262,11 @@ export class CreateEditEndpointComponent implements AfterViewChecked, OnDestroy
             delete payload[prop];
         }
         let changes: {[key: string]: any} = this.getChanges();
-        let changekeys: {[key: string]: any} = Object.keys(this.getChanges());
         if (isEmptyObject(changes)) {
             return;
         }
+        let changekeys: {[key: string]: any} = Object.keys(changes);
+
         changekeys.forEach((key: string) => {
             payload[key] = changes[key];
         });

src/ui_ng/lib/src/create-edit-rule/create-edit-rule.component.html.ts

@@ -74,7 +74,7 @@ export const CREATE_EDIT_RULE_TEMPLATE: string = `
         </div>
         <div class="form-group">
           <label for="destination_insecure" class="col-md-4 form-group-label-override">{{'CONFIG.VERIFY_REMOTE_CERT' | translate }}</label>
-          <clr-checkbox #insecure  class="col-md-8" name="insecure" id="destination_insecure" [clrDisabled]="testOngoing"  [clrChecked]="!createEditRule.insecure"  [clrDisabled]="readonly || !isCreateEndpoint" (clrCheckedChange)="setInsecureValue($event)">
+          <clr-checkbox #insecure  class="col-md-8" name="insecure" id="destination_insecure" [clrChecked]="!createEditRule.insecure"  [clrDisabled]="readonly || !isCreateEndpoint || testOngoing" (clrCheckedChange)="setInsecureValue($event)">
              <a href="javascript:void(0)" role="tooltip" aria-haspopup="true" class="tooltip tooltip-top-right" style="top:-7px;">
                     <clr-icon shape="info-circle" class="info-tips-icon" size="24"></clr-icon>
                     <span class="tooltip-content">{{'CONFIG.TOOLTIP.VERIFY_REMOTE_CERT' | translate}}</span>

src/ui_ng/lib/src/create-edit-rule/create-edit-rule.component.ts

@@ -424,6 +424,9 @@ export class CreateEditRuleComponent implements AfterViewChecked {
       pingTarget.password = this.createEditRule.password;
       pingTarget.insecure = this.createEditRule.insecure;
     } else {
+      for (let prop in pingTarget) {
+        delete pingTarget[prop];
+      }
       pingTarget.id = this.createEditRule.endpointId;
     }
     toPromise<Endpoint>(this.endpointService

src/ui_ng/package.json

@@ -31,7 +31,7 @@
         "clarity-icons": "^0.9.8",
         "clarity-ui": "^0.9.8",
         "core-js": "^2.4.1",
-        "harbor-ui": "0.5.24",
+        "harbor-ui": "0.5.27",
         "intl": "^1.2.5",
         "mutationobserver-shim": "^0.3.2",
         "ngx-cookie": "^1.0.0",

tests/apitests/api-testing/client/docker_client.go

@@ -0,0 +1,97 @@
+package client
+
+import "os/exec"
+import "strings"
+import "errors"
+import "bufio"
+import "fmt"
+
+//DockerClient : Run docker commands
+type DockerClient struct{}
+
+//Status : Check if docker daemon is there
+func (dc *DockerClient) Status() error {
+	cmdName := "docker"
+	args := []string{"info"}
+
+	return dc.runCommand(cmdName, args)
+}
+
+//Pull : Pull image
+func (dc *DockerClient) Pull(image string) error {
+	if len(strings.TrimSpace(image)) == 0 {
+		return errors.New("Empty image")
+	}
+
+	cmdName := "docker"
+	args := []string{"pull", image}
+
+	return dc.runCommandWithOutput(cmdName, args)
+}
+
+//Tag :Tag image
+func (dc *DockerClient) Tag(source, target string) error {
+	if len(strings.TrimSpace(source)) == 0 ||
+		len(strings.TrimSpace(target)) == 0 {
+		return errors.New("Empty images")
+	}
+
+	cmdName := "docker"
+	args := []string{"tag", source, target}
+
+	return dc.runCommandWithOutput(cmdName, args)
+}
+
+//Push : push image
+func (dc *DockerClient) Push(image string) error {
+	if len(strings.TrimSpace(image)) == 0 {
+		return errors.New("Empty image")
+	}
+
+	cmdName := "docker"
+	args := []string{"push", image}
+
+	return dc.runCommandWithOutput(cmdName, args)
+}
+
+//Login : Login docker
+func (dc *DockerClient) Login(userName, password string, uri string) error {
+	if len(strings.TrimSpace(userName)) == 0 ||
+		len(strings.TrimSpace(password)) == 0 {
+		return errors.New("Invlaid credential")
+	}
+
+	cmdName := "docker"
+	args := []string{"login", "-u", userName, "-p", password, uri}
+
+	return dc.runCommandWithOutput(cmdName, args)
+}
+
+func (dc *DockerClient) runCommand(cmdName string, args []string) error {
+	return exec.Command(cmdName, args...).Run()
+}
+
+func (dc *DockerClient) runCommandWithOutput(cmdName string, args []string) error {
+	cmd := exec.Command(cmdName, args...)
+	cmdReader, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	scanner := bufio.NewScanner(cmdReader)
+	go func() {
+		for scanner.Scan() {
+			fmt.Printf("%s out | %s\n", cmdName, scanner.Text())
+		}
+	}()
+
+	if err = cmd.Start(); err != nil {
+		return err
+	}
+
+	if err = cmd.Wait(); err != nil {
+		return err
+	}
+
+	return nil
+}

tests/apitests/api-testing/client/harbor_api_client.go

@@ -0,0 +1,170 @@
+package client
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+const (
+	httpHeaderJSON        = "application/json"
+	httpHeaderContentType = "Content-Type"
+	httpHeaderAccept      = "Accept"
+)
+
+//APIClientConfig : Keep config options for APIClient
+type APIClientConfig struct {
+	Username string
+	Password string
+	CaFile   string
+	CertFile string
+	KeyFile  string
+}
+
+//APIClient provided the http client for trigger http requests
+type APIClient struct {
+	//http client
+	client *http.Client
+
+	//Configuration
+	config APIClientConfig
+}
+
+//NewAPIClient is constructor of APIClient
+func NewAPIClient(config APIClientConfig) (*APIClient, error) {
+	//Load client cert
+	cert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	//Add ca
+	caCert, err := ioutil.ReadFile(config.CaFile)
+	if err != nil {
+		return nil, err
+	}
+
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      caCertPool,
+	}
+
+	tlsConfig.BuildNameToCertificate()
+	transport := &http.Transport{
+		TLSClientConfig: tlsConfig,
+	}
+
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	return &APIClient{
+		client: client,
+		config: config,
+	}, nil
+
+}
+
+//Get data
+func (ac *APIClient) Get(url string) ([]byte, error) {
+	if strings.TrimSpace(url) == "" {
+		return nil, errors.New("empty url")
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set(httpHeaderAccept, httpHeaderJSON)
+	req.SetBasicAuth(ac.config.Username, ac.config.Password)
+
+	resp, err := ac.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if resp.Body != nil {
+			resp.Body.Close()
+		}
+	}()
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New(resp.Status)
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+//Post data
+func (ac *APIClient) Post(url string, data []byte) error {
+	if strings.TrimSpace(url) == "" {
+		return errors.New("Empty url")
+	}
+
+	req, err := http.NewRequest("POST", url, strings.NewReader(string(data)))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set(httpHeaderContentType, httpHeaderJSON)
+	req.SetBasicAuth(ac.config.Username, ac.config.Password)
+	resp, err := ac.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusCreated &&
+		resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	return nil
+}
+
+//Delete data
+func (ac *APIClient) Delete(url string) error {
+	if strings.TrimSpace(url) == "" {
+		return errors.New("Empty url")
+	}
+
+	req, err := http.NewRequest("DELETE", url, nil)
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set(httpHeaderAccept, httpHeaderJSON)
+	req.SetBasicAuth(ac.config.Username, ac.config.Password)
+
+	resp, err := ac.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return errors.New(resp.Status)
+	}
+
+	return nil
+}
+
+//SwitchAccount : Switch account
+func (ac *APIClient) SwitchAccount(username, password string) {
+	if len(strings.TrimSpace(username)) == 0 ||
+		len(strings.TrimSpace(password)) == 0 {
+		return
+	}
+
+	ac.config.Username = username
+	ac.config.Password = password
+}

tests/apitests/api-testing/envs/concourse_ci.go

@@ -0,0 +1,17 @@
+package envs
+
+//ConcourseCIEnv : Env for concourse pipeline
+var ConcourseCIEnv = Environment{
+	Protocol:       "https",
+	TestingProject: "concoursecitesting01",
+	ImageName:      "busybox",
+	ImageTag:       "latest",
+	CAFile:         "../../../ca.crt",
+	KeyFile:        "../../../key.crt",
+	CertFile:       "../../../cert.crt",
+	Account:        "cody",
+	Password:       "Admin!23",
+	Admin:          "admin",
+	AdminPass:      "pksxgxmifc0cnwa5px9h",
+	Hostname:       "10.112.122.1",
+}

tests/apitests/api-testing/envs/environment.go

@@ -0,0 +1,127 @@
+package envs
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/vmware/harbor/tests/apitests/api-testing/client"
+)
+
+//Environment keeps the testing env info
+type Environment struct {
+	Protocol       string //env var: HTTP_PROTOCOL
+	Hostname       string //env var: TESTING_ENV_HOSTNAME
+	Account        string //env var: TESTING_ENV_ACCOUNT
+	Password       string //env var: TESTING_ENV_PASSWORD
+	Admin          string //env var: TESTING_ENV_ADMIN
+	AdminPass      string //env var: TESTING_ENV_ADMIN_PASS
+	TestingProject string //env var: TESTING_PROJECT_NAME
+	ImageName      string //env var: TESTING_IMAGE_NAME
+	ImageTag       string //env var: TESTING_IMAGE_TAG
+	CAFile         string //env var: CA_FILE_PATH
+	CertFile       string //env var: CERT_FILE_PATH
+	KeyFile        string //env var: KEY_FILE_PATH
+
+	//API client
+	HTTPClient *client.APIClient
+
+	//Docker client
+	DockerClient *client.DockerClient
+
+	//Initialize status
+	loaded bool
+}
+
+//Load test env info
+func (env *Environment) Load() error {
+	host := os.Getenv("TESTING_ENV_HOSTNAME")
+	if isNotEmpty(host) {
+		env.Hostname = host
+	}
+
+	account := os.Getenv("TESTING_ENV_ACCOUNT")
+	if isNotEmpty(account) {
+		env.Account = account
+	}
+
+	pwd := os.Getenv("TESTING_ENV_PASSWORD")
+	if isNotEmpty(pwd) {
+		env.Password = pwd
+	}
+
+	admin := os.Getenv("TESTING_ENV_ADMIN")
+	if isNotEmpty(admin) {
+		env.Admin = admin
+	}
+
+	adminPwd := os.Getenv("TESTING_ENV_ADMIN_PASS")
+	if isNotEmpty(adminPwd) {
+		env.AdminPass = adminPwd
+	}
+
+	pro := os.Getenv("TESTING_PROJECT_NAME")
+	if isNotEmpty(pro) {
+		env.TestingProject = pro
+	}
+
+	imgName := os.Getenv("TESTING_IMAGE_NAME")
+	if isNotEmpty(imgName) {
+		env.ImageName = imgName
+	}
+
+	imgTag := os.Getenv("TESTING_IMAGE_TAG")
+	if isNotEmpty(imgTag) {
+		env.ImageTag = imgTag
+	}
+
+	protocol := os.Getenv("HTTP_PROTOCOL")
+	if isNotEmpty(protocol) {
+		env.Protocol = protocol
+	}
+
+	caFile := os.Getenv("CA_FILE_PATH")
+	if isNotEmpty(caFile) {
+		env.CAFile = caFile
+	}
+
+	keyFile := os.Getenv("KEY_FILE_PATH")
+	if isNotEmpty(keyFile) {
+		env.KeyFile = keyFile
+	}
+
+	certFile := os.Getenv("CERT_FILE_PATH")
+	if isNotEmpty(certFile) {
+		env.CertFile = certFile
+	}
+
+	if !env.loaded {
+		cfg := client.APIClientConfig{
+			Username: env.Admin,
+			Password: env.AdminPass,
+			CaFile:   env.CAFile,
+			CertFile: env.CertFile,
+			KeyFile:  env.KeyFile,
+		}
+
+		httpClient, err := client.NewAPIClient(cfg)
+		if err != nil {
+			return err
+		}
+		env.HTTPClient = httpClient
+		env.DockerClient = &client.DockerClient{}
+
+		env.loaded = true
+	}
+
+	return nil
+}
+
+//RootURI : The root URI like https://<hostname>
+func (env *Environment) RootURI() string {
+	return fmt.Sprintf("%s://%s", env.Protocol, env.Hostname)
+}
+
+func isNotEmpty(str string) bool {
+	return len(strings.TrimSpace(str)) > 0
+}

tests/apitests/api-testing/lib/image.go

@@ -0,0 +1,137 @@
+package lib
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/vmware/harbor/tests/apitests/api-testing/client"
+	"github.com/vmware/harbor/tests/apitests/api-testing/models"
+)
+
+//ImageUtil : For repository and tag functions
+type ImageUtil struct {
+	rootURI       string
+	testingClient *client.APIClient
+}
+
+//NewImageUtil : Constructor
+func NewImageUtil(rootURI string, httpClient *client.APIClient) *ImageUtil {
+	if len(strings.TrimSpace(rootURI)) == 0 || httpClient == nil {
+		return nil
+	}
+
+	return &ImageUtil{
+		rootURI:       rootURI,
+		testingClient: httpClient,
+	}
+}
+
+//DeleteRepo : Delete repo
+func (iu *ImageUtil) DeleteRepo(repoName string) error {
+	if len(strings.TrimSpace(repoName)) == 0 {
+		return errors.New("Empty repo name for deleting")
+	}
+
+	url := fmt.Sprintf("%s%s%s", iu.rootURI, "/api/repositories/", repoName)
+	if err := iu.testingClient.Delete(url); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+//ScanTag :Scan a tag
+func (iu *ImageUtil) ScanTag(repoName string, tagName string) error {
+	if len(strings.TrimSpace(repoName)) == 0 {
+		return errors.New("Empty repo name for scanning")
+	}
+
+	if len(strings.TrimSpace(tagName)) == 0 {
+		return errors.New("Empty tag name for scanning")
+	}
+
+	url := fmt.Sprintf("%s%s%s%s%s%s", iu.rootURI, "/api/repositories/", repoName, "/tags/", tagName, "/scan")
+	if err := iu.testingClient.Post(url, nil); err != nil {
+		return err
+	}
+
+	tk := time.NewTicker(1 * time.Second)
+	defer tk.Stop()
+	done := make(chan bool)
+	errchan := make(chan error)
+	url = fmt.Sprintf("%s%s%s%s%s", iu.rootURI, "/api/repositories/", repoName, "/tags/", tagName)
+	go func() {
+		for _ = range tk.C {
+			data, err := iu.testingClient.Get(url)
+			if err != nil {
+				errchan <- err
+				return
+			}
+			var tag models.Tag
+			if err = json.Unmarshal(data, &tag); err != nil {
+				errchan <- err
+				return
+			}
+
+			if tag.ScanOverview != nil && tag.ScanOverview.Status == "finished" {
+				done <- true
+			}
+		}
+	}()
+
+	select {
+	case <-done:
+		return nil
+	case <-time.After(20 * time.Second):
+		return errors.New("Scan timeout after 30 seconds")
+	}
+}
+
+//GetRepos : Get repos in the project
+func (iu *ImageUtil) GetRepos(projectName string) ([]models.Repository, error) {
+	if len(strings.TrimSpace(projectName)) == 0 {
+		return nil, errors.New("Empty project name for getting repos")
+	}
+
+	proUtil := NewProjectUtil(iu.rootURI, iu.testingClient)
+	pid := proUtil.GetProjectID(projectName)
+	if pid == -1 {
+		return nil, fmt.Errorf("Failed to get project ID with name %s", projectName)
+	}
+
+	url := fmt.Sprintf("%s%s%d", iu.rootURI, "/api/repositories?project_id=", pid)
+	data, err := iu.testingClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	var repos []models.Repository
+	if err = json.Unmarshal(data, &repos); err != nil {
+		return nil, err
+	}
+
+	return repos, nil
+}
+
+//GetTags : Get tags
+func (iu *ImageUtil) GetTags(repoName string) ([]models.Tag, error) {
+	if len(strings.TrimSpace(repoName)) == 0 {
+		return nil, errors.New("Empty repository name for getting tags")
+	}
+
+	url := fmt.Sprintf("%s%s%s%s", iu.rootURI, "/api/repositories/", repoName, "/tags")
+	tagData, err := iu.testingClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	var tags []models.Tag
+	if err = json.Unmarshal(tagData, &tags); err != nil {
+		return nil, err
+	}
+
+	return tags, nil
+}

tests/apitests/api-testing/lib/project.go

@@ -0,0 +1,169 @@
+package lib
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/vmware/harbor/tests/apitests/api-testing/client"
+	"github.com/vmware/harbor/tests/apitests/api-testing/models"
+)
+
+//ProjectUtil : Util methods for project related
+type ProjectUtil struct {
+	rootURI       string
+	testingClient *client.APIClient
+}
+
+//NewProjectUtil : Constructor
+func NewProjectUtil(rootURI string, httpClient *client.APIClient) *ProjectUtil {
+	if len(strings.TrimSpace(rootURI)) == 0 || httpClient == nil {
+		return nil
+	}
+
+	return &ProjectUtil{
+		rootURI:       rootURI,
+		testingClient: httpClient,
+	}
+}
+
+//GetProjects : Get projects
+//If name specified, then only get the specified project
+func (pu *ProjectUtil) GetProjects(name string) ([]models.ExistingProject, error) {
+	url := pu.rootURI + "/api/projects"
+	if len(strings.TrimSpace(name)) > 0 {
+		url = url + "?name=" + name
+	}
+	data, err := pu.testingClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	var pros []models.ExistingProject
+	if err = json.Unmarshal(data, &pros); err != nil {
+		return nil, err
+	}
+
+	return pros, nil
+}
+
+//GetProjectID : Get the project ID
+//If no project existing with the name, then return -1
+func (pu *ProjectUtil) GetProjectID(projectName string) int {
+	pros, err := pu.GetProjects(projectName)
+	if err != nil {
+		return -1
+	}
+
+	if len(pros) == 0 {
+		return -1
+	}
+
+	for _, pro := range pros {
+		if pro.Name == projectName {
+			return pro.ID
+		}
+	}
+
+	return -1
+}
+
+//CreateProject :Create project
+func (pu *ProjectUtil) CreateProject(projectName string, accessLevel bool) error {
+	if len(strings.TrimSpace(projectName)) == 0 {
+		return errors.New("Empty project name for creating")
+	}
+
+	p := models.Project{
+		Name: projectName,
+		Metadata: &models.Metadata{
+			AccessLevel: fmt.Sprintf("%v", accessLevel),
+		},
+	}
+
+	body, err := json.Marshal(&p)
+	if err != nil {
+		return err
+	}
+
+	url := pu.rootURI + "/api/projects"
+	if err = pu.testingClient.Post(url, body); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+//DeleteProject : Delete project
+func (pu *ProjectUtil) DeleteProject(projectName string) error {
+	if len(strings.TrimSpace(projectName)) == 0 {
+		return errors.New("Empty project name for deleting")
+	}
+
+	pid := pu.GetProjectID(projectName)
+	if pid == -1 {
+		return errors.New("Failed to get project ID")
+	}
+
+	url := fmt.Sprintf("%s%s%d", pu.rootURI, "/api/projects/", pid)
+
+	if err := pu.testingClient.Delete(url); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+//AssignRole : Assign role to user
+func (pu *ProjectUtil) AssignRole(projectName, username string) error {
+	if len(strings.TrimSpace(projectName)) == 0 ||
+		len(strings.TrimSpace(username)) == 0 {
+		return errors.New("Project name and username are required for assigning role")
+	}
+
+	pid := pu.GetProjectID(projectName)
+	if pid == -1 {
+		return fmt.Errorf("Failed to get project ID with name %s", projectName)
+	}
+
+	m := models.Member{
+		UserName: username,
+		Roles:    []int{2},
+	}
+
+	body, err := json.Marshal(&m)
+	if err != nil {
+		return err
+	}
+
+	url := fmt.Sprintf("%s%s%d%s", pu.rootURI, "/api/projects/", pid, "/members")
+	if err := pu.testingClient.Post(url, body); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+//RevokeRole : RevokeRole role from user
+func (pu *ProjectUtil) RevokeRole(projectName string, uid int) error {
+	if len(strings.TrimSpace(projectName)) == 0 {
+		return errors.New("Project name is required for revoking role")
+	}
+
+	if uid == 0 {
+		return errors.New("User ID is required for revoking role")
+	}
+
+	pid := pu.GetProjectID(projectName)
+	if pid == -1 {
+		return fmt.Errorf("Failed to get project ID with name %s", projectName)
+	}
+
+	url := fmt.Sprintf("%s%s%d%s%d", pu.rootURI, "/api/projects/", pid, "/members/", uid)
+	if err := pu.testingClient.Delete(url); err != nil {
+		return err
+	}
+
+	return nil
+}

tests/apitests/api-testing/lib/report.go

@@ -0,0 +1,43 @@
+package lib
+
+import (
+	"fmt"
+)
+
+type Report struct {
+	passed []string
+	failed []string
+}
+
+//Passed case
+func (r *Report) Passed(caseName string) {
+	r.passed = append(r.passed, fmt.Sprintf("%s: [%s]", caseName, "PASSED"))
+}
+
+//Failed case
+func (r *Report) Failed(caseName string, err error) {
+	r.failed = append(r.failed, fmt.Sprintf("%s: [%s] %s", caseName, "FAILED", err.Error()))
+}
+
+//Print report
+func (r *Report) Print() {
+	passed := len(r.passed)
+	failed := len(r.failed)
+	total := passed + failed
+
+	fmt.Println("=====================================")
+	fmt.Printf("Overall: %d/%d passed , %d/%d failed\n", passed, total, failed, total)
+	fmt.Println("=====================================")
+	for _, res := range r.passed {
+		fmt.Println(res)
+	}
+
+	for _, res := range r.failed {
+		fmt.Println(res)
+	}
+}
+
+//IsFail : Overall result
+func (r *Report) IsFail() bool {
+	return len(r.failed) > 0
+}

tests/apitests/api-testing/lib/system.go

@@ -0,0 +1,50 @@
+package lib
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/vmware/harbor/tests/apitests/api-testing/client"
+	"github.com/vmware/harbor/tests/apitests/api-testing/models"
+)
+
+//SystemUtil : For getting system info
+type SystemUtil struct {
+	rootURI       string
+	hostname      string
+	testingClient *client.APIClient
+}
+
+//NewSystemUtil : Constructor
+func NewSystemUtil(rootURI, hostname string, httpClient *client.APIClient) *SystemUtil {
+	if len(strings.TrimSpace(rootURI)) == 0 || httpClient == nil {
+		return nil
+	}
+
+	return &SystemUtil{
+		rootURI:       rootURI,
+		hostname:      hostname,
+		testingClient: httpClient,
+	}
+}
+
+//GetSystemInfo : Get systeminfo
+func (nsu *SystemUtil) GetSystemInfo() error {
+	url := nsu.rootURI + "/api/systeminfo"
+	data, err := nsu.testingClient.Get(url)
+	if err != nil {
+		return err
+	}
+
+	var info models.SystemInfo
+	if err := json.Unmarshal(data, &info); err != nil {
+		return err
+	}
+
+	if info.RegistryURL != nsu.hostname {
+		return fmt.Errorf("Invalid registry url in system info: expect %s got %s ", nsu.hostname, info.RegistryURL)
+	}
+
+	return nil
+}

tests/apitests/api-testing/lib/user.go

@@ -0,0 +1,118 @@
+package lib
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/vmware/harbor/tests/apitests/api-testing/client"
+	"github.com/vmware/harbor/tests/apitests/api-testing/models"
+)
+
+//UserUtil : For user related
+type UserUtil struct {
+	rootURI       string
+	testingClient *client.APIClient
+}
+
+//NewUserUtil : Constructor
+func NewUserUtil(rootURI string, httpClient *client.APIClient) *UserUtil {
+	if len(strings.TrimSpace(rootURI)) == 0 || httpClient == nil {
+		return nil
+	}
+
+	return &UserUtil{
+		rootURI:       rootURI,
+		testingClient: httpClient,
+	}
+}
+
+//CreateUser : Create user
+func (uu *UserUtil) CreateUser(username, password string) error {
+	if len(strings.TrimSpace(username)) == 0 ||
+		len(strings.TrimSpace(password)) == 0 {
+		return errors.New("Username and password required for creating user")
+	}
+
+	u := models.User{
+		Username: username,
+		Password: password,
+		Email:    username + "@vmware.com",
+		RealName: username + "pks",
+		Comment:  "testing",
+	}
+
+	body, err := json.Marshal(&u)
+	if err != nil {
+		return err
+	}
+
+	url := fmt.Sprintf("%s%s", uu.rootURI, "/api/users")
+	if err := uu.testingClient.Post(url, body); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+//DeleteUser : Delete testing account
+func (uu *UserUtil) DeleteUser(username string) error {
+	uid := uu.GetUserID(username)
+	if uid == -1 {
+		return fmt.Errorf("Failed to get user with name %s", username)
+	}
+
+	url := fmt.Sprintf("%s%s%d", uu.rootURI, "/api/users/", uid)
+	if err := uu.testingClient.Delete(url); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+//GetUsers : Get users
+//If name specified, then return that one
+func (uu *UserUtil) GetUsers(name string) ([]models.ExistingUser, error) {
+	url := fmt.Sprintf("%s%s", uu.rootURI, "/api/users")
+	if len(strings.TrimSpace(name)) > 0 {
+		url = url + "?username=" + name
+	}
+
+	data, err := uu.testingClient.Get(url)
+	if err != nil {
+		return nil, err
+	}
+
+	var users []models.ExistingUser
+	if err = json.Unmarshal(data, &users); err != nil {
+		return nil, err
+	}
+
+	return users, nil
+}
+
+//GetUserID : Get user ID
+//If user with the username is not existing, then return -1
+func (uu *UserUtil) GetUserID(username string) int {
+	if len(strings.TrimSpace(username)) == 0 {
+		return -1
+	}
+
+	users, err := uu.GetUsers(username)
+	if err != nil {
+		return -1
+	}
+
+	if len(users) == 0 {
+		return -1
+	}
+
+	for _, u := range users {
+		if u.Username == username {
+			return u.ID
+		}
+	}
+
+	return -1
+}

tests/apitests/api-testing/models/endpoint.go

@@ -0,0 +1,10 @@
+package models
+
+//Endpoint : For /api/targets
+type Endpoint struct {
+	Endpoint string `json:"endpoint"`
+	Name     string `json:"name"`
+	Username string `json:"username"`
+	Password string `json:"password"`
+	Type     int    `json:"type"`
+}

tests/apitests/api-testing/models/image.go

@@ -0,0 +1,20 @@
+package models
+
+//Repository : For /api/repositories
+type Repository struct {
+	ID   int    `json:"id"`
+	Name string `json:"name"`
+}
+
+//Tag : For /api/repositories/:repo/tags
+type Tag struct {
+	Digest       string                 `json:"digest"`
+	Name         string                 `json:"name"`
+	Signature    map[string]interface{} `json:"signature, omitempty"`
+	ScanOverview *ScanOverview          `json:"scan_overview, omitempty"`
+}
+
+//ScanOverview : For scanning
+type ScanOverview struct {
+	Status string `json:"scan_status"`
+}

tests/apitests/api-testing/models/member.go

@@ -0,0 +1,7 @@
+package models
+
+//Member : For /api/projects/:pid/members
+type Member struct {
+	UserName string `json:"username"`
+	Roles    []int  `json:"roles"`
+}

tests/apitests/api-testing/models/project.go

@@ -0,0 +1,18 @@
+package models
+
+//Project : For /api/projects
+type Project struct {
+	Name     string    `json:"project_name"`
+	Metadata *Metadata `json:"metadata, omitempty"`
+}
+
+//Metadata : Metadata for project
+type Metadata struct {
+	AccessLevel string `json:"public"`
+}
+
+//ExistingProject : For /api/projects?name=***
+type ExistingProject struct {
+	Name string `json:"name"`
+	ID   int    `json:"project_id"`
+}

tests/apitests/api-testing/models/replication.go

@@ -0,0 +1,10 @@
+package models
+
+//ReplicationPolicy : For /api/replications
+type ReplicationPolicy struct {
+	ProjectID int `json:"project_id"`
+	
+}
+
+type ExistingReplicationPolicy struct {
+}

tests/apitests/api-testing/models/system_info.go

@@ -0,0 +1,7 @@
+package models
+
+//SystemInfo : For GET /api/systeminfo
+type SystemInfo struct {
+	AuthMode    string `json:"auth_mode"`
+	RegistryURL string `json:"registry_url"`
+}

tests/apitests/api-testing/models/user.go

@@ -0,0 +1,16 @@
+package models
+
+//User : For /api/users
+type User struct {
+	Username string `json:"username"`
+	RealName string `json:"realname"`
+	Password string `json:"password"`
+	Email    string `json:"email"`
+	Comment  string `json:"comment"`
+}
+
+//ExistingUser : For GET /api/users
+type ExistingUser struct {
+	User
+	ID int `json:"user_id"`
+}

tests/apitests/api-testing/tests/suites/suite.go

@@ -0,0 +1,11 @@
+package suites
+
+import (
+	"github.com/vmware/harbor/tests/apitests/api-testing/envs"
+	"github.com/vmware/harbor/tests/apitests/api-testing/lib"
+)
+
+//Suite : Run a group of test cases
+type Suite interface {
+	Run(onEnvironment envs.Environment) *lib.Report
+}

tests/apitests/api-testing/tests/suites/suite01/run_test.go

@@ -0,0 +1,22 @@
+package suite01
+
+import (
+	"testing"
+
+	"github.com/vmware/harbor/tests/apitests/api-testing/envs"
+)
+
+//TestRun : Start to run the case
+func TestRun(t *testing.T) {
+	//Initialize env
+	if err := envs.ConcourseCIEnv.Load(); err != nil {
+		t.Fatal(err.Error())
+	}
+
+	suite := ConcourseCiSuite01{}
+	report := suite.Run(&envs.ConcourseCIEnv)
+	report.Print()
+	if report.IsFail() {
+		t.Fail()
+	}
+}

tests/apitests/api-testing/tests/suites/suite01/suite.go

@@ -0,0 +1,177 @@
+package suite01
+
+import (
+	"fmt"
+
+	"github.com/vmware/harbor/tests/apitests/api-testing/envs"
+	"github.com/vmware/harbor/tests/apitests/api-testing/lib"
+)
+
+//Steps of suite01:
+//  s0: Get systeminfo
+//  s1: create project
+//  s2: create user "cody"
+//  s3: assign cody as developer
+//  s4: push a busybox image to project
+//  s5: scan image
+//  s6: pull image from project
+//  s7: remove "cody" from project member list
+//  s8: pull image from project [FAIL]
+//  s9: remove repository busybox
+//  s10: delete project
+//  s11: delete user
+
+//ConcourseCiSuite01 : For harbor journey in concourse pipeline
+type ConcourseCiSuite01 struct{}
+
+//Run : Run a group of cases
+func (ccs *ConcourseCiSuite01) Run(onEnvironment *envs.Environment) *lib.Report {
+	report := &lib.Report{}
+
+	//s0
+	sys := lib.NewSystemUtil(onEnvironment.RootURI(), onEnvironment.Hostname, onEnvironment.HTTPClient)
+	if err := sys.GetSystemInfo(); err != nil {
+		report.Failed("GetSystemInfo", err)
+	} else {
+		report.Passed("GetSystemInfo")
+	}
+
+	//s1
+	pro := lib.NewProjectUtil(onEnvironment.RootURI(), onEnvironment.HTTPClient)
+	if err := pro.CreateProject(onEnvironment.TestingProject, false); err != nil {
+		report.Failed("CreateProject", err)
+	} else {
+		report.Passed("CreateProject")
+	}
+
+	//s2
+	usr := lib.NewUserUtil(onEnvironment.RootURI(), onEnvironment.HTTPClient)
+	if err := usr.CreateUser(onEnvironment.Account, onEnvironment.Password); err != nil {
+		report.Failed("CreateUser", err)
+	} else {
+		report.Passed("CreateUser")
+	}
+
+	//s3
+	if err := pro.AssignRole(onEnvironment.TestingProject, onEnvironment.Account); err != nil {
+		report.Failed("AssignRole", err)
+	} else {
+		report.Passed("AssignRole")
+	}
+
+	//s4
+	if err := ccs.pushImage(onEnvironment); err != nil {
+		report.Failed("pushImage", err)
+	} else {
+		report.Passed("pushImage")
+	}
+
+	//s5
+	img := lib.NewImageUtil(onEnvironment.RootURI(), onEnvironment.HTTPClient)
+	repoName := fmt.Sprintf("%s/%s", onEnvironment.TestingProject, onEnvironment.ImageName)
+	if err := img.ScanTag(repoName, onEnvironment.ImageTag); err != nil {
+		report.Failed("ScanTag", err)
+	} else {
+		report.Passed("ScanTag")
+	}
+
+	//s6
+	if err := ccs.pullImage(onEnvironment); err != nil {
+		report.Failed("pullImage[1]", err)
+	} else {
+		report.Passed("pullImage[1]")
+	}
+
+	//s7
+	uid := usr.GetUserID(onEnvironment.Account)
+	if err := pro.RevokeRole(onEnvironment.TestingProject, uid); err != nil {
+		report.Failed("RevokeRole", err)
+	} else {
+		report.Passed("RevokeRole")
+	}
+
+	//s8
+	if err := ccs.pullImage(onEnvironment); err == nil {
+		report.Failed("pullImage[2]", err)
+	} else {
+		report.Passed("pullImage[2]")
+	}
+
+	//s9
+	if err := img.DeleteRepo(repoName); err != nil {
+		report.Failed("DeleteRepo", err)
+	} else {
+		report.Passed("DeleteRepo")
+	}
+
+	//s10
+	if err := pro.DeleteProject(onEnvironment.TestingProject); err != nil {
+		report.Failed("DeleteProject", err)
+	} else {
+		report.Passed("DeleteProject")
+	}
+
+	//s11
+	if err := usr.DeleteUser(onEnvironment.Account); err != nil {
+		report.Failed("DeleteUser", err)
+	} else {
+		report.Passed("DeleteUser")
+	}
+
+	return report
+}
+
+func (ccs *ConcourseCiSuite01) pushImage(onEnvironment *envs.Environment) error {
+	docker := onEnvironment.DockerClient
+	if err := docker.Status(); err != nil {
+		return err
+	}
+
+	imagePulling := fmt.Sprintf("%s:%s", onEnvironment.ImageName, onEnvironment.ImageTag)
+	if err := docker.Pull(imagePulling); err != nil {
+		return err
+	}
+
+	if err := docker.Login(onEnvironment.Account, onEnvironment.Password, onEnvironment.Hostname); err != nil {
+		return err
+	}
+
+	imagePushing := fmt.Sprintf("%s/%s/%s:%s",
+		onEnvironment.Hostname,
+		onEnvironment.TestingProject,
+		onEnvironment.ImageName,
+		onEnvironment.ImageTag)
+
+	if err := docker.Tag(imagePulling, imagePushing); err != nil {
+		return err
+	}
+
+	if err := docker.Push(imagePushing); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ccs *ConcourseCiSuite01) pullImage(onEnvironment *envs.Environment) error {
+	docker := onEnvironment.DockerClient
+	if err := docker.Status(); err != nil {
+		return err
+	}
+
+	if err := docker.Login(onEnvironment.Account, onEnvironment.Password, onEnvironment.Hostname); err != nil {
+		return err
+	}
+
+	imagePulling := fmt.Sprintf("%s/%s/%s:%s",
+		onEnvironment.Hostname,
+		onEnvironment.TestingProject,
+		onEnvironment.ImageName,
+		onEnvironment.ImageTag)
+
+	if err := docker.Pull(imagePulling); err != nil {
+		return err
+	}
+
+	return nil
+}

tests/robot-cases/Group0-Concourse-CI/Basic.robot

@@ -0,0 +1,116 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  BAT
+
+*** Variables ***
+${HARBOR_URL}  https://${ip}
+
+*** Test Cases ***
+Test Case - Sign With Admin
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
+    Close Browser
+	
+Test Case - Create An New Project
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Create An New User  url=${HARBOR_URL}  username=tester${d}  email=tester${d}@vmware.com  realname=harbortest  newPassword=Test1@34  comment=harbortest
+    Create An New Project  test${d}
+    Close Browser	
+	
+Test Case - Push Image
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Create An New User  url=${HARBOR_URL}  username=tester${d}  email=tester${d}@vmware.com  realname=harbortest  newPassword=Test1@34  comment=harbortest
+    Create An New Project  test${d}
+
+    Push image  ${ip}  tester${d}  Test1@34  test${d}  hello-world:latest
+    Go Into Project  test${d}
+    Wait Until Page Contains  test${d}/hello-world
+
+Test Case - Project Level Policy Public
+    Init Chrome Driver
+    ${d}=  Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
+    Create An New Project  project${d}
+    Go Into Project  project${d}
+    Goto Project Config
+    Click Project Public
+    Save Project Config
+    #verify
+    Public Should Be Selected 
+    Back To Projects
+    #project${d}  default should be private
+    Project Should Be Public  project${d}
+    Close Browser
+
+Test Case - Project Level Policy Content Trust
+    Init Chrome Driver
+    ${d}=  Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
+    Create An New Project  project${d}
+    Push Image  ${ip}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}  project${d}  hello-world:latest
+    Go Into Project  project${d}
+    Goto Project Config
+    Click Content Trust
+    Save Project Config
+    #verify
+    Content Trust Should Be Selected
+    Cannot Pull Unsigned Image  ${ip}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}  project${d}  hello-world:latest
+    Close Browser
+
+Test Case - Create An Replication Rule New Endpoint
+    Init Chrome Driver
+    ${d}=  Get current date  result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
+    Create An New Project  project${d}
+    Go Into Project  project${d}
+    Switch To Replication
+    Create An New Rule With New Endpoint  policy_name=test_policy_${d}  policy_description=test_description  destination_name=test_destination_name_${d}  destination_url=test_destination_url_${d}  destination_username=test_destination_username  destination_password=test_destination_password
+    Close Browser
+
+Test Case - Scan A Tag
+    Init Chrome Driver
+    ${d}=  get current date  result_format=%m%s
+    Create An New Project With New User  url=${HARBOR_URL}  username=tester${d}  email=tester${d}@vmware.com  realname=tester${d}  newPassword=Test1@34  comment=harbor  projectname=project${d}  public=false
+    Push Image  ${ip}  tester${d}  Test1@34  project${d}  hello-world
+    Go Into Project  project${d}
+    Expand Repo  project${d}
+    Scan Repo  latest
+    Summary Chart Should Display  latest
+    Close Browser
+
+Test Case - Admin Push Signed Image
+    Enabe Notary Client
+
+    ${rc}  ${output}=  Run And Return Rc And Output  docker pull hello-world:latest
+    Log  ${output}
+		
+    Push image  ${ip}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}  library  hello-world:latest
+    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group9-Content-trust/notary-push-image.sh
+    Log  ${output}
+    Should Be Equal As Integers  ${rc}  0
+
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -u admin:Harbor12345 -s --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/repositories/library/tomcat/signatures"
+    Log To Console  ${output}
+    Should Be Equal As Integers  ${rc}  0
+    #Should Contain  ${output}  sha256
+
+Test Case - Admin Push Un-Signed Image	
+    ${rc}  ${output}=  Run And Return Rc And Output  docker push ${ip}/library/hello-world:latest
+    Log To Console  ${output}

tests/robot-cases/Group0-Concourse-CI/LDAP.robot

@@ -0,0 +1,59 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  BAT
+
+*** Variables ***
+${HARBOR_URL}  https://${ip}
+
+*** Test Cases ***
+Test Case - Ldap Verify Cert
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
+    Switch To Configure
+    Test Ldap Connection
+    Close Browser
+
+Test Case - Ldap Sign in and out
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
+    Switch To Configure
+    Init LDAP
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  mike  zhu88jie
+    Close Browser
+
+Test Case - Ldap User Create Project
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  mike  zhu88jie
+    Create An New Project  project${d}
+    Close Browser
+
+Test Case - Ldap User Push An Image
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  mike  zhu88jie
+    Create An New Project  project${d}
+    
+    Push Image  ${ip}  mike  zhu88jie  project${d}  hello-world:latest
+    Go Into Project  project${d}
+    Wait Until Page Contains  project${d}/hello-world
+    Close Browser
+
+Test Case - Ldap User Can Not login
+    Docker Login Fail  ${ip}  test  123456

tests/testprepare.sh

@@ -7,9 +7,10 @@ cp make/common/config/ui/private_key.pem /etc/ui/.
 
 mkdir conf
 cp make/common/config/ui/app.conf conf/.
-
+IP=`ip addr s eth0 |grep "inet "|awk '{print $2}' |awk -F "/" '{print $1}'`
-sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=127.0.0.1/" make/common/config/adminserver/env
+echo "server ip is "$IP
-sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://127.0.0.1:5000|" make/common/config/adminserver/env
+sed -i -r "s/MYSQL_HOST=mysql/MYSQL_HOST=$IP/" make/common/config/adminserver/env
+sed -i -r "s|REGISTRY_URL=http://registry:5000|REGISTRY_URL=http://$IP:5000|" make/common/config/adminserver/env
 sed -i -r "s/UI_SECRET=.*/UI_SECRET=$UI_SECRET/" make/common/config/adminserver/env
 
 chmod 777 /data/

tools/migration/changelog.md

@@ -56,3 +56,6 @@ Changelog for harbor database schema
   - insert data into table `project_metadata`
   - delete column `public` from table `project`
   - add column `insecure` to table `replication_target`
+## 1.3.x
+  - add pk `id` to table `properties`
+  - remove pk index from colum 'k' of table `properties`