Merge pull request #5345 from wy65701436/master-basea

Update e2e cases

tests/nightly-test/deployment/deployer.py

@@ -1,158 +0,0 @@
-import abc
-import logging
-import os
-import sys
-import time
-import subprocess
-from datetime import datetime
-dir_path = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(dir_path + '../utils')
-import govc_utils
-import nlogging
-logger = nlogging.create_logger(__name__)
-
-class Deployer(object):
-    __metaclass__ = abc.ABCMeta
-
-    @abc.abstractmethod
-    def deploy(self):
-        return
-
-    @abc.abstractmethod
-    def destory(self):
-        return
-
-class OVADeployer(Deployer):
-
-    def __init__(self, vc_host, vc_user, vc_password, ds, cluster, ova_path, ova_name, ova_root_password, count, 
-                dry_run, auth_mode, ldap_url, ldap_searchdn, ldap_search_pwd, ldap_filter, ldap_basedn, ldap_uid,
-                ldap_scope, ldap_timeout):
-        self.vc_host = vc_host 
-        self.vc_user = vc_user 
-        self.vc_password = vc_password 
-        self.ds = ds 
-        self.cluster = cluster  
-        self.ova_path = ova_path 
-        self.ova_name = ova_name 
-        self.ova_root_password = ova_root_password 
-        self.dry_run = dry_run
-        self.count = count
-        self.auth_mode=auth_mode
-
-        if auth_mode == 'ldap_auth':
-            self.ldap_url = ldap_url
-            self.ldap_searchdn = ldap_searchdn
-            self.ldap_search_pwd = ldap_search_pwd
-            self.ldap_filter = ldap_filter
-            self.ldap_basedn = ldap_basedn
-            self.ldap_uid = ldap_uid
-            self.ldap_scope = ldap_scope
-            self.ldap_timeout = ldap_timeout
-                
-        self.harbor_password='Harbor12345'
-        self.log_path=None 
-        self.ip=None
-        self.netmask=None
-        self.gateway=None
-        self.dns=None
-        self.ovf_tool_path=None
-        self.DEFAULT_LOCAL_OVF_TOOL_PATH = '/home/harbor-ci/ovftool/ovftool'
-        self.ova_endpoints = []
-        self.ova_names = []
-
-    def __generate_ova_names(self):
-        for i in range(0, self.count):
-            ova_name_temp = ''
-            ova_name_temp = self.ova_name +"-"+ datetime.now().isoformat().replace(":", "-").replace(".", "-")
-            time.sleep(1)
-            self.ova_names.append(ova_name_temp)
-    
-    def __set_ovf_tool(self):
-        if not self.ova_endpoints:
-            self.ovf_tool_path = self.DEFAULT_LOCAL_OVF_TOOL_PATH
-        if not os.path.isfile(self.ovf_tool_path):
-            logger.error("ovftool not found.")
-        return
-
-    def __compose_cmd(self, ova_name):
-        cmd = ''
-
-        if self.auth_mode == "db_auth":
-            cmd = (
-                '"%s" --X:"logFile"="./deploy_oms.log" --overwrite --powerOn --datastore=\'%s\' --noSSLVerify --acceptAllEulas --name=%s \
-                --X:injectOvfEnv --X:enableHiddenProperties  --prop:root_pwd=\'%s\' --prop:permit_root_login=true --prop:auth_mode=\'%s\' \
-                --prop:harbor_admin_password=\'%s\' --prop:max_job_workers=5   %s  \
-                vi://%s:\'%s\'@%s/Datacenter/host/%s'
-                % (self.ovf_tool_path, self.ds, ova_name,
-                    self.ova_root_password, self.auth_mode,
-                    self.harbor_password, self.ova_path,
-                    self.vc_user, self.vc_password, self.vc_host, self.cluster
-                )
-            )
-
-        if self.auth_mode == "ldap_auth":
-            cmd = (
-                '"%s" --X:"logFile"="./deploy_oms.log" --overwrite --powerOn --datastore=\'%s\' --noSSLVerify --acceptAllEulas --name=%s \
-                --X:injectOvfEnv --X:enableHiddenProperties  --prop:root_pwd=\'%s\' --prop:permit_root_login=true --prop:auth_mode=\'%s\' \
-                --prop:harbor_admin_password=\'%s\' --prop:max_job_workers=5 \
-                --prop:ldap_url=\'%s\' --prop:ldap_searchdn=\'%s\' --prop:ldap_search_pwd=\'%s\' \
-                --prop:ldap_filter=\'%s\' \
-                --prop:ldap_basedn=\'%s\' \
-                --prop:ldap_uid=\'%s\' --prop:ldap_scope=\'%s\'  --prop:ldap_timeout=\'%s\'   %s  \
-                vi://%s:\'%s\'@%s/Datacenter/host/%s'
-                % (self.ovf_tool_path, self.ds, ova_name,
-                    self.ova_root_password, self.auth_mode,
-                    self.harbor_password, 
-                    self.ldap_url, self.ldap_searchdn,
-                    self.ldap_search_pwd, self.ldap_filter,
-                    self.ldap_basedn, self.ldap_uid,
-                    self.ldap_scope, self.ldap_timeout,
-                    self.ova_path,
-                    self.vc_user, self.vc_password, self.vc_host, self.cluster
-                )
-            )
-        return cmd     
-   
-    def deploy(self):
-        self.__generate_ova_names()
-        self.__set_ovf_tool()
-
-        for i in range(0, self.count):
-            cmd = self.__compose_cmd(self.ova_names[i])
-            logger.info(cmd)
-            if self.dry_run == "true" :
-                logger.info("Dry run ...")
-            else:
-                try:
-                    subprocess.check_output(cmd, shell=True)
-                except Exception, e:
-                    logger.info(e)
-                    time.sleep(5)
-                    # try onre more time if any failure.
-                    subprocess.check_output(cmd, shell=True)
-            logger.info("Successfully deployed harbor OVA.")
-
-            ova_endpoint = ''
-            ova_endpoint = govc_utils.getvmip(self.vc_host, self.vc_user, self.vc_password, self.ova_names[i])
-            if ova_endpoint is not '':
-                self.ova_endpoints.append(ova_endpoint) 
-
-        return self.ova_endpoints, self.ova_names
-
-    def destory(self):
-        for item in self.ova_names:
-            govc_utils.destoryvm(self.vc_host, self.vc_user, self.vc_password, item)
-
-
-class OfflineDeployer(Deployer):
-
-    def __init__(self):
-        self.vm_host = '' 
-        self.vm_user = '' 
-        self.vm_password = '' 
-
-    def deploy(self):
-        pass
-
-    def destory(self):
-        pass

tests/nightly-test/launch.py

@@ -1,114 +0,0 @@
-#!/usr/bin/python2
-
-import sys
-import os
-import ConfigParser
-from subprocess import call
-from datetime import datetime
-import time
-dir_path = os.path.dirname(os.path.realpath(__file__))
-sys.path.append(dir_path + '/utils')
-sys.path.append(dir_path + '/deployment')
-import harbor_util
-import buildweb_utils
-import govc_utils
-import nlogging
-logger = nlogging.create_logger(__name__)
-import test_executor
-from deployer import *
-
-if len(sys.argv)!=9 :
-    logger.info("python launch.py <build_type> <image_url> <test suitename> <config_file> <dry_run> <auth_mode> <destory>")
-    logger.info("Wrong parameters, quit test")
-    quit()
-
-build_type = sys.argv[1]
-image_url = sys.argv[2]
-test_suite = sys.argv[3]
-config_file = sys.argv[4]
-deploy_count = int(sys.argv[5])
-dry_run = sys.argv[6]
-auth_mode = sys.argv[7]
-destory = sys.argv[8]
-config_file = os.getcwd() + "/harbor_nightly_test/testenv.ini"
-config = ConfigParser.ConfigParser()
-config.read(config_file)
-harbor_endpoints = []
-vm_names = []
-
-# ----- deploy harbor build -----
-if build_type == "ova" :
-    vc_host = config.get("vcenter", "vc_host")
-    vc_user = config.get("vcenter", "vc_user")
-    vc_password = config.get("vcenter", "vc_password")
-    datastore = config.get("vcenter", "datastore")
-    cluster = config.get("vcenter", "cluster")
-    ova_password = config.get("vcenter", "ova_password")
-    ova_name = config.get("vcenter", "ova_name")
-
-    ldap_url = config.get("ldap", "ldap_url")
-    ldap_searchdn = config.get("ldap", "ldap_searchdn")
-    ldap_search_pwd = config.get("ldap", "ldap_search_pwd")
-    ldap_filter = config.get("ldap", "ldap_filter")
-    ldap_basedn = config.get("ldap", "ldap_basedn")
-    ldap_uid = config.get("ldap", "ldap_uid")
-    ldap_scope = config.get("ldap", "ldap_scope")
-    ldap_timeout = config.get("ldap", "ldap_timeout")
-
-    if image_url == "latest" :
-        image_url = buildweb_utils.get_latest_build_url('master','beta')
-    logger.info("Get latest image url:" + image_url)
-
-    ova_deployer = OVADeployer(vc_host, 
-                vc_user,
-                vc_password, 
-                datastore, 
-                cluster, 
-                image_url, 
-                ova_name, 
-                ova_password,
-                deploy_count,
-                dry_run,
-                auth_mode,
-                ldap_url,
-                ldap_searchdn,
-                ldap_search_pwd,
-                ldap_filter,
-                ldap_basedn,
-                ldap_uid,
-                ldap_scope,
-                ldap_timeout)
-
-    logger.info("Going to deploy harbor ova..")
-    harbor_endpoints, vm_names = ova_deployer.deploy()
-
-    # ----- wait for harbor ready -----
-    for item in harbor_endpoints:
-        is_harbor_ready = harbor_util.wait_for_harbor_ready("https://"+item)
-        if not is_harbor_ready:
-            logger.info("Harbor is not ready after 10 minutes.")
-            sys.exit(-1)
-        logger.info("%s is ready for test now..." % item)
-        # ----- log harbor version -----
-        harbor_version = harbor_util.get_harbor_version(item, 'admin', 'Harbor12345')
-        logger.info("Harbor version: %s ..." % harbor_version)
-        with open(os.getcwd() + '/build.properties', 'w') as the_file:
-            the_file.write('harbor_version=%s' % harbor_version)
-
-    # ----- execute test cases -----
-    try:
-        execute_results = test_executor.execute(harbor_endpoints, vm_names, ova_password, test_suite, auth_mode, vc_host, vc_user, vc_password)
-        if not execute_results:
-            logger.info("execute test failure.")
-            sys.exit(-1)
-    except Exception, e:
-        logger.info(e)
-        sys.exit(-1)
-    finally:
-        if destory == "true":
-            ova_deployer.destory()
-
-elif build_type == "installer" :
-    logger.info("Going to download installer image to install")
-elif build_type == "all" :
-    logger.info("launch ova and installer")

tests/nightly-test/shellscript/destoryvm.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-export GOVC_USERNAME=$2
-export GOVC_PASSWORD=$3
-export GOVC_INSECURE=1
-export GOVC_URL=$1
-govc vm.destroy $4

tests/nightly-test/shellscript/getvmip.sh

@@ -1,7 +0,0 @@
-#!/bin/sh
-
-export GOVC_USERNAME=$2
-export GOVC_PASSWORD=$3
-export GOVC_INSECURE=1
-export GOVC_URL=$1
-govc vm.info -json $4 | jq -r .VirtualMachines[].Guest.HostName

tests/nightly-test/utils/buildweb_utils.py

@@ -1,64 +0,0 @@
-import os
-import logging
-
-from buildwebapi import api as buildapi
-LOG = logging.getLogger(__name__)
-
-
-def get_build_type(build_id):
-    build = get_build(build_id)
-    LOG.debug('%s is %s build', build_id, build.buildtype)
-    return build.buildtype
-
-
-def get_build_id_and_system(build_id):
-    build_system = 'ob'
-    if '-' in str(build_id):
-        temp = build_id.split('-')
-        build_id = temp[1]
-        build_system = temp[0]
-    return build_id, build_system
-
-
-def get_ova_url(build_id):
-    return get_url(build_id, '_OVF10.ova')
-
-
-def get_url(build_id, deliverable_name):
-    build = get_build(build_id)
-    deliverables = buildapi.ListResource.by_url(build._deliverables_url)
-    deliverable = [d for d in deliverables
-                   if d.matches(path=deliverable_name)][0]
-    LOG.debug('Download URL of %s is %s', build_id, deliverable._download_url)
-    return deliverable._download_url
-
-
-def get_product(build_id):
-    build = get_build(build_id)
-    LOG.debug('Product of %s is %s.', build_id, build.product)
-    return build.product
-
-
-def get_latest_build_url(branch, build_type, product='harbor_build'):
-    build_id = get_latest_build_id(branch, build_type, product)
-    print build_id
-    return get_ova_url(build_id)
-
-
-def get_latest_build_id(branch, build_type, product='harbor_build'):
-    return buildapi.MetricResource.by_name('build',
-                                           product=product,
-                                           buildstate='succeeded',
-                                           buildtype=build_type,
-                                           branch=branch).get_max_id()
-
-
-def get_build(build_id):
-    build_id, build_system = get_build_id_and_system(build_id)
-    return buildapi.ItemResource.by_id('build', int(build_id), build_system)
-
-
-def get_build_version(build_id):
-    build = get_build(build_id)
-    LOG.debug('Version of %s is %s.', build_id, build.version)
-    return build.version

tests/nightly-test/utils/cmd_utils.py

@@ -1,37 +0,0 @@
-import datetime
-import logging
-import time
-
-
-LOG = logging.getLogger(__name__)
-
-
-def wait_for(func, timeout, delay, *args, **kargs):
-    """Decorator for waiting for until a function finished running."""
-
-    poll_timeout = timeout
-    poll_sleep_retry = delay
-
-    begin_poll = datetime.datetime.now()
-    while True:
-        try:
-            return func(*args, **kargs)
-            break
-        except Exception as e:
-            if (datetime.datetime.now() - begin_poll).seconds > poll_timeout:
-                LOG.exception('Time out after %s seconds.' % poll_timeout)
-                raise TimeoutError('Timed out after %s seconds. Reason: '
-                                   '%s' % (poll_timeout, e))
-            else:
-                LOG.debug('Sleeping %s seconds before retrying'
-                          '' % poll_sleep_retry)
-                time.sleep(poll_sleep_retry)
-
-
-def safe_run(cmd, msg, sleep_time=180):
-    exit_code = shell.local(cmd)[0]
-    if exit_code:
-        LOG.warning('Failed to %s. Retry it after %s seconds' %
-                    (msg, sleep_time))
-        time.sleep(sleep_time)
-        shell.local(cmd, raise_error=True)

tests/nightly-test/utils/govc_utils.py

@@ -1,32 +0,0 @@
-#!/usr/bin/python2
-
-import os, subprocess
-import time
-
-SHELL_SCRIPT_DIR = os.getcwd() + '/tests/nightly-test/shellscript/'
-
-def getvmip(vc_url, vc_user, vc_password, vm_name, timeout=600) :
-    cmd = (SHELL_SCRIPT_DIR+'getvmip.sh %s %s %s %s ' % (vc_url, vc_user, getPasswordInShell(vc_password), vm_name))
-    interval = 10
-    while True:
-        try:
-            if timeout <= 0:
-                return ''
-            result = subprocess.check_output(cmd,shell=True).strip()
-            if result is not '':
-                if result != 'photon-machine':
-                    return result
-        except Exception, e:
-            timeout -= interval
-            time.sleep(interval)
-            continue
-        timeout -= interval
-        time.sleep(interval)
-
-def destoryvm(vc_url, vc_user, vc_password, vm_name) :
-    cmd = (SHELL_SCRIPT_DIR+'destoryvm.sh %s %s %s %s ' % (vc_url, vc_user, getPasswordInShell(vc_password), vm_name))  
-    result = subprocess.check_output(cmd, shell=True)
-    return result
-
-def getPasswordInShell(password) :
-    return password.replace("!", "\!")

tests/nightly-test/utils/harbor_util.py

@@ -1,53 +0,0 @@
-from urllib2 import urlopen
-import ssl
-import time
-import os
-try:
-    import json
-except ImportError:
-    import simplejson as json
-
-import requests
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-
-def request(harbor_endpoint, url, method, user, pwd, **kwargs):
-    url = "https://" + harbor_endpoint + "/api" + url
-    kwargs.setdefault('headers', kwargs.get('headers', {}))
-    kwargs['headers']['Accept'] = 'application/json'
-    if 'body' in kwargs:
-        kwargs['headers']['Content-Type'] = 'application/json'
-        kwargs['data'] = json.dumps(kwargs['body'])
-        del kwargs['body']
-
-    resp = requests.request(method, url, verify=False, auth=(user, pwd), **kwargs)
-    if resp.status_code >= 400:
-        raise Exception("Error: %s" % resp.text)
-    try:
-        body = json.loads(resp.text)
-    except ValueError:
-        body = resp.text
-    return body
-
-# wait for 10 minutes as OVA needs about 7 minutes to startup harbor.
-def wait_for_harbor_ready(harbor_endpoint, timeout=600):
-    ctx = ssl.create_default_context()
-    ctx.check_hostname = False
-    ctx.verify_mode = ssl.CERT_NONE
-    interval = 10
-    while True:
-        try:
-            if timeout <= 0:
-                return False
-            code = urlopen(harbor_endpoint, context=ctx).code
-            if code == 200:
-                return True
-        except Exception, e:
-            timeout -= interval
-            time.sleep(interval)
-            continue
-        timeout -= interval
-        time.sleep(interval)
-
-def get_harbor_version(harbor_endpoint, harbor_user, harbor_pwd):
-   return request(harbor_endpoint, '/systeminfo', 'get', harbor_user, harbor_pwd)['harbor_version']

tests/nightly-test/utils/nlogging.py

@@ -1,14 +0,0 @@
-from __future__ import absolute_import
-
-import logging
-import sys
-
-def create_logger(name):
-    default_handler = logging.StreamHandler(sys.stderr)
-    default_handler.setFormatter(logging.Formatter(
-        '[%(asctime)s] %(levelname)s in %(module)s: %(message)s'
-    ))
-    logger = logging.getLogger(name)
-    logger.setLevel(logging.DEBUG)
-    logger.addHandler(default_handler) 
-    return logger

tests/nightly-test/utils/test_executor.py

@@ -1,67 +0,0 @@
-#!/usr/bin/python2
-
-import os, subprocess
-import time
-import sys
-
-from subprocess import call
-import json
-
-import nlogging
-logger = nlogging.create_logger(__name__)
-
-# Needs have docker installed.
-def execute(harbor_endpoints, vm_names, harbor_root_pwd, test_suite, auth_mode ,vc_host, vc_user, vc_password, harbor_pwd='Harbor12345') :
-    cmd = ''
-    exe_result = -1
-    cmd_base = "docker run -i --privileged -v %s:/drone -w /drone vmware/harbor-e2e-engine:1.38 " % os.getcwd()
-
-    if len(harbor_endpoints) == 1:
-        cmd_pybot = "pybot -v ip:%s -v vm_name:%s -v ip1: -v HARBOR_PASSWORD:%s -v SSH_PWD:%s -v vc_host:%s -v vc_user:%s -v vc_password:%s " % (harbor_endpoints[0], vm_names[0], harbor_pwd, harbor_root_pwd, vc_host, vc_user, vc_password)
-    
-    if len(harbor_endpoints) == 2:
-        cmd_pybot = "pybot -v ip:%s -v vm_name:%s -v ip1:%s -v vm_name1:%s -v HARBOR_PASSWORD:%s -v SSH_PWD:%s -v vc_host:%s -v vc_user:%s -v vc_password:%s " % (harbor_endpoints[0], vm_names[0], harbor_endpoints[1], vm_names[1], harbor_pwd, harbor_root_pwd, vc_host, vc_user, vc_password)
-
-    cmd = cmd_base + cmd_pybot
-    if test_suite == 'Nightly':
-        if auth_mode == 'ldap_auth':
-            cmd = cmd + "/drone/tests/robot-cases/Group11-Nightly/LDAP.robot"           
-        else:           
-            cmd = cmd + "/drone/tests/robot-cases/Group11-Nightly/Nightly.robot"
-
-        logger.info(cmd)
-        p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE)
-        while True:
-            out = p.stderr.read(1)
-            if out == '' and p.poll() != None:
-                break
-            if out != '':
-                sys.stdout.write(out)
-                sys.stdout.flush()
-        exe_result = p.returncode
-
-    if test_suite == 'Replication':
-        cmd = cmd + "/drone/tests/robot-cases/Group11-Nightly/Replication.robot"
-
-        logger.info(cmd)
-        p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE)
-        while True:
-            out = p.stderr.read(1)
-            if out == '' and p.poll() != None:
-                break
-            if out != '':
-                sys.stdout.write(out)
-                sys.stdout.flush()
-        exe_result = p.returncode
-
-    if test_suite == 'Longevity':
-        cmd = cmd + "/drone/tests/robot-cases/Group12-Longevity/Longevity.robot > /dev/null 2>&1"
-        logger.info(cmd)
-        exe_result = subprocess.call(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE)
-    
-    collect_log()
-    return exe_result == 0
-
-# Needs to move log.html to another path it will be overwrite by any pybot run.
-def collect_log():
-    pass

tests/resources/Harbor-Pages/Project-Config.robot

@@ -28,6 +28,7 @@ Select Prevent Level
     [Arguments]  ${level}
     Click Element  //hbr-project-policy-config//select
     Click Element  //hbr-project-policy-config//select/option[contains(.,'${level}')]
+
 Click Auto Scan
     Mouse Down  //hbr-project-policy-config//input[@name='scan-image-on-push']
     Mouse Up  //hbr-project-policy-config//input[@name='scan-image-on-push']
@@ -36,16 +37,19 @@ Save Project Config
     Sleep  1
     Click Element  //hbr-project-policy-config//button[contains(.,'SAVE')]
 
-#assert
 Public Should Be Selected
     Checkbox Should Be Selected  //hbr-project-policy-config//input[@name='public']
+
 Project Should Be Public
     [Arguments]  ${projectName}
     Page Should Contain Element  //clr-dg-row[contains(.,'${projectName}')]//clr-dg-cell[contains(.,'Public')]
+
 Content Trust Should Be Selected
     Checkbox Should Be Selected  //hbr-project-policy-config//input[@name='content-trust']
+
 Prevent Running Should Be Selected
     Checkbox Should Be Selected  //hbr-project-policy-config//input[@name='prevent-vulnerability-image']
+
 Auto Scan Should Be Selected
     Checkbox Should Be Selected  //hbr-project-policy-config//input[@name='scan-image-on-push']
 

tests/resources/Harbor-Pages/Project-Members.robot

@@ -106,10 +106,12 @@ Add Guest Member To Project
 Delete Project Member
     [arguments]  ${member}
     Click Element  xpath=//clr-dg-row[contains(.,'${member}')]//input/../label
-    Click Element  xpath=${project_member_delete_button_xpath}
+    Click Element  ${member_action_xpath}
-    Sleep  2
+    Sleep  1
-    Click Element  xpath=//button[contains(.,'DELETE')]
+    Click Element  ${delete_action_xpath}
     Sleep  2
+    Click Element  //clr-modal//button[contains(.,'DELETE')]
+    Sleep  3
 
 User Should Be Owner Of Project
     [Arguments]  ${user}  ${pwd}  ${project}

tests/resources/Harbor-Pages/Project-Members_Elements.robot

@@ -17,7 +17,7 @@ Documentation  This resource provides any keywords related to the Harbor private
 
 *** Variables ***
 ${project_member_tag_xpath}  /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/nav/ul/li[2]/a
-${project_member_add_button_xpath}  //project-detail//button[contains(.,'Member')]
+${project_member_add_button_xpath}  //project-detail//button[contains(.,'User')]
 ${project_member_add_username_xpath}  //*[@id="member_name"]
 ${project_member_add_admin_xpath}  /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/ng-component/div/div[1]/div/div[1]/add-member/clr-modal/div/div[1]/div/div[1]/div/div[2]/form/section/div[2]/div[1]/label
 ${project_member_add_save_button_xpath}  /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/ng-component/div/div[1]/div/div[1]/add-member/clr-modal/div/div[1]/div/div[1]/div/div[3]/button[2]

tests/resources/Harbor-Pages/Project.robot

@@ -31,7 +31,7 @@ Create An New Project
     Run Keyword If  '${public}' == 'true'  Click Element  xpath=${project_public_xpath}
     Click Element  xpath=//button[contains(.,'OK')]
     Sleep  4
-    ${rc}  ${output}=  Run And Return Rc And Output  curl -u %{HARBOR_ADMIN}:%{HARBOR_PASSWORD} -k -X GET --header 'Accept: application/json' ${HARBOR_URL}/api/projects?name=${projectname}
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -u ${HARBOR_ADMIN}:${HARBOR_PASSWORD} -k -X GET --header 'Accept: application/json' ${HARBOR_URL}/api/projects?name=${projectname}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
     Should Contain  ${output}  ${projectname}

tests/resources/Harbor-Pages/ToolKit.robot

@@ -44,6 +44,7 @@ Select Object
     [Arguments]    ${obj}
     Click Element  //clr-dg-row[contains(.,'${obj}')]//label
 
+# This func cannot support as the delete user flow changed.
 Multi-delete Object
     [Arguments]    @{obj}
     :For  ${obj}  in  @{obj}
@@ -54,12 +55,26 @@ Multi-delete Object
     Click Element  //clr-modal//button[contains(.,'DELETE')]
     Sleep  3
 
+Multi-delete User
+    [Arguments]    @{obj}
+    :For  ${obj}  in  @{obj}
+    \    Click Element  //clr-dg-row[contains(.,'${obj}')]//label
+    Sleep  1
+    Click Element  ${member_action_xpath}
+    Sleep  1
+    Click Element  //clr-dropdown/clr-dropdown-menu/button[2]
+    Sleep  2
+    Click Element  //clr-modal//button[contains(.,'DELETE')]
+    Sleep  3
+
 Multi-delete Member 
     [Arguments]    @{obj}
     :For  ${obj}  in  @{obj}
     \    Click Element  //clr-dg-row[contains(.,'${obj}')]//label
     Sleep  1
-    Click Element  //button[contains(.,'REMOVE')]
+    Click Element  ${member_action_xpath}
+    Sleep  1
+    Click Element  ${delete_action_xpath}
     Sleep  2
     Click Element  //clr-modal//button[contains(.,'DELETE')]
     Sleep  3

tests/resources/Harbor-Pages/ToolKit_Elements.robot

@@ -0,0 +1,20 @@
+# Copyright 2016-2017 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#	http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+*** Settings ***
+Documentation  This resource provides any keywords related to the Harbor private registry appliance
+
+*** Variables ***
+${member_action_xpath}  //*[@id="member-action"]
+${delete_action_xpath}  //clr-dropdown/clr-dropdown-menu/button[4]

tests/resources/Harbor-Pages/Vulnerability.robot

@@ -45,7 +45,7 @@ Scan Repo
     #select one tag
     Click Element  //clr-dg-row[contains(.,"${tagname}")]//label
     Click Element  //button[contains(.,'Scan')]
-    Sleep  5
+    Sleep  20
     Run Keyword If  '${status}' == 'Succeed'  Wait Until Element Is Visible  //hbr-vulnerability-bar//hbr-vulnerability-summary-chart
     Run Keyword If  '${status}' == 'Fail'  Wait Until Element Is Visible  //hbr-vulnerability-bar//a
 
@@ -59,6 +59,7 @@ Enable Scan On Push
     Click Element  //clr-checkbox[@name="scan-image-on-push"]//label
     Checkbox Should Be Selected  //clr-checkbox[@name="scan-image-on-push"]//input
     Click Element  //hbr-project-policy-config//button[contains(.,'SAVE')]
+    Sleep  10
 
 Vulnerability Not Ready Project Hint
     Page Should Contain Element  //span[@class="db-status-warning"]

tests/resources/Nightly-Util.robot

@@ -21,22 +21,17 @@ ${SSH_USER}  root
 
 *** Keywords ***
 Nightly Test Setup
-    [Arguments]  ${ip}  ${SSH_PWD}  ${HARBOR_PASSWORD}  ${ip1}==${EMPTY}
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${ip1}==${EMPTY}
-    Run Keyword  CA setup  ${ip}  ${SSH_PWD}  ${HARBOR_PASSWORD}
+    Run Keyword If  '${ip1}' != '${EMPTY}'  CA setup  ${ip1}  ${HARBOR_PASSWORD}  /ca/ca1.crt
-    Run Keyword  Prepare Docker Cert  ${ip}    
+    Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm -rf ./harbor_ca.crt
-    Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm harbor_ca.crt 
+    Run Keyword  CA setup  ${ip}  ${HARBOR_PASSWORD}
-    Run Keyword If  '${ip1}' != '${EMPTY}'  CA setup  ${ip1}  ${SSH_PWD}  ${HARBOR_PASSWORD}
-    Run Keyword If  '${ip1}' != '${EMPTY}'  Prepare Docker Cert  ${ip1}
     Run Keyword  Start Docker Daemon Locally
 
 CA Setup
-    [Arguments]  ${ip}  ${SSH_PWD}  ${HARBOR_PASSWORD}
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${cert}=/ca/ca.crt
-    Open Connection    ${ip}
+    Run  mv ${cert} harbor_ca.crt
-    Login    ${SSH_USER}    ${SSH_PWD}
+    Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}
-    SSHLibrary.Get File  /data/ca_download/ca.crt
+    Prepare Docker Cert  ${ip}	
-    Close All Connections
-    Run  mv ca.crt harbor_ca.crt
-    Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}	
 
 Collect Nightly Logs
     [Arguments]  ${ip}  ${SSH_PWD}  ${ip1}==${EMPTY}

tests/resources/SeleniumUtil.robot

@@ -28,11 +28,13 @@ Init Chrome Driver
     Run  pkill chromedriver
     Run  pkill chrome
     ${chrome options}=    Evaluate    sys.modules['selenium.webdriver'].ChromeOptions()    sys
+    ${capabilities}=    Evaluate    sys.modules['selenium.webdriver'].DesiredCapabilities.CHROME    sys
+    Set To Dictionary    ${capabilities}    acceptInsecureCerts    ${True}
     Call Method    ${chrome options}    add_argument    --headless
     Call Method    ${chrome options}    add_argument    --disable-gpu
     Call Method    ${chrome options}    add_argument    --start-maximized
     Call Method    ${chrome options}    add_argument    --no-sandbox
     Call Method    ${chrome options}    add_argument    --window-size\=1600,900
     ${chrome options.binary_location}    Set Variable    /usr/bin/google-chrome
-    Create Webdriver    Chrome    Chrome_headless    chrome_options=${chrome options}
+    Create Webdriver    Chrome    Chrome_headless    chrome_options=${chrome options}    desired_capabilities=${capabilities}
     Sleep  5

tests/resources/Util.robot

@@ -44,6 +44,7 @@ Resource  Harbor-Pages/Administration-Users_Elements.robot
 Resource  Harbor-Pages/Configuration.robot
 Resource  Harbor-Pages/Configuration_Elements.robot
 Resource  Harbor-Pages/ToolKit.robot
+Resource  Harbor-Pages/ToolKit_Elements.robot
 Resource  Harbor-Pages/Vulnerability.robot
 Resource  Harbor-Pages/LDAP-Mode.robot
 Resource  Docker-Util.robot

tests/robot-cases/Group11-Nightly/Common.robot

@@ -0,0 +1,626 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Variables ***
+${HARBOR_URL}  https://${ip}
+${SSH_USER}  root
+${HARBOR_ADMIN}  admin
+
+*** Test Cases ***
+Test Case - Sign With Admin
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Close Browser
+
+Test Case - Vulnerability Data Not Ready
+#This case must run before vulnerability db ready
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Vulnerability Not Ready Project Hint
+    Switch To Configure
+    Go To Vulnerability Config
+    Vulnerability Not Ready Config Hint
+    
+Test Case - Create An New Project
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  test${d}
+    Close Browser
+
+Test Case - Delete A Project
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  project${d}
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  hello-world
+    Project Should Not Be Deleted  project${d}
+    Go Into Project  project${d}
+    Delete Repo  project${d}
+    Back To projects
+    Project Should Be Deleted  project${d}
+    Close Browser
+
+Test Case - Read Only Mode
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  project${d}
+
+    Enable Read Only
+    Cannot Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  busybox:latest
+
+    Disable Read Only
+    Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  busybox:latest
+    Close Browser
+
+Test Case - Repo Size
+    Init Chrome Driver
+    ${d}=  Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Push Image With Tag  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  alpine  2.6  2.6
+    Go Into Project  library
+    Go Into Repo  alpine
+    Page Should Contain  1.92MB 
+    Close Browser
+
+Test Case - Staticsinfo
+    Init Chrome Driver
+    ${d}=  Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    ${privaterepocount1}=  Get Statics Private Repo
+    ${privateprojcount1}=  Get Statics Private Project
+    ${publicrepocount1}=  Get Statics Public Repo
+    ${publicprojcount1}=  Get Statics Public Project
+    ${totalrepocount1}=  Get Statics Total Repo
+    ${totalprojcount1}=  Get Statics Total Project
+    Create An New Project  private${d}
+    Create An New Project  public${d}  true
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  private${d}  hello-world
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  public${d}  hello-world
+    Reload Page
+    ${privaterepocount2}=  Get Statics Private Repo
+    ${privateprojcount2}=  get statics private project
+    ${publicrepocount2}=  get statics public repo
+    ${publicprojcount2}=  get statics public project
+    ${totalrepocount2}=  get statics total repo
+    ${totalprojcount2}=  get statics total project
+    ${privateprojcount}=  evaluate  ${privateprojcount1}+1
+    ${privaterepocount}=  evaluate  ${privaterepocount1}+1
+    ${publicprojcount}=  evaluate  ${publicprojcount1}+1
+    ${publicrepocount}=  evaluate  ${publicrepocount1}+1
+    ${totalrepocount}=  evaluate  ${totalrepocount1}+2
+    ${totalprojcount}=  evaluate  ${totalprojcount1}+2
+    Should Be Equal As Integers  ${privateprojcount2}  ${privateprojcount}
+    Should be equal as integers  ${privaterepocount2}  ${privaterepocount}
+    Should be equal as integers  ${publicprojcount2}  ${publicprojcount}
+    Should be equal as integers  ${publicrepocount2}  ${publicrepocount}
+    Should be equal as integers  ${totalprojcount2}  ${totalprojcount}
+    Should be equal as integers  ${totalrepocount2}  ${totalrepocount}
+
+Test Case - Push Image
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  test${d}
+
+    Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  test${d}  hello-world:latest
+    Go Into Project  test${d}
+    Wait Until Page Contains  test${d}/hello-world
+
+Test Case - Project Level Policy Public
+    Init Chrome Driver
+    ${d}=  Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  project${d}
+    Go Into Project  project${d}
+    Goto Project Config
+    Click Project Public
+    Save Project Config
+    # Verify
+    Public Should Be Selected
+    # Project${d}  default should be private
+    # Here logout and login to try avoid a bug only in autotest
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Filter Object  project${d}
+    Project Should Be Public  project${d}
+    Close Browser
+
+Test Case - Project Level Policy Content Trust
+    Init Chrome Driver
+    ${d}=  Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  project${d}
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  hello-world:latest
+    Go Into Project  project${d}
+    Goto Project Config
+    Click Content Trust
+    Save Project Config
+    # Verify
+    Content Trust Should Be Selected
+    Cannot Pull Unsigned Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  hello-world:latest
+    Close Browser
+
+Test Case - Verify Download Ca Link
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To System Settings
+    Page Should Contain  Registry Root Certificate
+    Close Browser
+
+Test Case - Edit Email Settings
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+
+    Switch To Email
+    Config Email
+
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+
+    Switch To Email
+    Verify Email
+
+    Close Browser
+
+Test Case - Edit Token Expire
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To System Settings
+    Modify Token Expiration  20
+    Logout Harbor
+
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To System Settings
+    Token Must Be Match  20
+
+    #reset to default
+    Modify Token Expiration  30
+    Close Browser
+
+Test Case - Create A New Labels
+    Init Chrome Driver
+    ${d}=    Get Current Date
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To System Labels
+    Create New Labels  label_${d}
+    Close Browser
+
+Test Case - Update Label
+   Init Chrome Driver
+   ${d}=    Get Current Date
+
+   Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+   Switch To System Labels
+   Create New Labels  label_${d}
+   Sleep  3
+   ${d1}=    Get Current Date
+   Update A Label  label_${d}
+   Close Browser
+
+Test Case - Delete Label
+    Init Chrome Driver
+    ${d}=    Get Current Date
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To System Labels
+    Create New Labels  label_${d}
+    Sleep  3
+    Delete A Label  label_${d}
+    Close Browser
+
+Test Case - Disable Scan Schedule
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Configure
+    Go To Vulnerability Config
+    Disable Scan Schedule
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Configure
+    Go To Vulnerability Config
+    Page Should Contain  None
+    Close Browser
+
+Test Case - User View Projects
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  user001  Test1@34
+    Create An New Project  test${d}1
+    Create An New Project  test${d}2
+    Create An New Project  test${d}3
+    Switch To Log
+    Wait Until Page Contains  test${d}1
+    Wait Until Page Contains  test${d}2
+    Wait Until Page Contains  test${d}3
+    Close Browser
+
+Test Case - User View Logs
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+
+    Sign In Harbor  ${HARBOR_URL}  user002  Test1@34
+    Create An New Project  project${d}
+    
+    Push image  ${ip}  user002  Test1@34  project${d}  busybox:latest
+    Pull image  ${ip}  user002  Test1@34  project${d}  busybox:latest
+
+    Go Into Project  project${d}
+    Delete Repo  project${d}
+
+    Go To Project Log
+    Advanced Search Should Display
+
+    Do Log Advanced Search
+    Close Browser
+
+
+Test Case - Manage Project Member
+    Init Chrome Driver
+    ${d}=    Get current Date  result_format=%m%s
+ 
+    Sign In Harbor  ${HARBOR_URL}  user004  Test1@34
+    Create An New Project  project${d}
+    Push image  ip=${ip}  user=user004  pwd=Test1@34  project=project${d}  image=hello-world
+    Logout Harbor
+
+    User Should Not Be A Member Of Project  user005  Test1@34  project${d}
+    Manage Project Member  user004  Test1@34  project${d}  user005  Add
+    User Should Be Guest  user005  Test1@34  project${d}
+    Change User Role In Project  user004  Test1@34  project${d}  user005  Developer
+    User Should Be Developer  user005  Test1@34  project${d}
+    Change User Role In Project  user004  Test1@34  project${d}  user005  Admin
+    User Should Be Admin  user005  Test1@34  project${d}  user006
+    Manage Project Member  user004  Test1@34  project${d}  user005  Remove
+    User Should Not Be A Member Of Project  user005  Test1@34  project${d}
+    User Should Be Guest  user006  Test1@34  project${d}
+
+    Close Browser
+
+Test Case - Manage project publicity
+    Init Chrome Driver
+    ${d}=    Get Current Date  result_format=%m%s
+
+    Sign In Harbor  ${HARBOR_URL}  user007  Test1@34
+    Create An New Project  project${d}  public=true
+
+    Push image  ${ip}  user007  Test1@34  project${d}  hello-world:latest
+    Pull image  ${ip}  user008  Test1@34  project${d}  hello-world:latest
+
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  user008  Test1@34
+    Project Should Display  project${d}
+    Search Private Projects
+    Project Should Not Display  project${d}
+
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  user007  Test1@34
+    Make Project Private  project${d}
+
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  user008  Test1@34
+    Project Should Not Display  project${d}
+    Cannot Pull image  ${ip}  user008  Test1@34  project${d}  hello-world:latest
+
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  user007  Test1@34
+    Make Project Public  project${d}
+
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  user008  Test1@34
+    Project Should Display  project${d}
+    Close Browser
+
+Test Case - Assign Sys Admin
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  user009  Test1@34
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch to User Tag
+    Assign User Admin  user009
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  user009  Test1@34
+    Administration Tag Should Display
+    Close Browser
+
+Test Case - Edit Project Creation
+    # Create normal user and login
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  user010  Test1@34
+    Project Creation Should Display
+    Logout Harbor
+
+    Sleep  3
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Set Pro Create Admin Only
+    Logout Harbor
+
+    Sign In Harbor  ${HARBOR_URL}  user010  Test1@34
+    Project Creation Should Not Display
+    Logout Harbor
+
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Set Pro Create Every One
+    Close browser
+
+Test Case - Edit Repo Info
+    Init Chrome Driver
+    ${d}=  Get Current Date  result_format=%m%s
+    
+    Sign In Harbor  ${HARBOR_URL}  user011  Test1@34
+    Create An New Project  project${d}
+    Push Image  ${ip}  user011  Test1@34  project${d}  hello-world
+    Go Into Project  project${d}
+    Go Into Repo  project${d}/hello-world
+    Edit Repo Info
+    Close Browser
+
+Test Case - Delete Multi Project
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    
+    Sign In Harbor  ${HARBOR_URL}  user012  Test1@34
+    Create An New Project  projecta${d}
+    Create An New Project  projectb${d}
+    Push Image  ${ip}  user012  Test1@34  projecta${d}  hello-world
+    Filter Object  project
+    Multi-delete Object  projecta  projectb
+    # Verify delete project with image should not be deleted directly
+    Delete Fail  projecta${d}
+    Delete Success  projectb${d}
+    Close Browser
+
+Test Case - Delete Multi Repo
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    
+    Sign In Harbor  ${HARBOR_URL}  user013  Test1@34
+    Create An New Project  project${d}
+    Push Image  ${ip}  user013  Test1@34  project${d}  hello-world
+    Push Image  ${ip}  user013  Test1@34  project${d}  busybox
+    Sleep  2
+    Go Into Project  project${d}
+    Multi-delete Object  hello-world  busybox
+    # Verify
+    Delete Success  hello-world  busybox
+    Close Browser
+
+Test Case - Delete Multi Tag
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    
+    Sign In Harbor  ${HARBOR_URL}  user014  Test1@34
+    Create An New Project  project${d}
+    Push Image With Tag  ${ip}  user014  Test1@34  project${d}  redis  3.2.10-alpine  3.2.10-alpine
+    Push Image With Tag  ${ip}  user014  Test1@34  project${d}  redis  4.0.7-alpine  4.0.7-alpine
+    Sleep  2
+    Go Into Project  project${d}
+    Go Into Repo  redis
+    Multi-delete object  3.2.10-alpine  4.0.7-alpine
+    # Verify
+    Delete Success  3.2.10-alpine  4.0.7-alpine
+    Close Browser
+
+Test Case - Delete Repo on CardView
+    Init Chrome Driver
+    ${d}=   Get Current Date  result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  user015  Test1@34
+    Create An New Project  project${d}
+    Push Image  ${ip}  user015  Test1@34  project${d}  hello-world
+    Push Image  ${ip}  user015  Test1@34  project${d}  busybox
+    Sleep  2
+    Go Into Project  project${d}
+    Switch To CardView
+    Delete Repo on CardView  busybox
+    # Verify
+    Delete Success  busybox
+    Close Browser
+
+Test Case - Delete Multi Member
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  user016  Test1@34    
+    Create An New Project  project${d}
+    Go Into Project  project${d}
+    Switch To Member
+    Add Guest Member To Project  user017
+    Add Guest Member To Project  user018
+    Multi-delete Member  user017  user018
+    Delete Success  user017  user018
+    Close Browser
+
+Test Case - Project Admin Operate Labels
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  user019  Test1@34
+    Create An New Project  project${d}
+    Go Into Project  project${d}
+    Sleep  2
+    # Add labels
+    Switch To Project Label
+    Create New Labels  label_${d}
+    Sleep  2
+    Update A Label  label_${d}
+    Sleep  2
+    Delete A Label  label_${d}
+    Close Browser
+
+Test Case - Project Admin Add Labels To Repo
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  user020  Test1@34
+    Create An New Project  project${d}
+    Push Image With Tag  ${ip}  user020  Test1@34  project${d}  redis  3.2.10-alpine  3.2.10-alpine
+    Push Image With Tag  ${ip}  user020  Test1@34  project${d}  redis  4.0.7-alpine  4.0.7-alpine
+
+    Go Into Project  project${d}
+    Sleep  2
+    # Add labels
+    Switch To Project Label
+    Create New Labels  label111
+    Create New Labels  label22
+    Sleep  2
+    Switch To Project Repo
+    Go Into Repo  project${d}/redis
+    Add Labels To Tag  3.2.10-alpine  label111
+    Add Labels To Tag  4.0.7-alpine  label22
+    Filter Labels In Tags  label111  label22
+    Close Browser
+
+Test Case - Developer Operate Labels
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+
+    Sign In Harbor  ${HARBOR_URL}  user021  Test1@34
+    Create An New Project  project${d}
+    Logout Harbor
+    
+    Manage Project Member  user021  Test1@34  project${d}  user022  Add
+    Change User Role In Project  user021  Test1@34  project${d}  user022  Developer
+
+    Sign In Harbor  ${HARBOR_URL}  user022  Test1@34
+    Go Into Project  project${d}
+    Sleep  3
+    Page Should Not Contain Element  xpath=//a[contains(.,'Labels')]
+    Close Browser
+
+Test Case - Scan A Tag In The Repo
+    Init Chrome Driver
+    ${d}=  get current date  result_format=%m%s
+
+    Sign In Harbor  ${HARBOR_URL}  user023  Test1@34
+    Create An New Project  project${d}
+
+    Go Into Project  project${d}    
+    Push Image  ${ip}  user023  Test1@34  project${d}  hello-world
+    Go Into Project  project${d}
+    Go Into Repo  project${d}/hello-world
+    Scan Repo  latest  Succeed
+    Summary Chart Should Display  latest
+    Pull Image  ${ip}  user023  Test1@34  project${d}  hello-world
+    # Edit Repo Info
+    Close Browser
+
+Test Case - Scan As An Unprivileged User
+    Init Chrome Driver
+    ${d}=    get current date    result_format=%m%s
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  hello-world
+ 
+    Sign In Harbor  ${HARBOR_URL}  user024  Test1@34
+    Go Into Project  library
+    Go Into Repo  hello-world
+    Select Object  latest
+    Scan Is Disabled
+    Close Browser
+
+Test Case - Scan Image With Empty Vul
+    Init Chrome Driver
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  hello-world
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Go Into Repo  hello-world
+    Scan Repo  latest  Succeed
+    Move To Summary Chart
+    Wait Until Page Contains  Unknow
+    Close Browser
+
+Test Case - Manual Scan All
+    Init Chrome Driver
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  redis
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Configure
+    Go To Vulnerability Config
+    Trigger Scan Now
+    Back To Projects
+    Go Into Project  library
+    Go Into Repo  redis
+    Summary Chart Should Display  latest
+    Close Browser
+
+Test Case - Scan Image On Push
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Goto Project Config
+    Enable Scan On Push
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  memcached
+    Back To Projects
+    Go Into Project  library
+    Go Into Repo  memcached
+    Summary Chart Should Display  latest
+    Close Browser
+
+Test Case - View Scan Results
+    Init Chrome Driver
+    ${d}=  get current date  result_format=%m%s
+
+    Sign In Harbor  ${HARBOR_URL}  user025  Test1@34
+    Create An New Project  project${d}    
+    Push Image  ${ip}  user025  Test1@34  project${d}  tomcat
+    Go Into Project  project${d}
+    Go Into Repo  project${d}/tomcat
+    Scan Repo  latest  Succeed
+    Summary Chart Should Display  latest
+    View Repo Scan Details
+    Close Browser
+
+Test Case - View Scan Error
+    Init Chrome Driver
+    ${d}=  get current date  result_format=%m%s
+
+    Sign In Harbor  ${HARBOR_URL}  user026  Test1@34
+    Create An New Project  project${d}   
+    Push Image  ${ip}  user026  Test1@34  project${d}  vmware/photon:1.0
+    Go Into Project  project${d}
+    Go Into Repo  project${d}/vmware/photon
+    Scan Repo  1.0  Fail
+    View Scan Error Log
+    Close Browser
+
+Test Case - Project Level Image Serverity Policy
+    Init Chrome Driver
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  haproxy
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Go Into Repo  haproxy
+    Scan Repo  latest  Succeed
+    Back To Projects
+    Go Into Project  library
+    Set Vulnerabilty Serverity  0
+    Cannot pull image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  haproxy
+    Close Browser
+
+Test Case - Admin Push Signed Image
+    Enable Notary Client
+
+    ${rc}  ${output}=  Run And Return Rc And Output  docker pull hello-world:latest
+    Log  ${output}
+
+    Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  hello-world:latest
+    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group9-Content-trust/notary-push-image.sh ${ip}
+    Log  ${output}
+    Should Be Equal As Integers  ${rc}  0
+
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -u admin:Harbor12345 -s --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/repositories/library/tomcat/signatures"
+    Log To Console  ${output}
+    Should Be Equal As Integers  ${rc}  0
+    Should Contain  ${output}  sha256

tests/robot-cases/Group11-Nightly/DB.robot

@@ -0,0 +1,81 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Variables ***
+${HARBOR_URL}  https://${ip}
+${SSH_USER}  root
+${HARBOR_ADMIN}  admin
+
+*** Test Cases ***
+Test Case - Create An New User
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Create An New User  url=${HARBOR_URL}  username=tester${d}  email=tester${d}@vmware.com  realname=harbortest  newPassword=Test1@34  comment=harbortest
+    Close Browser
+
+Test Case - Update User Comment
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Create An New User  url=${HARBOR_URL}  username=tester${d}  email=tester${d}@vmware.com  realname=harbortest  newPassword=Test1@34  comment=harbortest
+    Update User Comment  Test12#4
+    Logout Harbor
+
+Test Case - Update Password
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Create An New User  url=${HARBOR_URL}  username=tester${d}  email=tester${d}@vmware.com  realname=harbortest  newPassword=Test1@34  comment=harbortest
+    Change Password  Test1@34  Test12#4
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  tester${d}  Test12#4
+    Close Browser
+
+Test Case - Delete Multi User
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    Create An New User  ${HARBOR_URL}  deletea${d}  testa${d}@vmware.com  test${d}  Test1@34  harbor
+    Logout Harbor
+    Create An New User  ${HARBOR_URL}  deleteb${d}  testb${d}@vmware.com  test${d}  Test1@34  harbor
+    Logout Harbor
+    Create An New User  ${HARBOR_URL}  deletec${d}  testc${d}@vmware.com  test${d}  Test1@34  harbor
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  admin  Harbor12345
+    Switch To User Tag
+    Filter Object  delete
+    Multi-delete User  deletea  deleteb  deletec
+    # Assert delete
+    Delete Success  deletea  deleteb  deletec
+    Sleep  1
+    Close Browser
+
+Test Case - Edit Self-Registration
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Disable Self Reg
+    Logout Harbor
+
+    Sign Up Should Not Display
+
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Configure
+    Self Reg Should Be Disabled
+    Sleep  1
+
+    # Restore setting
+    Enable Self Reg
+    Close Browser

tests/robot-cases/Group11-Nightly/LDAP.robot

@@ -15,8 +15,6 @@
 *** Settings ***
 Documentation  Harbor BATs
 Resource  ../../resources/Util.robot
-Suite Setup  Nightly Test Setup  ${ip}  ${SSH_PWD}  ${HARBOR_PASSWORD}  ${ip1}
-Suite Teardown  Collect Nightly Logs  ${ip}  ${SSH_PWD}  ${ip1}
 Default Tags  Nightly
 
 *** Variables ***
@@ -110,4 +108,4 @@ Test Case - Ldap User Push An Image
     Close Browser
 
 Test Case - Ldap User Can Not login
-    Docker Login Fail  ${ip}  test  123456
+    Docker Login Fail  ${ip}  testerDeesExist  123456

tests/robot-cases/Group11-Nightly/Replication.robot

@@ -15,14 +15,13 @@
 *** Settings ***
 Documentation  Harbor BATs
 Resource  ../../resources/Util.robot
-Suite Setup  Nightly Test Setup  ${ip}  ${SSH_PWD}  ${HARBOR_PASSWORD}  ${ip1}
-Suite Teardown  Collect Nightly Logs  ${ip}  ${SSH_PWD}  ${ip1}
 Default Tags  Replication
 
 *** Variables ***
 ${HARBOR_URL}  https://${ip}
 ${SSH_USER}  root
 ${HARBOR_ADMIN}  admin
+
 *** Test Cases ***
 Test Case - Get Harbor Version
 #Just get harbor version and log it

tests/robot-cases/Group11-Nightly/Setup.robot

@@ -0,0 +1,22 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Test Cases ***
+Test Suites Setup
+    Nightly Test Setup  ${ip}  ${HARBOR_PASSWORD}  ${ip1}

tests/robot-cases/Group11-Nightly/Teardown.robot

@@ -0,0 +1,22 @@
+// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Test Cases ***
+Test Suites Teardown
+    Log To Console  Teardown do nothing here.