From 85ded71816e0ca326785deae7a643388dba21523 Mon Sep 17 00:00:00 2001 From: Yan Date: Thu, 19 Jul 2018 17:02:09 +0800 Subject: [PATCH] Update e2e cases refactor e2e test cases to support new CI pipeline fix issues caused by recently UI change --- tests/nightly-test/deployment/deployer.py | 158 ----- tests/nightly-test/launch.py | 114 ---- tests/nightly-test/shellscript/destoryvm.sh | 7 - tests/nightly-test/shellscript/getvmip.sh | 7 - tests/nightly-test/utils/buildweb_utils.py | 64 -- tests/nightly-test/utils/cmd_utils.py | 37 -- tests/nightly-test/utils/govc_utils.py | 32 - tests/nightly-test/utils/harbor_util.py | 53 -- tests/nightly-test/utils/nlogging.py | 14 - tests/nightly-test/utils/test_executor.py | 67 -- .../Harbor-Pages/Project-Config.robot | 6 +- .../Harbor-Pages/Project-Members.robot | 8 +- .../Project-Members_Elements.robot | 2 +- tests/resources/Harbor-Pages/Project.robot | 2 +- tests/resources/Harbor-Pages/ToolKit.robot | 17 +- .../Harbor-Pages/ToolKit_Elements.robot | 20 + .../Harbor-Pages/Vulnerability.robot | 3 +- tests/resources/Nightly-Util.robot | 21 +- tests/resources/SeleniumUtil.robot | 4 +- tests/resources/Util.robot | 1 + .../robot-cases/Group11-Nightly/Common.robot | 626 ++++++++++++++++++ tests/robot-cases/Group11-Nightly/DB.robot | 81 +++ tests/robot-cases/Group11-Nightly/LDAP.robot | 4 +- .../Group11-Nightly/Replication.robot | 3 +- tests/robot-cases/Group11-Nightly/Setup.robot | 22 + .../Group11-Nightly/Teardown.robot | 22 + 26 files changed, 815 insertions(+), 580 deletions(-) delete mode 100644 tests/nightly-test/deployment/deployer.py delete mode 100644 tests/nightly-test/launch.py delete mode 100755 tests/nightly-test/shellscript/destoryvm.sh delete mode 100755 tests/nightly-test/shellscript/getvmip.sh delete mode 100644 tests/nightly-test/utils/buildweb_utils.py delete mode 100644 tests/nightly-test/utils/cmd_utils.py delete mode 100644 tests/nightly-test/utils/govc_utils.py delete mode 100644 tests/nightly-test/utils/harbor_util.py delete mode 100644 tests/nightly-test/utils/nlogging.py delete mode 100644 tests/nightly-test/utils/test_executor.py create mode 100644 tests/resources/Harbor-Pages/ToolKit_Elements.robot create mode 100644 tests/robot-cases/Group11-Nightly/Common.robot create mode 100644 tests/robot-cases/Group11-Nightly/DB.robot create mode 100644 tests/robot-cases/Group11-Nightly/Setup.robot create mode 100644 tests/robot-cases/Group11-Nightly/Teardown.robot diff --git a/tests/nightly-test/deployment/deployer.py b/tests/nightly-test/deployment/deployer.py deleted file mode 100644 index f91d1a7f1..000000000 --- a/tests/nightly-test/deployment/deployer.py +++ /dev/null @@ -1,158 +0,0 @@ -import abc -import logging -import os -import sys -import time -import subprocess -from datetime import datetime -dir_path = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(dir_path + '../utils') -import govc_utils -import nlogging -logger = nlogging.create_logger(__name__) - -class Deployer(object): - __metaclass__ = abc.ABCMeta - - @abc.abstractmethod - def deploy(self): - return - - @abc.abstractmethod - def destory(self): - return - -class OVADeployer(Deployer): - - def __init__(self, vc_host, vc_user, vc_password, ds, cluster, ova_path, ova_name, ova_root_password, count, - dry_run, auth_mode, ldap_url, ldap_searchdn, ldap_search_pwd, ldap_filter, ldap_basedn, ldap_uid, - ldap_scope, ldap_timeout): - self.vc_host = vc_host - self.vc_user = vc_user - self.vc_password = vc_password - self.ds = ds - self.cluster = cluster - self.ova_path = ova_path - self.ova_name = ova_name - self.ova_root_password = ova_root_password - self.dry_run = dry_run - self.count = count - self.auth_mode=auth_mode - - if auth_mode == 'ldap_auth': - self.ldap_url = ldap_url - self.ldap_searchdn = ldap_searchdn - self.ldap_search_pwd = ldap_search_pwd - self.ldap_filter = ldap_filter - self.ldap_basedn = ldap_basedn - self.ldap_uid = ldap_uid - self.ldap_scope = ldap_scope - self.ldap_timeout = ldap_timeout - - self.harbor_password='Harbor12345' - self.log_path=None - self.ip=None - self.netmask=None - self.gateway=None - self.dns=None - self.ovf_tool_path=None - self.DEFAULT_LOCAL_OVF_TOOL_PATH = '/home/harbor-ci/ovftool/ovftool' - self.ova_endpoints = [] - self.ova_names = [] - - def __generate_ova_names(self): - for i in range(0, self.count): - ova_name_temp = '' - ova_name_temp = self.ova_name +"-"+ datetime.now().isoformat().replace(":", "-").replace(".", "-") - time.sleep(1) - self.ova_names.append(ova_name_temp) - - def __set_ovf_tool(self): - if not self.ova_endpoints: - self.ovf_tool_path = self.DEFAULT_LOCAL_OVF_TOOL_PATH - if not os.path.isfile(self.ovf_tool_path): - logger.error("ovftool not found.") - return - - def __compose_cmd(self, ova_name): - cmd = '' - - if self.auth_mode == "db_auth": - cmd = ( - '"%s" --X:"logFile"="./deploy_oms.log" --overwrite --powerOn --datastore=\'%s\' --noSSLVerify --acceptAllEulas --name=%s \ - --X:injectOvfEnv --X:enableHiddenProperties --prop:root_pwd=\'%s\' --prop:permit_root_login=true --prop:auth_mode=\'%s\' \ - --prop:harbor_admin_password=\'%s\' --prop:max_job_workers=5 %s \ - vi://%s:\'%s\'@%s/Datacenter/host/%s' - % (self.ovf_tool_path, self.ds, ova_name, - self.ova_root_password, self.auth_mode, - self.harbor_password, self.ova_path, - self.vc_user, self.vc_password, self.vc_host, self.cluster - ) - ) - - if self.auth_mode == "ldap_auth": - cmd = ( - '"%s" --X:"logFile"="./deploy_oms.log" --overwrite --powerOn --datastore=\'%s\' --noSSLVerify --acceptAllEulas --name=%s \ - --X:injectOvfEnv --X:enableHiddenProperties --prop:root_pwd=\'%s\' --prop:permit_root_login=true --prop:auth_mode=\'%s\' \ - --prop:harbor_admin_password=\'%s\' --prop:max_job_workers=5 \ - --prop:ldap_url=\'%s\' --prop:ldap_searchdn=\'%s\' --prop:ldap_search_pwd=\'%s\' \ - --prop:ldap_filter=\'%s\' \ - --prop:ldap_basedn=\'%s\' \ - --prop:ldap_uid=\'%s\' --prop:ldap_scope=\'%s\' --prop:ldap_timeout=\'%s\' %s \ - vi://%s:\'%s\'@%s/Datacenter/host/%s' - % (self.ovf_tool_path, self.ds, ova_name, - self.ova_root_password, self.auth_mode, - self.harbor_password, - self.ldap_url, self.ldap_searchdn, - self.ldap_search_pwd, self.ldap_filter, - self.ldap_basedn, self.ldap_uid, - self.ldap_scope, self.ldap_timeout, - self.ova_path, - self.vc_user, self.vc_password, self.vc_host, self.cluster - ) - ) - return cmd - - def deploy(self): - self.__generate_ova_names() - self.__set_ovf_tool() - - for i in range(0, self.count): - cmd = self.__compose_cmd(self.ova_names[i]) - logger.info(cmd) - if self.dry_run == "true" : - logger.info("Dry run ...") - else: - try: - subprocess.check_output(cmd, shell=True) - except Exception, e: - logger.info(e) - time.sleep(5) - # try onre more time if any failure. - subprocess.check_output(cmd, shell=True) - logger.info("Successfully deployed harbor OVA.") - - ova_endpoint = '' - ova_endpoint = govc_utils.getvmip(self.vc_host, self.vc_user, self.vc_password, self.ova_names[i]) - if ova_endpoint is not '': - self.ova_endpoints.append(ova_endpoint) - - return self.ova_endpoints, self.ova_names - - def destory(self): - for item in self.ova_names: - govc_utils.destoryvm(self.vc_host, self.vc_user, self.vc_password, item) - - -class OfflineDeployer(Deployer): - - def __init__(self): - self.vm_host = '' - self.vm_user = '' - self.vm_password = '' - - def deploy(self): - pass - - def destory(self): - pass \ No newline at end of file diff --git a/tests/nightly-test/launch.py b/tests/nightly-test/launch.py deleted file mode 100644 index d1f36d166..000000000 --- a/tests/nightly-test/launch.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/python2 - -import sys -import os -import ConfigParser -from subprocess import call -from datetime import datetime -import time -dir_path = os.path.dirname(os.path.realpath(__file__)) -sys.path.append(dir_path + '/utils') -sys.path.append(dir_path + '/deployment') -import harbor_util -import buildweb_utils -import govc_utils -import nlogging -logger = nlogging.create_logger(__name__) -import test_executor -from deployer import * - -if len(sys.argv)!=9 : - logger.info("python launch.py ") - logger.info("Wrong parameters, quit test") - quit() - -build_type = sys.argv[1] -image_url = sys.argv[2] -test_suite = sys.argv[3] -config_file = sys.argv[4] -deploy_count = int(sys.argv[5]) -dry_run = sys.argv[6] -auth_mode = sys.argv[7] -destory = sys.argv[8] -config_file = os.getcwd() + "/harbor_nightly_test/testenv.ini" -config = ConfigParser.ConfigParser() -config.read(config_file) -harbor_endpoints = [] -vm_names = [] - -# ----- deploy harbor build ----- -if build_type == "ova" : - vc_host = config.get("vcenter", "vc_host") - vc_user = config.get("vcenter", "vc_user") - vc_password = config.get("vcenter", "vc_password") - datastore = config.get("vcenter", "datastore") - cluster = config.get("vcenter", "cluster") - ova_password = config.get("vcenter", "ova_password") - ova_name = config.get("vcenter", "ova_name") - - ldap_url = config.get("ldap", "ldap_url") - ldap_searchdn = config.get("ldap", "ldap_searchdn") - ldap_search_pwd = config.get("ldap", "ldap_search_pwd") - ldap_filter = config.get("ldap", "ldap_filter") - ldap_basedn = config.get("ldap", "ldap_basedn") - ldap_uid = config.get("ldap", "ldap_uid") - ldap_scope = config.get("ldap", "ldap_scope") - ldap_timeout = config.get("ldap", "ldap_timeout") - - if image_url == "latest" : - image_url = buildweb_utils.get_latest_build_url('master','beta') - logger.info("Get latest image url:" + image_url) - - ova_deployer = OVADeployer(vc_host, - vc_user, - vc_password, - datastore, - cluster, - image_url, - ova_name, - ova_password, - deploy_count, - dry_run, - auth_mode, - ldap_url, - ldap_searchdn, - ldap_search_pwd, - ldap_filter, - ldap_basedn, - ldap_uid, - ldap_scope, - ldap_timeout) - - logger.info("Going to deploy harbor ova..") - harbor_endpoints, vm_names = ova_deployer.deploy() - - # ----- wait for harbor ready ----- - for item in harbor_endpoints: - is_harbor_ready = harbor_util.wait_for_harbor_ready("https://"+item) - if not is_harbor_ready: - logger.info("Harbor is not ready after 10 minutes.") - sys.exit(-1) - logger.info("%s is ready for test now..." % item) - # ----- log harbor version ----- - harbor_version = harbor_util.get_harbor_version(item, 'admin', 'Harbor12345') - logger.info("Harbor version: %s ..." % harbor_version) - with open(os.getcwd() + '/build.properties', 'w') as the_file: - the_file.write('harbor_version=%s' % harbor_version) - - # ----- execute test cases ----- - try: - execute_results = test_executor.execute(harbor_endpoints, vm_names, ova_password, test_suite, auth_mode, vc_host, vc_user, vc_password) - if not execute_results: - logger.info("execute test failure.") - sys.exit(-1) - except Exception, e: - logger.info(e) - sys.exit(-1) - finally: - if destory == "true": - ova_deployer.destory() - -elif build_type == "installer" : - logger.info("Going to download installer image to install") -elif build_type == "all" : - logger.info("launch ova and installer") diff --git a/tests/nightly-test/shellscript/destoryvm.sh b/tests/nightly-test/shellscript/destoryvm.sh deleted file mode 100755 index 75ed7c7f2..000000000 --- a/tests/nightly-test/shellscript/destoryvm.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -export GOVC_USERNAME=$2 -export GOVC_PASSWORD=$3 -export GOVC_INSECURE=1 -export GOVC_URL=$1 -govc vm.destroy $4 diff --git a/tests/nightly-test/shellscript/getvmip.sh b/tests/nightly-test/shellscript/getvmip.sh deleted file mode 100755 index 0340a82dd..000000000 --- a/tests/nightly-test/shellscript/getvmip.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh - -export GOVC_USERNAME=$2 -export GOVC_PASSWORD=$3 -export GOVC_INSECURE=1 -export GOVC_URL=$1 -govc vm.info -json $4 | jq -r .VirtualMachines[].Guest.HostName \ No newline at end of file diff --git a/tests/nightly-test/utils/buildweb_utils.py b/tests/nightly-test/utils/buildweb_utils.py deleted file mode 100644 index f76fbf9a1..000000000 --- a/tests/nightly-test/utils/buildweb_utils.py +++ /dev/null @@ -1,64 +0,0 @@ -import os -import logging - -from buildwebapi import api as buildapi -LOG = logging.getLogger(__name__) - - -def get_build_type(build_id): - build = get_build(build_id) - LOG.debug('%s is %s build', build_id, build.buildtype) - return build.buildtype - - -def get_build_id_and_system(build_id): - build_system = 'ob' - if '-' in str(build_id): - temp = build_id.split('-') - build_id = temp[1] - build_system = temp[0] - return build_id, build_system - - -def get_ova_url(build_id): - return get_url(build_id, '_OVF10.ova') - - -def get_url(build_id, deliverable_name): - build = get_build(build_id) - deliverables = buildapi.ListResource.by_url(build._deliverables_url) - deliverable = [d for d in deliverables - if d.matches(path=deliverable_name)][0] - LOG.debug('Download URL of %s is %s', build_id, deliverable._download_url) - return deliverable._download_url - - -def get_product(build_id): - build = get_build(build_id) - LOG.debug('Product of %s is %s.', build_id, build.product) - return build.product - - -def get_latest_build_url(branch, build_type, product='harbor_build'): - build_id = get_latest_build_id(branch, build_type, product) - print build_id - return get_ova_url(build_id) - - -def get_latest_build_id(branch, build_type, product='harbor_build'): - return buildapi.MetricResource.by_name('build', - product=product, - buildstate='succeeded', - buildtype=build_type, - branch=branch).get_max_id() - - -def get_build(build_id): - build_id, build_system = get_build_id_and_system(build_id) - return buildapi.ItemResource.by_id('build', int(build_id), build_system) - - -def get_build_version(build_id): - build = get_build(build_id) - LOG.debug('Version of %s is %s.', build_id, build.version) - return build.version \ No newline at end of file diff --git a/tests/nightly-test/utils/cmd_utils.py b/tests/nightly-test/utils/cmd_utils.py deleted file mode 100644 index fe8203e1f..000000000 --- a/tests/nightly-test/utils/cmd_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import datetime -import logging -import time - - -LOG = logging.getLogger(__name__) - - -def wait_for(func, timeout, delay, *args, **kargs): - """Decorator for waiting for until a function finished running.""" - - poll_timeout = timeout - poll_sleep_retry = delay - - begin_poll = datetime.datetime.now() - while True: - try: - return func(*args, **kargs) - break - except Exception as e: - if (datetime.datetime.now() - begin_poll).seconds > poll_timeout: - LOG.exception('Time out after %s seconds.' % poll_timeout) - raise TimeoutError('Timed out after %s seconds. Reason: ' - '%s' % (poll_timeout, e)) - else: - LOG.debug('Sleeping %s seconds before retrying' - '' % poll_sleep_retry) - time.sleep(poll_sleep_retry) - - -def safe_run(cmd, msg, sleep_time=180): - exit_code = shell.local(cmd)[0] - if exit_code: - LOG.warning('Failed to %s. Retry it after %s seconds' % - (msg, sleep_time)) - time.sleep(sleep_time) - shell.local(cmd, raise_error=True) \ No newline at end of file diff --git a/tests/nightly-test/utils/govc_utils.py b/tests/nightly-test/utils/govc_utils.py deleted file mode 100644 index 4cc582449..000000000 --- a/tests/nightly-test/utils/govc_utils.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python2 - -import os, subprocess -import time - -SHELL_SCRIPT_DIR = os.getcwd() + '/tests/nightly-test/shellscript/' - -def getvmip(vc_url, vc_user, vc_password, vm_name, timeout=600) : - cmd = (SHELL_SCRIPT_DIR+'getvmip.sh %s %s %s %s ' % (vc_url, vc_user, getPasswordInShell(vc_password), vm_name)) - interval = 10 - while True: - try: - if timeout <= 0: - return '' - result = subprocess.check_output(cmd,shell=True).strip() - if result is not '': - if result != 'photon-machine': - return result - except Exception, e: - timeout -= interval - time.sleep(interval) - continue - timeout -= interval - time.sleep(interval) - -def destoryvm(vc_url, vc_user, vc_password, vm_name) : - cmd = (SHELL_SCRIPT_DIR+'destoryvm.sh %s %s %s %s ' % (vc_url, vc_user, getPasswordInShell(vc_password), vm_name)) - result = subprocess.check_output(cmd, shell=True) - return result - -def getPasswordInShell(password) : - return password.replace("!", "\!") \ No newline at end of file diff --git a/tests/nightly-test/utils/harbor_util.py b/tests/nightly-test/utils/harbor_util.py deleted file mode 100644 index 7040c0f04..000000000 --- a/tests/nightly-test/utils/harbor_util.py +++ /dev/null @@ -1,53 +0,0 @@ -from urllib2 import urlopen -import ssl -import time -import os -try: - import json -except ImportError: - import simplejson as json - -import requests -from requests.packages.urllib3.exceptions import InsecureRequestWarning -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - -def request(harbor_endpoint, url, method, user, pwd, **kwargs): - url = "https://" + harbor_endpoint + "/api" + url - kwargs.setdefault('headers', kwargs.get('headers', {})) - kwargs['headers']['Accept'] = 'application/json' - if 'body' in kwargs: - kwargs['headers']['Content-Type'] = 'application/json' - kwargs['data'] = json.dumps(kwargs['body']) - del kwargs['body'] - - resp = requests.request(method, url, verify=False, auth=(user, pwd), **kwargs) - if resp.status_code >= 400: - raise Exception("Error: %s" % resp.text) - try: - body = json.loads(resp.text) - except ValueError: - body = resp.text - return body - -# wait for 10 minutes as OVA needs about 7 minutes to startup harbor. -def wait_for_harbor_ready(harbor_endpoint, timeout=600): - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - interval = 10 - while True: - try: - if timeout <= 0: - return False - code = urlopen(harbor_endpoint, context=ctx).code - if code == 200: - return True - except Exception, e: - timeout -= interval - time.sleep(interval) - continue - timeout -= interval - time.sleep(interval) - -def get_harbor_version(harbor_endpoint, harbor_user, harbor_pwd): - return request(harbor_endpoint, '/systeminfo', 'get', harbor_user, harbor_pwd)['harbor_version'] diff --git a/tests/nightly-test/utils/nlogging.py b/tests/nightly-test/utils/nlogging.py deleted file mode 100644 index bc17eaedb..000000000 --- a/tests/nightly-test/utils/nlogging.py +++ /dev/null @@ -1,14 +0,0 @@ -from __future__ import absolute_import - -import logging -import sys - -def create_logger(name): - default_handler = logging.StreamHandler(sys.stderr) - default_handler.setFormatter(logging.Formatter( - '[%(asctime)s] %(levelname)s in %(module)s: %(message)s' - )) - logger = logging.getLogger(name) - logger.setLevel(logging.DEBUG) - logger.addHandler(default_handler) - return logger \ No newline at end of file diff --git a/tests/nightly-test/utils/test_executor.py b/tests/nightly-test/utils/test_executor.py deleted file mode 100644 index 50286277b..000000000 --- a/tests/nightly-test/utils/test_executor.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/python2 - -import os, subprocess -import time -import sys - -from subprocess import call -import json - -import nlogging -logger = nlogging.create_logger(__name__) - -# Needs have docker installed. -def execute(harbor_endpoints, vm_names, harbor_root_pwd, test_suite, auth_mode ,vc_host, vc_user, vc_password, harbor_pwd='Harbor12345') : - cmd = '' - exe_result = -1 - cmd_base = "docker run -i --privileged -v %s:/drone -w /drone vmware/harbor-e2e-engine:1.38 " % os.getcwd() - - if len(harbor_endpoints) == 1: - cmd_pybot = "pybot -v ip:%s -v vm_name:%s -v ip1: -v HARBOR_PASSWORD:%s -v SSH_PWD:%s -v vc_host:%s -v vc_user:%s -v vc_password:%s " % (harbor_endpoints[0], vm_names[0], harbor_pwd, harbor_root_pwd, vc_host, vc_user, vc_password) - - if len(harbor_endpoints) == 2: - cmd_pybot = "pybot -v ip:%s -v vm_name:%s -v ip1:%s -v vm_name1:%s -v HARBOR_PASSWORD:%s -v SSH_PWD:%s -v vc_host:%s -v vc_user:%s -v vc_password:%s " % (harbor_endpoints[0], vm_names[0], harbor_endpoints[1], vm_names[1], harbor_pwd, harbor_root_pwd, vc_host, vc_user, vc_password) - - cmd = cmd_base + cmd_pybot - if test_suite == 'Nightly': - if auth_mode == 'ldap_auth': - cmd = cmd + "/drone/tests/robot-cases/Group11-Nightly/LDAP.robot" - else: - cmd = cmd + "/drone/tests/robot-cases/Group11-Nightly/Nightly.robot" - - logger.info(cmd) - p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE) - while True: - out = p.stderr.read(1) - if out == '' and p.poll() != None: - break - if out != '': - sys.stdout.write(out) - sys.stdout.flush() - exe_result = p.returncode - - if test_suite == 'Replication': - cmd = cmd + "/drone/tests/robot-cases/Group11-Nightly/Replication.robot" - - logger.info(cmd) - p = subprocess.Popen(cmd, shell=True, stderr=subprocess.PIPE) - while True: - out = p.stderr.read(1) - if out == '' and p.poll() != None: - break - if out != '': - sys.stdout.write(out) - sys.stdout.flush() - exe_result = p.returncode - - if test_suite == 'Longevity': - cmd = cmd + "/drone/tests/robot-cases/Group12-Longevity/Longevity.robot > /dev/null 2>&1" - logger.info(cmd) - exe_result = subprocess.call(cmd, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) - - collect_log() - return exe_result == 0 - -# Needs to move log.html to another path it will be overwrite by any pybot run. -def collect_log(): - pass \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Project-Config.robot b/tests/resources/Harbor-Pages/Project-Config.robot index be42ba3bc..045238386 100644 --- a/tests/resources/Harbor-Pages/Project-Config.robot +++ b/tests/resources/Harbor-Pages/Project-Config.robot @@ -28,6 +28,7 @@ Select Prevent Level [Arguments] ${level} Click Element //hbr-project-policy-config//select Click Element //hbr-project-policy-config//select/option[contains(.,'${level}')] + Click Auto Scan Mouse Down //hbr-project-policy-config//input[@name='scan-image-on-push'] Mouse Up //hbr-project-policy-config//input[@name='scan-image-on-push'] @@ -36,16 +37,19 @@ Save Project Config Sleep 1 Click Element //hbr-project-policy-config//button[contains(.,'SAVE')] -#assert Public Should Be Selected Checkbox Should Be Selected //hbr-project-policy-config//input[@name='public'] + Project Should Be Public [Arguments] ${projectName} Page Should Contain Element //clr-dg-row[contains(.,'${projectName}')]//clr-dg-cell[contains(.,'Public')] + Content Trust Should Be Selected Checkbox Should Be Selected //hbr-project-policy-config//input[@name='content-trust'] + Prevent Running Should Be Selected Checkbox Should Be Selected //hbr-project-policy-config//input[@name='prevent-vulnerability-image'] + Auto Scan Should Be Selected Checkbox Should Be Selected //hbr-project-policy-config//input[@name='scan-image-on-push'] diff --git a/tests/resources/Harbor-Pages/Project-Members.robot b/tests/resources/Harbor-Pages/Project-Members.robot index 5c9ce1b2f..c14400dc4 100644 --- a/tests/resources/Harbor-Pages/Project-Members.robot +++ b/tests/resources/Harbor-Pages/Project-Members.robot @@ -106,10 +106,12 @@ Add Guest Member To Project Delete Project Member [arguments] ${member} Click Element xpath=//clr-dg-row[contains(.,'${member}')]//input/../label - Click Element xpath=${project_member_delete_button_xpath} - Sleep 2 - Click Element xpath=//button[contains(.,'DELETE')] + Click Element ${member_action_xpath} + Sleep 1 + Click Element ${delete_action_xpath} Sleep 2 + Click Element //clr-modal//button[contains(.,'DELETE')] + Sleep 3 User Should Be Owner Of Project [Arguments] ${user} ${pwd} ${project} diff --git a/tests/resources/Harbor-Pages/Project-Members_Elements.robot b/tests/resources/Harbor-Pages/Project-Members_Elements.robot index 43714778e..b9cd8f2bd 100644 --- a/tests/resources/Harbor-Pages/Project-Members_Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Members_Elements.robot @@ -17,7 +17,7 @@ Documentation This resource provides any keywords related to the Harbor private *** Variables *** ${project_member_tag_xpath} /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/nav/ul/li[2]/a -${project_member_add_button_xpath} //project-detail//button[contains(.,'Member')] +${project_member_add_button_xpath} //project-detail//button[contains(.,'User')] ${project_member_add_username_xpath} //*[@id="member_name"] ${project_member_add_admin_xpath} /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/ng-component/div/div[1]/div/div[1]/add-member/clr-modal/div/div[1]/div/div[1]/div/div[2]/form/section/div[2]/div[1]/label ${project_member_add_save_button_xpath} /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/ng-component/div/div[1]/div/div[1]/add-member/clr-modal/div/div[1]/div/div[1]/div/div[3]/button[2] diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index e0a86572d..e77ea6043 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -31,7 +31,7 @@ Create An New Project Run Keyword If '${public}' == 'true' Click Element xpath=${project_public_xpath} Click Element xpath=//button[contains(.,'OK')] Sleep 4 - ${rc} ${output}= Run And Return Rc And Output curl -u %{HARBOR_ADMIN}:%{HARBOR_PASSWORD} -k -X GET --header 'Accept: application/json' ${HARBOR_URL}/api/projects?name=${projectname} + ${rc} ${output}= Run And Return Rc And Output curl -u ${HARBOR_ADMIN}:${HARBOR_PASSWORD} -k -X GET --header 'Accept: application/json' ${HARBOR_URL}/api/projects?name=${projectname} Log ${output} Should Be Equal As Integers ${rc} 0 Should Contain ${output} ${projectname} diff --git a/tests/resources/Harbor-Pages/ToolKit.robot b/tests/resources/Harbor-Pages/ToolKit.robot index 74a601426..da09dec26 100644 --- a/tests/resources/Harbor-Pages/ToolKit.robot +++ b/tests/resources/Harbor-Pages/ToolKit.robot @@ -44,6 +44,7 @@ Select Object [Arguments] ${obj} Click Element //clr-dg-row[contains(.,'${obj}')]//label +# This func cannot support as the delete user flow changed. Multi-delete Object [Arguments] @{obj} :For ${obj} in @{obj} @@ -54,12 +55,26 @@ Multi-delete Object Click Element //clr-modal//button[contains(.,'DELETE')] Sleep 3 +Multi-delete User + [Arguments] @{obj} + :For ${obj} in @{obj} + \ Click Element //clr-dg-row[contains(.,'${obj}')]//label + Sleep 1 + Click Element ${member_action_xpath} + Sleep 1 + Click Element //clr-dropdown/clr-dropdown-menu/button[2] + Sleep 2 + Click Element //clr-modal//button[contains(.,'DELETE')] + Sleep 3 + Multi-delete Member [Arguments] @{obj} :For ${obj} in @{obj} \ Click Element //clr-dg-row[contains(.,'${obj}')]//label Sleep 1 - Click Element //button[contains(.,'REMOVE')] + Click Element ${member_action_xpath} + Sleep 1 + Click Element ${delete_action_xpath} Sleep 2 Click Element //clr-modal//button[contains(.,'DELETE')] Sleep 3 diff --git a/tests/resources/Harbor-Pages/ToolKit_Elements.robot b/tests/resources/Harbor-Pages/ToolKit_Elements.robot new file mode 100644 index 000000000..b2866146b --- /dev/null +++ b/tests/resources/Harbor-Pages/ToolKit_Elements.robot @@ -0,0 +1,20 @@ +# Copyright 2016-2017 VMware, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +*** Settings *** +Documentation This resource provides any keywords related to the Harbor private registry appliance + +*** Variables *** +${member_action_xpath} //*[@id="member-action"] +${delete_action_xpath} //clr-dropdown/clr-dropdown-menu/button[4] diff --git a/tests/resources/Harbor-Pages/Vulnerability.robot b/tests/resources/Harbor-Pages/Vulnerability.robot index be2b0efde..f7970af6a 100644 --- a/tests/resources/Harbor-Pages/Vulnerability.robot +++ b/tests/resources/Harbor-Pages/Vulnerability.robot @@ -45,7 +45,7 @@ Scan Repo #select one tag Click Element //clr-dg-row[contains(.,"${tagname}")]//label Click Element //button[contains(.,'Scan')] - Sleep 5 + Sleep 20 Run Keyword If '${status}' == 'Succeed' Wait Until Element Is Visible //hbr-vulnerability-bar//hbr-vulnerability-summary-chart Run Keyword If '${status}' == 'Fail' Wait Until Element Is Visible //hbr-vulnerability-bar//a @@ -59,6 +59,7 @@ Enable Scan On Push Click Element //clr-checkbox[@name="scan-image-on-push"]//label Checkbox Should Be Selected //clr-checkbox[@name="scan-image-on-push"]//input Click Element //hbr-project-policy-config//button[contains(.,'SAVE')] + Sleep 10 Vulnerability Not Ready Project Hint Page Should Contain Element //span[@class="db-status-warning"] diff --git a/tests/resources/Nightly-Util.robot b/tests/resources/Nightly-Util.robot index eb9dff447..dae642f10 100644 --- a/tests/resources/Nightly-Util.robot +++ b/tests/resources/Nightly-Util.robot @@ -21,22 +21,17 @@ ${SSH_USER} root *** Keywords *** Nightly Test Setup - [Arguments] ${ip} ${SSH_PWD} ${HARBOR_PASSWORD} ${ip1}==${EMPTY} - Run Keyword CA setup ${ip} ${SSH_PWD} ${HARBOR_PASSWORD} - Run Keyword Prepare Docker Cert ${ip} - Run Keyword If '${ip1}' != '${EMPTY}' Run rm harbor_ca.crt - Run Keyword If '${ip1}' != '${EMPTY}' CA setup ${ip1} ${SSH_PWD} ${HARBOR_PASSWORD} - Run Keyword If '${ip1}' != '${EMPTY}' Prepare Docker Cert ${ip1} + [Arguments] ${ip} ${HARBOR_PASSWORD} ${ip1}==${EMPTY} + Run Keyword If '${ip1}' != '${EMPTY}' CA setup ${ip1} ${HARBOR_PASSWORD} /ca/ca1.crt + Run Keyword If '${ip1}' != '${EMPTY}' Run rm -rf ./harbor_ca.crt + Run Keyword CA setup ${ip} ${HARBOR_PASSWORD} Run Keyword Start Docker Daemon Locally CA Setup - [Arguments] ${ip} ${SSH_PWD} ${HARBOR_PASSWORD} - Open Connection ${ip} - Login ${SSH_USER} ${SSH_PWD} - SSHLibrary.Get File /data/ca_download/ca.crt - Close All Connections - Run mv ca.crt harbor_ca.crt - Generate Certificate Authority For Chrome ${HARBOR_PASSWORD} + [Arguments] ${ip} ${HARBOR_PASSWORD} ${cert}=/ca/ca.crt + Run mv ${cert} harbor_ca.crt + Generate Certificate Authority For Chrome ${HARBOR_PASSWORD} + Prepare Docker Cert ${ip} Collect Nightly Logs [Arguments] ${ip} ${SSH_PWD} ${ip1}==${EMPTY} diff --git a/tests/resources/SeleniumUtil.robot b/tests/resources/SeleniumUtil.robot index b65703e0f..d79a9213c 100644 --- a/tests/resources/SeleniumUtil.robot +++ b/tests/resources/SeleniumUtil.robot @@ -28,11 +28,13 @@ Init Chrome Driver Run pkill chromedriver Run pkill chrome ${chrome options}= Evaluate sys.modules['selenium.webdriver'].ChromeOptions() sys + ${capabilities}= Evaluate sys.modules['selenium.webdriver'].DesiredCapabilities.CHROME sys + Set To Dictionary ${capabilities} acceptInsecureCerts ${True} Call Method ${chrome options} add_argument --headless Call Method ${chrome options} add_argument --disable-gpu Call Method ${chrome options} add_argument --start-maximized Call Method ${chrome options} add_argument --no-sandbox Call Method ${chrome options} add_argument --window-size\=1600,900 ${chrome options.binary_location} Set Variable /usr/bin/google-chrome - Create Webdriver Chrome Chrome_headless chrome_options=${chrome options} + Create Webdriver Chrome Chrome_headless chrome_options=${chrome options} desired_capabilities=${capabilities} Sleep 5 diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index 181de5734..87b787774 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -44,6 +44,7 @@ Resource Harbor-Pages/Administration-Users_Elements.robot Resource Harbor-Pages/Configuration.robot Resource Harbor-Pages/Configuration_Elements.robot Resource Harbor-Pages/ToolKit.robot +Resource Harbor-Pages/ToolKit_Elements.robot Resource Harbor-Pages/Vulnerability.robot Resource Harbor-Pages/LDAP-Mode.robot Resource Docker-Util.robot diff --git a/tests/robot-cases/Group11-Nightly/Common.robot b/tests/robot-cases/Group11-Nightly/Common.robot new file mode 100644 index 000000000..ef261ca07 --- /dev/null +++ b/tests/robot-cases/Group11-Nightly/Common.robot @@ -0,0 +1,626 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${SSH_USER} root +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - Sign With Admin + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Close Browser + +Test Case - Vulnerability Data Not Ready +#This case must run before vulnerability db ready + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Vulnerability Not Ready Project Hint + Switch To Configure + Go To Vulnerability Config + Vulnerability Not Ready Config Hint + +Test Case - Create An New Project + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project test${d} + Close Browser + +Test Case - Delete A Project + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world + Project Should Not Be Deleted project${d} + Go Into Project project${d} + Delete Repo project${d} + Back To projects + Project Should Be Deleted project${d} + Close Browser + +Test Case - Read Only Mode + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project project${d} + + Enable Read Only + Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest + + Disable Read Only + Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest + Close Browser + +Test Case - Repo Size + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library alpine 2.6 2.6 + Go Into Project library + Go Into Repo alpine + Page Should Contain 1.92MB + Close Browser + +Test Case - Staticsinfo + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + ${privaterepocount1}= Get Statics Private Repo + ${privateprojcount1}= Get Statics Private Project + ${publicrepocount1}= Get Statics Public Repo + ${publicprojcount1}= Get Statics Public Project + ${totalrepocount1}= Get Statics Total Repo + ${totalprojcount1}= Get Statics Total Project + Create An New Project private${d} + Create An New Project public${d} true + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} private${d} hello-world + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} public${d} hello-world + Reload Page + ${privaterepocount2}= Get Statics Private Repo + ${privateprojcount2}= get statics private project + ${publicrepocount2}= get statics public repo + ${publicprojcount2}= get statics public project + ${totalrepocount2}= get statics total repo + ${totalprojcount2}= get statics total project + ${privateprojcount}= evaluate ${privateprojcount1}+1 + ${privaterepocount}= evaluate ${privaterepocount1}+1 + ${publicprojcount}= evaluate ${publicprojcount1}+1 + ${publicrepocount}= evaluate ${publicrepocount1}+1 + ${totalrepocount}= evaluate ${totalrepocount1}+2 + ${totalprojcount}= evaluate ${totalprojcount1}+2 + Should Be Equal As Integers ${privateprojcount2} ${privateprojcount} + Should be equal as integers ${privaterepocount2} ${privaterepocount} + Should be equal as integers ${publicprojcount2} ${publicprojcount} + Should be equal as integers ${publicrepocount2} ${publicrepocount} + Should be equal as integers ${totalprojcount2} ${totalprojcount} + Should be equal as integers ${totalrepocount2} ${totalrepocount} + +Test Case - Push Image + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project test${d} + + Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} test${d} hello-world:latest + Go Into Project test${d} + Wait Until Page Contains test${d}/hello-world + +Test Case - Project Level Policy Public + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project project${d} + Go Into Project project${d} + Goto Project Config + Click Project Public + Save Project Config + # Verify + Public Should Be Selected + # Project${d} default should be private + # Here logout and login to try avoid a bug only in autotest + Logout Harbor + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Filter Object project${d} + Project Should Be Public project${d} + Close Browser + +Test Case - Project Level Policy Content Trust + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest + Go Into Project project${d} + Goto Project Config + Click Content Trust + Save Project Config + # Verify + Content Trust Should Be Selected + Cannot Pull Unsigned Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest + Close Browser + +Test Case - Verify Download Ca Link + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To System Settings + Page Should Contain Registry Root Certificate + Close Browser + +Test Case - Edit Email Settings + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + + Switch To Email + Config Email + + Logout Harbor + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + + Switch To Email + Verify Email + + Close Browser + +Test Case - Edit Token Expire + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To System Settings + Modify Token Expiration 20 + Logout Harbor + + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To System Settings + Token Must Be Match 20 + + #reset to default + Modify Token Expiration 30 + Close Browser + +Test Case - Create A New Labels + Init Chrome Driver + ${d}= Get Current Date + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To System Labels + Create New Labels label_${d} + Close Browser + +Test Case - Update Label + Init Chrome Driver + ${d}= Get Current Date + + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To System Labels + Create New Labels label_${d} + Sleep 3 + ${d1}= Get Current Date + Update A Label label_${d} + Close Browser + +Test Case - Delete Label + Init Chrome Driver + ${d}= Get Current Date + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To System Labels + Create New Labels label_${d} + Sleep 3 + Delete A Label label_${d} + Close Browser + +Test Case - Disable Scan Schedule + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Configure + Go To Vulnerability Config + Disable Scan Schedule + Logout Harbor + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Configure + Go To Vulnerability Config + Page Should Contain None + Close Browser + +Test Case - User View Projects + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} user001 Test1@34 + Create An New Project test${d}1 + Create An New Project test${d}2 + Create An New Project test${d}3 + Switch To Log + Wait Until Page Contains test${d}1 + Wait Until Page Contains test${d}2 + Wait Until Page Contains test${d}3 + Close Browser + +Test Case - User View Logs + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user002 Test1@34 + Create An New Project project${d} + + Push image ${ip} user002 Test1@34 project${d} busybox:latest + Pull image ${ip} user002 Test1@34 project${d} busybox:latest + + Go Into Project project${d} + Delete Repo project${d} + + Go To Project Log + Advanced Search Should Display + + Do Log Advanced Search + Close Browser + + +Test Case - Manage Project Member + Init Chrome Driver + ${d}= Get current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user004 Test1@34 + Create An New Project project${d} + Push image ip=${ip} user=user004 pwd=Test1@34 project=project${d} image=hello-world + Logout Harbor + + User Should Not Be A Member Of Project user005 Test1@34 project${d} + Manage Project Member user004 Test1@34 project${d} user005 Add + User Should Be Guest user005 Test1@34 project${d} + Change User Role In Project user004 Test1@34 project${d} user005 Developer + User Should Be Developer user005 Test1@34 project${d} + Change User Role In Project user004 Test1@34 project${d} user005 Admin + User Should Be Admin user005 Test1@34 project${d} user006 + Manage Project Member user004 Test1@34 project${d} user005 Remove + User Should Not Be A Member Of Project user005 Test1@34 project${d} + User Should Be Guest user006 Test1@34 project${d} + + Close Browser + +Test Case - Manage project publicity + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user007 Test1@34 + Create An New Project project${d} public=true + + Push image ${ip} user007 Test1@34 project${d} hello-world:latest + Pull image ${ip} user008 Test1@34 project${d} hello-world:latest + + Logout Harbor + Sign In Harbor ${HARBOR_URL} user008 Test1@34 + Project Should Display project${d} + Search Private Projects + Project Should Not Display project${d} + + Logout Harbor + Sign In Harbor ${HARBOR_URL} user007 Test1@34 + Make Project Private project${d} + + Logout Harbor + Sign In Harbor ${HARBOR_URL} user008 Test1@34 + Project Should Not Display project${d} + Cannot Pull image ${ip} user008 Test1@34 project${d} hello-world:latest + + Logout Harbor + Sign In Harbor ${HARBOR_URL} user007 Test1@34 + Make Project Public project${d} + + Logout Harbor + Sign In Harbor ${HARBOR_URL} user008 Test1@34 + Project Should Display project${d} + Close Browser + +Test Case - Assign Sys Admin + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} user009 Test1@34 + Logout Harbor + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch to User Tag + Assign User Admin user009 + Logout Harbor + Sign In Harbor ${HARBOR_URL} user009 Test1@34 + Administration Tag Should Display + Close Browser + +Test Case - Edit Project Creation + # Create normal user and login + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} user010 Test1@34 + Project Creation Should Display + Logout Harbor + + Sleep 3 + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Set Pro Create Admin Only + Logout Harbor + + Sign In Harbor ${HARBOR_URL} user010 Test1@34 + Project Creation Should Not Display + Logout Harbor + + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Set Pro Create Every One + Close browser + +Test Case - Edit Repo Info + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user011 Test1@34 + Create An New Project project${d} + Push Image ${ip} user011 Test1@34 project${d} hello-world + Go Into Project project${d} + Go Into Repo project${d}/hello-world + Edit Repo Info + Close Browser + +Test Case - Delete Multi Project + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user012 Test1@34 + Create An New Project projecta${d} + Create An New Project projectb${d} + Push Image ${ip} user012 Test1@34 projecta${d} hello-world + Filter Object project + Multi-delete Object projecta projectb + # Verify delete project with image should not be deleted directly + Delete Fail projecta${d} + Delete Success projectb${d} + Close Browser + +Test Case - Delete Multi Repo + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user013 Test1@34 + Create An New Project project${d} + Push Image ${ip} user013 Test1@34 project${d} hello-world + Push Image ${ip} user013 Test1@34 project${d} busybox + Sleep 2 + Go Into Project project${d} + Multi-delete Object hello-world busybox + # Verify + Delete Success hello-world busybox + Close Browser + +Test Case - Delete Multi Tag + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user014 Test1@34 + Create An New Project project${d} + Push Image With Tag ${ip} user014 Test1@34 project${d} redis 3.2.10-alpine 3.2.10-alpine + Push Image With Tag ${ip} user014 Test1@34 project${d} redis 4.0.7-alpine 4.0.7-alpine + Sleep 2 + Go Into Project project${d} + Go Into Repo redis + Multi-delete object 3.2.10-alpine 4.0.7-alpine + # Verify + Delete Success 3.2.10-alpine 4.0.7-alpine + Close Browser + +Test Case - Delete Repo on CardView + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} user015 Test1@34 + Create An New Project project${d} + Push Image ${ip} user015 Test1@34 project${d} hello-world + Push Image ${ip} user015 Test1@34 project${d} busybox + Sleep 2 + Go Into Project project${d} + Switch To CardView + Delete Repo on CardView busybox + # Verify + Delete Success busybox + Close Browser + +Test Case - Delete Multi Member + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} user016 Test1@34 + Create An New Project project${d} + Go Into Project project${d} + Switch To Member + Add Guest Member To Project user017 + Add Guest Member To Project user018 + Multi-delete Member user017 user018 + Delete Success user017 user018 + Close Browser + +Test Case - Project Admin Operate Labels + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} user019 Test1@34 + Create An New Project project${d} + Go Into Project project${d} + Sleep 2 + # Add labels + Switch To Project Label + Create New Labels label_${d} + Sleep 2 + Update A Label label_${d} + Sleep 2 + Delete A Label label_${d} + Close Browser + +Test Case - Project Admin Add Labels To Repo + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Sign In Harbor ${HARBOR_URL} user020 Test1@34 + Create An New Project project${d} + Push Image With Tag ${ip} user020 Test1@34 project${d} redis 3.2.10-alpine 3.2.10-alpine + Push Image With Tag ${ip} user020 Test1@34 project${d} redis 4.0.7-alpine 4.0.7-alpine + + Go Into Project project${d} + Sleep 2 + # Add labels + Switch To Project Label + Create New Labels label111 + Create New Labels label22 + Sleep 2 + Switch To Project Repo + Go Into Repo project${d}/redis + Add Labels To Tag 3.2.10-alpine label111 + Add Labels To Tag 4.0.7-alpine label22 + Filter Labels In Tags label111 label22 + Close Browser + +Test Case - Developer Operate Labels + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user021 Test1@34 + Create An New Project project${d} + Logout Harbor + + Manage Project Member user021 Test1@34 project${d} user022 Add + Change User Role In Project user021 Test1@34 project${d} user022 Developer + + Sign In Harbor ${HARBOR_URL} user022 Test1@34 + Go Into Project project${d} + Sleep 3 + Page Should Not Contain Element xpath=//a[contains(.,'Labels')] + Close Browser + +Test Case - Scan A Tag In The Repo + Init Chrome Driver + ${d}= get current date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user023 Test1@34 + Create An New Project project${d} + + Go Into Project project${d} + Push Image ${ip} user023 Test1@34 project${d} hello-world + Go Into Project project${d} + Go Into Repo project${d}/hello-world + Scan Repo latest Succeed + Summary Chart Should Display latest + Pull Image ${ip} user023 Test1@34 project${d} hello-world + # Edit Repo Info + Close Browser + +Test Case - Scan As An Unprivileged User + Init Chrome Driver + ${d}= get current date result_format=%m%s + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world + + Sign In Harbor ${HARBOR_URL} user024 Test1@34 + Go Into Project library + Go Into Repo hello-world + Select Object latest + Scan Is Disabled + Close Browser + +Test Case - Scan Image With Empty Vul + Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Go Into Repo hello-world + Scan Repo latest Succeed + Move To Summary Chart + Wait Until Page Contains Unknow + Close Browser + +Test Case - Manual Scan All + Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library redis + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Configure + Go To Vulnerability Config + Trigger Scan Now + Back To Projects + Go Into Project library + Go Into Repo redis + Summary Chart Should Display latest + Close Browser + +Test Case - Scan Image On Push + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Goto Project Config + Enable Scan On Push + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library memcached + Back To Projects + Go Into Project library + Go Into Repo memcached + Summary Chart Should Display latest + Close Browser + +Test Case - View Scan Results + Init Chrome Driver + ${d}= get current date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user025 Test1@34 + Create An New Project project${d} + Push Image ${ip} user025 Test1@34 project${d} tomcat + Go Into Project project${d} + Go Into Repo project${d}/tomcat + Scan Repo latest Succeed + Summary Chart Should Display latest + View Repo Scan Details + Close Browser + +Test Case - View Scan Error + Init Chrome Driver + ${d}= get current date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user026 Test1@34 + Create An New Project project${d} + Push Image ${ip} user026 Test1@34 project${d} vmware/photon:1.0 + Go Into Project project${d} + Go Into Repo project${d}/vmware/photon + Scan Repo 1.0 Fail + View Scan Error Log + Close Browser + +Test Case - Project Level Image Serverity Policy + Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library haproxy + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Go Into Repo haproxy + Scan Repo latest Succeed + Back To Projects + Go Into Project library + Set Vulnerabilty Serverity 0 + Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library haproxy + Close Browser + +Test Case - Admin Push Signed Image + Enable Notary Client + + ${rc} ${output}= Run And Return Rc And Output docker pull hello-world:latest + Log ${output} + + Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world:latest + ${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group9-Content-trust/notary-push-image.sh ${ip} + Log ${output} + Should Be Equal As Integers ${rc} 0 + + ${rc} ${output}= Run And Return Rc And Output curl -u admin:Harbor12345 -s --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/repositories/library/tomcat/signatures" + Log To Console ${output} + Should Be Equal As Integers ${rc} 0 + Should Contain ${output} sha256 \ No newline at end of file diff --git a/tests/robot-cases/Group11-Nightly/DB.robot b/tests/robot-cases/Group11-Nightly/DB.robot new file mode 100644 index 000000000..7f61950a5 --- /dev/null +++ b/tests/robot-cases/Group11-Nightly/DB.robot @@ -0,0 +1,81 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${SSH_USER} root +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - Create An New User + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Create An New User url=${HARBOR_URL} username=tester${d} email=tester${d}@vmware.com realname=harbortest newPassword=Test1@34 comment=harbortest + Close Browser + +Test Case - Update User Comment + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Create An New User url=${HARBOR_URL} username=tester${d} email=tester${d}@vmware.com realname=harbortest newPassword=Test1@34 comment=harbortest + Update User Comment Test12#4 + Logout Harbor + +Test Case - Update Password + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Create An New User url=${HARBOR_URL} username=tester${d} email=tester${d}@vmware.com realname=harbortest newPassword=Test1@34 comment=harbortest + Change Password Test1@34 Test12#4 + Logout Harbor + Sign In Harbor ${HARBOR_URL} tester${d} Test12#4 + Close Browser + +Test Case - Delete Multi User + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + Create An New User ${HARBOR_URL} deletea${d} testa${d}@vmware.com test${d} Test1@34 harbor + Logout Harbor + Create An New User ${HARBOR_URL} deleteb${d} testb${d}@vmware.com test${d} Test1@34 harbor + Logout Harbor + Create An New User ${HARBOR_URL} deletec${d} testc${d}@vmware.com test${d} Test1@34 harbor + Logout Harbor + Sign In Harbor ${HARBOR_URL} admin Harbor12345 + Switch To User Tag + Filter Object delete + Multi-delete User deletea deleteb deletec + # Assert delete + Delete Success deletea deleteb deletec + Sleep 1 + Close Browser + +Test Case - Edit Self-Registration + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Disable Self Reg + Logout Harbor + + Sign Up Should Not Display + + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Configure + Self Reg Should Be Disabled + Sleep 1 + + # Restore setting + Enable Self Reg + Close Browser diff --git a/tests/robot-cases/Group11-Nightly/LDAP.robot b/tests/robot-cases/Group11-Nightly/LDAP.robot index dbb720cfb..cd961016a 100644 --- a/tests/robot-cases/Group11-Nightly/LDAP.robot +++ b/tests/robot-cases/Group11-Nightly/LDAP.robot @@ -15,8 +15,6 @@ *** Settings *** Documentation Harbor BATs Resource ../../resources/Util.robot -Suite Setup Nightly Test Setup ${ip} ${SSH_PWD} ${HARBOR_PASSWORD} ${ip1} -Suite Teardown Collect Nightly Logs ${ip} ${SSH_PWD} ${ip1} Default Tags Nightly *** Variables *** @@ -110,4 +108,4 @@ Test Case - Ldap User Push An Image Close Browser Test Case - Ldap User Can Not login - Docker Login Fail ${ip} test 123456 + Docker Login Fail ${ip} testerDeesExist 123456 \ No newline at end of file diff --git a/tests/robot-cases/Group11-Nightly/Replication.robot b/tests/robot-cases/Group11-Nightly/Replication.robot index 851a454a0..5f58272bd 100644 --- a/tests/robot-cases/Group11-Nightly/Replication.robot +++ b/tests/robot-cases/Group11-Nightly/Replication.robot @@ -15,14 +15,13 @@ *** Settings *** Documentation Harbor BATs Resource ../../resources/Util.robot -Suite Setup Nightly Test Setup ${ip} ${SSH_PWD} ${HARBOR_PASSWORD} ${ip1} -Suite Teardown Collect Nightly Logs ${ip} ${SSH_PWD} ${ip1} Default Tags Replication *** Variables *** ${HARBOR_URL} https://${ip} ${SSH_USER} root ${HARBOR_ADMIN} admin + *** Test Cases *** Test Case - Get Harbor Version #Just get harbor version and log it diff --git a/tests/robot-cases/Group11-Nightly/Setup.robot b/tests/robot-cases/Group11-Nightly/Setup.robot new file mode 100644 index 000000000..beb1ac212 --- /dev/null +++ b/tests/robot-cases/Group11-Nightly/Setup.robot @@ -0,0 +1,22 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Test Cases *** +Test Suites Setup + Nightly Test Setup ${ip} ${HARBOR_PASSWORD} ${ip1} \ No newline at end of file diff --git a/tests/robot-cases/Group11-Nightly/Teardown.robot b/tests/robot-cases/Group11-Nightly/Teardown.robot new file mode 100644 index 000000000..260471d08 --- /dev/null +++ b/tests/robot-cases/Group11-Nightly/Teardown.robot @@ -0,0 +1,22 @@ +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Test Cases *** +Test Suites Teardown + Log To Console Teardown do nothing here. \ No newline at end of file