From 56a35437b4a1c2713a9bc448fd6950448a4eef79 Mon Sep 17 00:00:00 2001 From: danfengliu Date: Fri, 4 Dec 2020 18:28:29 +0800 Subject: [PATCH] Change image source in e2e pytest (#13640) Change source of most of test image samples from docker-hub to local building ones, so it will cost less docker-hub pull requests. And some of cases like push cnab, they have to use docker-hub, but image samples in cnab test will cost 17 quotas, in this PR, we replace those samples, now cnab case will cost 6 quotas. Signed-off-by: danfengliu --- tests/apitests/python/library/docker_api.py | 31 ++++++-- tests/apitests/python/library/repository.py | 7 ++ tests/apitests/python/populate.py | 77 ------------------- tests/apitests/python/sign_image.sh | 1 + .../python/test_add_sys_label_to_tag.py | 4 +- .../test_copy_artifact_outside_project.py | 4 +- tests/apitests/python/test_del_repo.py | 4 +- .../python/test_garbage_collection.py | 1 - .../python/test_manage_project_member.py | 4 +- tests/apitests/python/test_p2p.py | 1 - ...test_project_level_policy_content_trust.py | 4 +- tests/apitests/python/test_proxy_cache.py | 1 - .../apitests/python/test_push_cnab_bundle.py | 2 +- .../test_push_index_by_docker_manifest.py | 6 +- tests/apitests/python/test_registry_api.py | 8 +- tests/apitests/python/test_robot_account.py | 20 ++--- .../python/test_scan_image_artifact.py | 11 +-- ...t_scan_image_artifact_in_public_project.py | 4 +- tests/apitests/python/test_sign_image.py | 10 +-- .../python/test_system_level_scan_all.py | 6 +- tests/apitests/python/test_tag_crud.py | 4 +- .../apitests/python/test_tag_immutability.py | 5 +- tests/apitests/python/test_user_view_logs.py | 4 +- tests/resources/Harbor-Util.robot | 2 +- tests/resources/Nightly-Util.robot | 3 + tests/robot-cases/Group1-Nightly/Common.robot | 21 ++--- tests/robot-cases/Group3-Upgrade/prepare.py | 8 +- tests/robot-cases/Group3-Upgrade/run.sh | 2 +- tests/robot-cases/Group3-Upgrade/verify.robot | 6 +- 29 files changed, 104 insertions(+), 157 deletions(-) delete mode 100644 tests/apitests/python/populate.py diff --git a/tests/apitests/python/library/docker_api.py b/tests/apitests/python/library/docker_api.py index 2f892ca4c..47ea25207 100644 --- a/tests/apitests/python/library/docker_api.py +++ b/tests/apitests/python/library/docker_api.py @@ -95,7 +95,9 @@ class DockerAPI(object): registry = None ret = "" try: + print("Docker login: {}:{}:{}".format(registry,username,password)) ret = self.DCLIENT.login(registry = registry, username=username, password=password) + print("Docker image login commond return:", ret) return ret except docker.errors.APIError as err: if expected_error_message is not None: @@ -114,7 +116,8 @@ class DockerAPI(object): expected_error_message = None ret = "" try: - self.DCLIENT.pull(r'{}:{}'.format(image, _tag)) + ret = self.DCLIENT.pull(r'{}:{}'.format(image, _tag)) + print("Docker image pull commond return:", ret) return ret except Exception as err: if expected_error_message is not None: @@ -135,19 +138,23 @@ class DockerAPI(object): _tag = base._random_name("tag") if tag is not None: _tag = tag + ret = "" try: - self.DCLIENT.tag(image, harbor_registry, _tag, force=True) + ret = self.DCLIENT.tag(image, harbor_registry, _tag, force=True) + print("Docker image tag commond return:", ret) return harbor_registry, _tag except docker.errors.APIError as err: raise Exception(r" Docker tag image {} failed, error is [{}]".format (image, str(err))) def docker_image_push(self, harbor_registry, tag, expected_error_message = None): - ret = None + ret = "" if expected_error_message is "": expected_error_message = None try: ret = self.DCLIENT.push(harbor_registry, tag) + print("Docker image push commond return:", ret) except Exception as err: + print( "docker image push catch Exception:", str(err)) if expected_error_message is not None: print( "docker image push error:", str(err)) if str(err).lower().find(expected_error_message.lower()) < 0: @@ -155,10 +162,14 @@ class DockerAPI(object): else: raise Exception(r" Docker push image {} failed, error is [{}]".format (harbor_registry, message)) else: + print( "docker image push does not catch Exception:", str(expected_error_message)) if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: raise Exception(r" Failed to catch error [{}] when push image {}, return message: {}". format (expected_error_message, harbor_registry, str(ret))) + else: + print("docker image push action return expected error message [{}]".format(expected_error_message)) + else: if str(ret).lower().find("errorDetail".lower()) >= 0: raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]". @@ -170,6 +181,7 @@ class DockerAPI(object): baseimage='busybox:latest' self.DCLIENT.login(username=DOCKER_USER, password=DOCKER_PWD) if not self.DCLIENT.images(name=baseimage): + print( "docker pull is triggered when building {}".format(harbor_registry)) self.DCLIENT.pull(baseimage) c=self.DCLIENT.create_container(image='busybox:latest',command='dd if=/dev/urandom of=test bs=1M count=%d' % size ) self.DCLIENT.start(c) @@ -184,13 +196,14 @@ class DockerAPI(object): self.DCLIENT.tag(firstrepo, repo) for tag in tags: repo="%s:%s" % (harbor_registry, tag) - self.DCLIENT.push(repo) + ret = self.DCLIENT.push(repo) + print("docker_image_push ret:", ret) print("build image %s with size %d" % (repo, size)) self.DCLIENT.remove_image(repo) self.DCLIENT.remove_container(c) - self.DCLIENT.pull(repo) - image = self.DCLIENT2.images.get(repo) - return repo, image.id + #self.DCLIENT.pull(repo) + #image = self.DCLIENT2.images.get(repo) + return repo except Exception as err: if expected_error_message is not None: print( "docker image build error:", str(err)) @@ -199,10 +212,14 @@ class DockerAPI(object): else: raise Exception(r" Docker build image {} failed, error is [{}]".format (harbor_registry, str(err))) else: + print("docker image build does not catch Exception:", str(expected_error_message)) + print("Docker build -> docker image push ret:", ret) if expected_error_message is not None: if str(ret).lower().find(expected_error_message.lower()) < 0: raise Exception(r" Failed to catch error [{}] when build image {}, return message: {}". format (expected_error_message, harbor_registry, str(ret))) + else: + print("docker image build return expected error message [{}]".format(expected_error_message)) else: if str(ret).lower().find("errorDetail".lower()) >= 0: raise Exception(r" It's was not suppose to catch error when push image {}, return message is [{}]".format (harbor_registry, ret)) diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index 873009370..3ffdf7dd2 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -38,6 +38,13 @@ def push_image_to_project(project_name, registry, username, password, image, tag return r'{}/{}'.format(project_name, image), new_tag +def push_self_build_image_to_project(project_name, registry, username, password, image, tag, size=2, expected_login_error_message = None, expected_error_message = None): + _docker_api = DockerAPI() + _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message) + + push_special_image_to_project(project_name, registry, username, password, image, tags=[tag], size=size, expected_login_error_message = expected_login_error_message, expected_error_message = expected_error_message) + return r'{}/{}'.format(project_name, image), tag + def push_special_image_to_project(project_name, registry, username, password, image, tags=None, size=1, expected_login_error_message=None, expected_error_message = None): _docker_api = DockerAPI() _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message) diff --git a/tests/apitests/python/populate.py b/tests/apitests/python/populate.py deleted file mode 100644 index f4aa49ff9..000000000 --- a/tests/apitests/python/populate.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import absolute_import - - -import unittest -import numpy -import threading -from datetime import * -from time import sleep, ctime - -import library.repository -import library.docker_api -from library.base import _assert_status_code -from testutils import ADMIN_CLIENT -from testutils import harbor_server - -from testutils import TEARDOWN -from library.project import Project -from library.repository import Repository -from library.artifact import Artifact -from library.repository import push_image_to_project -from library.repository import pull_harbor_image -from library.repository import push_special_image_to_project -import argparse - -def do_populate(name_index, repo_count): - project= Project() - artifact = Artifact() - repo = Repository() - url = ADMIN_CLIENT["endpoint"] - ADMIN_CLIENT["password"] = "qA5ZgV" - - - #2. Create a new project(PA) by user(UA); - project_name = "project"+str(name_index) - if project.check_project_name_exist(name=project_name, **ADMIN_CLIENT) is not True: - project.create_project(name=project_name, metadata = {"public": "false"}, **ADMIN_CLIENT) - print("Create Project:", project_name) - - tag = 'latest' - for repo_index in range(int(repo_count)): - repo_name = "image"+ str(repo_index) - if artifact.check_reference_exist(project_name, repo_name, tag, ignore_not_found=True, **ADMIN_CLIENT) is not True: - push_special_image_to_project(project_name, harbor_server, ADMIN_CLIENT["username"], ADMIN_CLIENT["password"], repo_name, [tag], size=repo_index*30) - print("Push Image:", repo_name) - for tag_index in numpy.arange(1, 2, 0.1): - artifact.create_tag(project_name, repo_name, tag, str(tag_index), ignore_conflict = True, **ADMIN_CLIENT) - print("Add Tag:", str(tag_index)) - - -def get_parser(): - """ return a parser """ - parser = argparse.ArgumentParser("populate") - - parser.add_argument('--project-count','-p', dest='project_count', required=False, default=100) - parser.add_argument('--repo-count','-r', dest='repo_count', required=False, default=100) - args = parser.parse_args() - return (args.project_count, args.repo_count) - -def main(): - """ main entrance """ - project_count, repo_count = get_parser() - Threads = [] - for i in range(int(project_count)): - t = threading.Thread(target=do_populate, args=(str(i), repo_count), name='T'+str(i)) - t.setDaemon(True) - Threads.append(t) - sleep(3) - for t in Threads: - t.start() - for t in Threads: - t.join() - - print('Job Finished:', ctime()) - -if __name__ == '__main__': - main() - diff --git a/tests/apitests/python/sign_image.sh b/tests/apitests/python/sign_image.sh index 6ac3325ca..668f365e9 100755 --- a/tests/apitests/python/sign_image.sh +++ b/tests/apitests/python/sign_image.sh @@ -17,5 +17,6 @@ export DOCKER_CONTENT_TRUST_OFFLINE_PASSPHRASE=$PASSHRASE export DOCKER_CONTENT_TRUST_TAGGING_PASSPHRASE=$PASSHRASE docker login -u admin -p Harbor12345 $IP +docker tag $3:$4 $IP/$2/$3:$4 docker push $IP/$2/$3:$4 diff --git a/tests/apitests/python/test_add_sys_label_to_tag.py b/tests/apitests/python/test_add_sys_label_to_tag.py index a0373a2d1..e1cc4da76 100644 --- a/tests/apitests/python/test_add_sys_label_to_tag.py +++ b/tests/apitests/python/test_add_sys_label_to_tag.py @@ -9,7 +9,7 @@ from library.artifact import Artifact from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.label import Label class TestProjects(unittest.TestCase): @@ -72,7 +72,7 @@ class TestProjects(unittest.TestCase): expected_project_id = TestProjects.project_add_g_lbl_id, **TestProjects.USER_add_g_lbl_CLIENT) #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_add_g_lbl_name, harbor_server, user_add_g_lbl_name, user_001_password, "hello-world", "latest") + TestProjects.repo_name, tag = push_self_build_image_to_project(TestProjects.project_add_g_lbl_name, harbor_server, user_add_g_lbl_name, user_001_password, "hello-world", "latest") #6. Create a new label(LA) in project(PA) by admin; TestProjects.label_id, _ = self.label.create_label(**ADMIN_CLIENT) diff --git a/tests/apitests/python/test_copy_artifact_outside_project.py b/tests/apitests/python/test_copy_artifact_outside_project.py index 8f3d282d0..89bcb21d2 100644 --- a/tests/apitests/python/test_copy_artifact_outside_project.py +++ b/tests/apitests/python/test_copy_artifact_outside_project.py @@ -11,7 +11,7 @@ from library.artifact import Artifact from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.repository import pull_harbor_image class TestProjects(unittest.TestCase): @@ -83,7 +83,7 @@ class TestProjects(unittest.TestCase): self.project.update_project_member_role(TestProjects.project_dst_repo_id, retag_member_id, 3, **ADMIN_CLIENT) #5. Create a new repository(RA) in project(PA) by user(UA); - TestProjects.src_repo_name, tag_name = push_image_to_project(TestProjects.project_src_repo_name, harbor_server, 'admin', 'Harbor12345', "hello-world", pull_tag_name) + TestProjects.src_repo_name, tag_name = push_self_build_image_to_project(TestProjects.project_src_repo_name, harbor_server, 'admin', 'Harbor12345', "hello-world", pull_tag_name) #6. Get repository in project(PA), there should be one repository which was created by user(UA); src_repo_data = self.repo.list_repositories(TestProjects.project_src_repo_name, **TestProjects.USER_RETAG_CLIENT) diff --git a/tests/apitests/python/test_del_repo.py b/tests/apitests/python/test_del_repo.py index a6d6cb5c5..d018c0a73 100644 --- a/tests/apitests/python/test_del_repo.py +++ b/tests/apitests/python/test_del_repo.py @@ -10,7 +10,7 @@ from library.base import _assert_status_code from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project class TestProjects(unittest.TestCase): @suppress_urllib3_warning @@ -54,7 +54,7 @@ class TestProjects(unittest.TestCase): TestProjects.project_del_repo_id, TestProjects.project_del_repo_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_del_repo_CLIENT) #3. Create a new repository(RA) in project(PA) by user(UA); - repo_name, _ = push_image_to_project(TestProjects.project_del_repo_name, harbor_server, 'admin', 'Harbor12345', "hello-world", "latest") + repo_name, _ = push_self_build_image_to_project(TestProjects.project_del_repo_name, harbor_server, 'admin', 'Harbor12345', "hello-world", "latest") #4. Get repository in project(PA), there should be one repository which was created by user(UA); repo_data = self.repo.list_repositories(TestProjects.project_del_repo_name, **TestProjects.USER_del_repo_CLIENT) diff --git a/tests/apitests/python/test_garbage_collection.py b/tests/apitests/python/test_garbage_collection.py index 412c5c3ff..0f1158f5a 100644 --- a/tests/apitests/python/test_garbage_collection.py +++ b/tests/apitests/python/test_garbage_collection.py @@ -10,7 +10,6 @@ from library.user import User from library.system import System from library.project import Project from library.repository import Repository -from library.repository import push_image_to_project from library.base import _assert_status_code from library.repository import push_special_image_to_project from library.artifact import Artifact diff --git a/tests/apitests/python/test_manage_project_member.py b/tests/apitests/python/test_manage_project_member.py index 19c291fd4..3203be46d 100644 --- a/tests/apitests/python/test_manage_project_member.py +++ b/tests/apitests/python/test_manage_project_member.py @@ -7,7 +7,7 @@ from testutils import TEARDOWN from testutils import ADMIN_CLIENT from library.project import Project from library.user import User -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.repository import Repository class TestProjects(unittest.TestCase): @@ -70,7 +70,7 @@ class TestProjects(unittest.TestCase): TestProjects.project_alice_id, TestProjects.project_alice_name = self.project.create_project(metadata = {"public": "false"}, **USER_ALICE_CLIENT) #2.2 Add a repository to project(PA) by Alice - TestProjects.repo_name, _ = push_image_to_project(TestProjects.project_alice_name, harbor_server, user_alice_name, user_alice_password, "hello-world", "latest") + TestProjects.repo_name, _ = push_self_build_image_to_project(TestProjects.project_alice_name, harbor_server, user_alice_name, user_alice_password, "hello-world", "latest") #3. Bob is not a member of project(PA); self.project.check_project_member_not_exist(TestProjects.project_alice_id, user_bob_name, **USER_ALICE_CLIENT) diff --git a/tests/apitests/python/test_p2p.py b/tests/apitests/python/test_p2p.py index c11b7ba20..ed5f14a78 100644 --- a/tests/apitests/python/test_p2p.py +++ b/tests/apitests/python/test_p2p.py @@ -11,7 +11,6 @@ from library.base import _assert_status_code from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project from library.registry import Registry from library.repository import pull_harbor_image from library.artifact import Artifact diff --git a/tests/apitests/python/test_project_level_policy_content_trust.py b/tests/apitests/python/test_project_level_policy_content_trust.py index e598d3c55..493d505f4 100644 --- a/tests/apitests/python/test_project_level_policy_content_trust.py +++ b/tests/apitests/python/test_project_level_policy_content_trust.py @@ -9,7 +9,7 @@ from library.artifact import Artifact from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.repository import pull_harbor_image class TestProjects(unittest.TestCase): @@ -63,7 +63,7 @@ class TestProjects(unittest.TestCase): TestProjects.project_content_trust_id, TestProjects.project_content_trust_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CONTENT_TRUST_CLIENT) #3. Push a new image(IA) in project(PA) by admin; - TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_content_trust_name, harbor_server, admin_name, admin_password, image, "latest") + TestProjects.repo_name, tag = push_self_build_image_to_project(TestProjects.project_content_trust_name, harbor_server, admin_name, admin_password, image, "latest") #4. Image(IA) should exist; artifact = self.artifact.get_reference_info(TestProjects.project_content_trust_name, image, tag, **TestProjects.USER_CONTENT_TRUST_CLIENT) diff --git a/tests/apitests/python/test_proxy_cache.py b/tests/apitests/python/test_proxy_cache.py index d8c8c59f2..5111f3306 100644 --- a/tests/apitests/python/test_proxy_cache.py +++ b/tests/apitests/python/test_proxy_cache.py @@ -13,7 +13,6 @@ from library.base import _assert_status_code from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project from library.registry import Registry from library.repository import pull_harbor_image from library.artifact import Artifact diff --git a/tests/apitests/python/test_push_cnab_bundle.py b/tests/apitests/python/test_push_cnab_bundle.py index 86854394b..8583ed788 100644 --- a/tests/apitests/python/test_push_cnab_bundle.py +++ b/tests/apitests/python/test_push_cnab_bundle.py @@ -63,7 +63,7 @@ class TestProjects(unittest.TestCase): #3. Push bundle to harbor as repository(RA); target = harbor_server + "/" + TestProjects.project_push_bundle_name + "/" + self.cnab_repo_name + ":" + self.cnab_tag - reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, user_name, self.user_push_cnab_password, "alpine:latest", "haproxy:latest", target) + reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, user_name, self.user_push_cnab_password, "photon:latest", "kong:latest", target) #4. Get repository from Harbor successfully; index_data = self.repo.get_repository(TestProjects.project_push_bundle_name, self.cnab_repo_name, **TestProjects.USER_CLIENT) diff --git a/tests/apitests/python/test_push_index_by_docker_manifest.py b/tests/apitests/python/test_push_index_by_docker_manifest.py index 3a66a1ee0..55dfdfa3b 100644 --- a/tests/apitests/python/test_push_index_by_docker_manifest.py +++ b/tests/apitests/python/test_push_index_by_docker_manifest.py @@ -14,7 +14,7 @@ from library.project import Project from library.user import User from library.repository import Repository from library.artifact import Artifact -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.repository import pull_harbor_image class TestProjects(unittest.TestCase): @@ -75,8 +75,8 @@ class TestProjects(unittest.TestCase): TestProjects.project_push_index_id, TestProjects.project_push_index_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT) #3. Create 2 new repositorys(RA,RB) in project(PA) by user(UA); - repo_name_a, tag_a = push_image_to_project(TestProjects.project_push_index_name, harbor_server, 'admin', 'Harbor12345', self.image_a, "latest") - repo_name_b, tag_b = push_image_to_project(TestProjects.project_push_index_name, harbor_server, 'admin', 'Harbor12345', self.image_b, "latest") + repo_name_a, tag_a = push_self_build_image_to_project(TestProjects.project_push_index_name, harbor_server, 'admin', 'Harbor12345', self.image_a, "latest") + repo_name_b, tag_b = push_self_build_image_to_project(TestProjects.project_push_index_name, harbor_server, 'admin', 'Harbor12345', self.image_b, "latest") #4. Push an index(IA) to Harbor by docker manifest CLI successfully; manifests = [harbor_server+"/"+repo_name_a+":"+tag_a, harbor_server+"/"+repo_name_b+":"+tag_b] diff --git a/tests/apitests/python/test_registry_api.py b/tests/apitests/python/test_registry_api.py index 95043c85c..da0be5820 100644 --- a/tests/apitests/python/test_registry_api.py +++ b/tests/apitests/python/test_registry_api.py @@ -10,7 +10,7 @@ from library.system import System from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.artifact import Artifact from library.scanner import Scanner from library.docker_api import list_image_tags @@ -73,11 +73,11 @@ class TestProjects(unittest.TestCase): src_tag = "latest" image_a = "busybox" - TestProjects.repo_a, tag_a = push_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) + TestProjects.repo_a, tag_a = push_self_build_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) image_b = "alpine" - TestProjects.repo_b, tag_b = push_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_b, src_tag) + TestProjects.repo_b, tag_b = push_self_build_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_b, src_tag) image_c = "hello-world" - TestProjects.repo_c, tag_c = push_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_c, src_tag) + TestProjects.repo_c, tag_c = push_self_build_image_to_project(TestProjects.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_c, src_tag) create_tags = ["1.0","2.0","3.0"] for tag in create_tags: self.artifact.create_tag(TestProjects.project_Alice_name, self.repo_name, tag_c, tag, **USER_ALICE_CLIENT) diff --git a/tests/apitests/python/test_robot_account.py b/tests/apitests/python/test_robot_account.py index 708694667..417f39a88 100644 --- a/tests/apitests/python/test_robot_account.py +++ b/tests/apitests/python/test_robot_account.py @@ -10,7 +10,7 @@ from library.project import Project from library.robot import Robot from library.repository import Repository from library.repository import pull_harbor_image -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.base import _assert_status_code class TestProjects(unittest.TestCase): @@ -23,10 +23,6 @@ class TestProjects(unittest.TestCase): @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") def tearDown(self): - print("Case completed") - - @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") - def test_ClearData(self): #1. Delete repository(RA) by user(UA); self.repo.delete_repoitory(TestProjects.project_ra_name_a, TestProjects.repo_name_in_project_a.split('/')[1], **TestProjects.USER_RA_CLIENT) self.repo.delete_repoitory(TestProjects.project_ra_name_b, TestProjects.repo_name_in_project_b.split('/')[1], **TestProjects.USER_RA_CLIENT) @@ -86,9 +82,9 @@ class TestProjects(unittest.TestCase): TestProjects.project_ra_id_c, TestProjects.project_ra_name_c = self.project.create_project(metadata = {"public": "true"}, **TestProjects.USER_RA_CLIENT) #3. Push image(ImagePA) to project(PA), image(ImagePB) to project(PB) and image(ImagePC) to project(PC) by user(UA); - TestProjects.repo_name_in_project_a, tag_a = push_image_to_project(TestProjects.project_ra_name_a, harbor_server, user_ra_name, user_ra_password, image_project_a, tag) - TestProjects.repo_name_in_project_b, tag_b = push_image_to_project(TestProjects.project_ra_name_b, harbor_server, user_ra_name, user_ra_password, image_project_b, tag) - TestProjects.repo_name_in_project_c, tag_c = push_image_to_project(TestProjects.project_ra_name_c, harbor_server, user_ra_name, user_ra_password, image_project_c, tag) + TestProjects.repo_name_in_project_a, tag_a = push_self_build_image_to_project(TestProjects.project_ra_name_a, harbor_server, user_ra_name, user_ra_password, image_project_a, tag) + TestProjects.repo_name_in_project_b, tag_b = push_self_build_image_to_project(TestProjects.project_ra_name_b, harbor_server, user_ra_name, user_ra_password, image_project_b, tag) + TestProjects.repo_name_in_project_c, tag_c = push_self_build_image_to_project(TestProjects.project_ra_name_c, harbor_server, user_ra_name, user_ra_password, image_project_c, tag) #4. Create a new robot account(RA) with pull and push privilege in project(PA) by user(UA); robot_id, robot_account = self.robot.create_project_robot(TestProjects.project_ra_name_a, @@ -102,10 +98,10 @@ class TestProjects(unittest.TestCase): pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, TestProjects.repo_name_in_project_a, tag_a) #7. Push image(ImageRA) to project(PA) by robot account(RA), it must be successful; - TestProjects.repo_name_pa, _ = push_image_to_project(TestProjects.project_ra_name_a, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag) + TestProjects.repo_name_pa, _ = push_self_build_image_to_project(TestProjects.project_ra_name_a, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag) #8. Push image(ImageRA) to project(PB) by robot account(RA), it must be not successful; - push_image_to_project(TestProjects.project_ra_name_b, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") + push_self_build_image_to_project(TestProjects.project_ra_name_b, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") #9. Pull image(ImagePB) from project(PB) by robot account(RA), it must be not successful; pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, TestProjects.repo_name_in_project_b, tag_b, expected_error_message = "unauthorized to access repository") @@ -114,7 +110,7 @@ class TestProjects(unittest.TestCase): pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, TestProjects.repo_name_in_project_c, tag_c) #11. Push image(ImageRA) to project(PC) by robot account(RA), it must be not successful; - push_image_to_project(TestProjects.project_ra_name_c, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") + push_self_build_image_to_project(TestProjects.project_ra_name_c, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_error_message = "unauthorized to access repository") #12. Update action property of robot account(RA);" self.robot.disable_robot_account(robot_id, True, **TestProjects.USER_RA_CLIENT) @@ -123,7 +119,7 @@ class TestProjects(unittest.TestCase): pull_harbor_image(harbor_server, robot_account.name, robot_account.secret, TestProjects.repo_name_in_project_a, tag_a, expected_login_error_message = "unauthorized: authentication required") #14. Push image(ImageRA) to project(PA) by robot account(RA), it must be not successful; - push_image_to_project(TestProjects.project_ra_name_a, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_login_error_message = "unauthorized: authentication required") + push_self_build_image_to_project(TestProjects.project_ra_name_a, harbor_server, robot_account.name, robot_account.secret, image_robot_account, tag, expected_login_error_message = "unauthorized: authentication required") #15. Delete robot account(RA), it must be not successful. self.robot.delete_robot_account(robot_id, **TestProjects.USER_RA_CLIENT) diff --git a/tests/apitests/python/test_scan_image_artifact.py b/tests/apitests/python/test_scan_image_artifact.py index 8b1301ea0..7879470b3 100644 --- a/tests/apitests/python/test_scan_image_artifact.py +++ b/tests/apitests/python/test_scan_image_artifact.py @@ -8,7 +8,7 @@ from testutils import ADMIN_CLIENT from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.artifact import Artifact from library.scan import Scan from library.sign import sign_image @@ -75,7 +75,7 @@ class TestScan(unittest.TestCase): image = "docker" src_tag = "1.13" #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - self.repo_name1, tag = push_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag) + self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag) #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; self.scan.scan_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) @@ -104,15 +104,16 @@ class TestScan(unittest.TestCase): #Note: Please make sure that this Image has never been pulled before by any other cases, # so it is a not-scanned image right after repository creation. - image = "redis" + #Note:busybox is pulled in setup phase, and setup is a essential phase. + image = "busybox" tag = "latest" #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - TestScan.repo_name_1, tag = push_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, tag) + #TestScan.repo_name_1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, tag) sign_image(harbor_server, self.project_name, image, tag) #6. Send scan image command and get tag(TA) information to check scan result, it should be finished; - self.scan.scan_artifact(self.project_name, TestScan.repo_name_1.split('/')[1], tag, **self.USER_CLIENT) + self.scan.scan_artifact(self.project_name, image, tag, **self.USER_CLIENT) self.artifact.check_image_scan_result(self.project_name, image, tag, **self.USER_CLIENT) if __name__ == '__main__': diff --git a/tests/apitests/python/test_scan_image_artifact_in_public_project.py b/tests/apitests/python/test_scan_image_artifact_in_public_project.py index 74f72eb18..3e82eaef1 100644 --- a/tests/apitests/python/test_scan_image_artifact_in_public_project.py +++ b/tests/apitests/python/test_scan_image_artifact_in_public_project.py @@ -4,7 +4,7 @@ import unittest from testutils import harbor_server, TEARDOWN, suppress_urllib3_warning from testutils import created_user, created_project from library.artifact import Artifact -from library.repository import Repository, push_image_to_project +from library.repository import Repository, push_self_build_image_to_project from library.scan import Scan @@ -41,7 +41,7 @@ class TestScanImageInPublicProject(unittest.TestCase): with created_user(password) as (user_id, username): with created_project(metadata={"public": "true"}, user_id=user_id) as (_, project_name): image, src_tag = "docker", "1.13" - full_name, tag = push_image_to_project(project_name, harbor_server, username, password, image, src_tag) + full_name, tag = push_self_build_image_to_project(project_name, harbor_server, username, password, image, src_tag) repo_name = full_name.split('/')[1] diff --git a/tests/apitests/python/test_sign_image.py b/tests/apitests/python/test_sign_image.py index 8fa410eb4..46842caa0 100644 --- a/tests/apitests/python/test_sign_image.py +++ b/tests/apitests/python/test_sign_image.py @@ -9,7 +9,6 @@ from library.artifact import Artifact from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project from library.repository import push_special_image_to_project class TestProjects(unittest.TestCase): @@ -66,10 +65,11 @@ class TestProjects(unittest.TestCase): self.project.projects_should_exist(dict(public=False), expected_count = 1, expected_project_id = TestProjects.project_sign_image_id, **TestProjects.USER_sign_image_CLIENT) - image = "hello-world" - src_tag = "latest" + #Note:busybox is pulled in setup phase, and setup is a essential phase. + image = "busybox" + tag = "latest" #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); - TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_sign_image_name, harbor_server, user_sign_image_name, user_001_password, image, src_tag) + #TestProjects.repo_name, tag = push_self_build_image_to_project(TestProjects.project_sign_image_name, harbor_server, user_sign_image_name, user_001_password, image, src_tag) #6. Sign image with tag(TA) which was tagged by step #5; sign_image(harbor_server, TestProjects.project_sign_image_name, image, tag) @@ -81,7 +81,7 @@ class TestProjects(unittest.TestCase): push_special_image_to_project(TestProjects.project_sign_image_name, harbor_server, user_sign_image_name, user_001_password, self.repo_name_1, ['1.0']) self.repo.delete_repoitory(TestProjects.project_sign_image_name, self.repo_name_1, **TestProjects.USER_sign_image_CLIENT) - self.repo.delete_repoitory(TestProjects.project_sign_image_name, TestProjects.repo_name.split('/')[1], expect_status_code=412, expect_response_body = "with signature cannot be deleted", **TestProjects.USER_sign_image_CLIENT) + self.repo.delete_repoitory(TestProjects.project_sign_image_name, image, expect_status_code=412, expect_response_body = "with signature cannot be deleted", **TestProjects.USER_sign_image_CLIENT) if __name__ == '__main__': unittest.main() diff --git a/tests/apitests/python/test_system_level_scan_all.py b/tests/apitests/python/test_system_level_scan_all.py index 07acf0909..dc3822d23 100644 --- a/tests/apitests/python/test_system_level_scan_all.py +++ b/tests/apitests/python/test_system_level_scan_all.py @@ -8,7 +8,7 @@ from library.system import System from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.artifact import Artifact class TestScanAll(unittest.TestCase): @suppress_urllib3_warning @@ -71,14 +71,14 @@ class TestScanAll(unittest.TestCase): image_a = "mariadb" src_tag = "latest" #3.1 Push a image to project_Alice; - TestScanAll.repo_Alice_name, tag_Alice = push_image_to_project(TestScanAll.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) + TestScanAll.repo_Alice_name, tag_Alice = push_self_build_image_to_project(TestScanAll.project_Alice_name, harbor_server, user_Alice_name, user_common_password, image_a, src_tag) #Note: Please make sure that this Image has never been pulled before by any other cases, # so it is a not-scanned image rigth after repository creation. image_b = "httpd" src_tag = "latest" #3.2 push another image to project_Luca; - TestScanAll.repo_Luca_name, tag_Luca = push_image_to_project(TestScanAll.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag) + TestScanAll.repo_Luca_name, tag_Luca = push_self_build_image_to_project(TestScanAll.project_Luca_name, harbor_server, user_Luca_name, user_common_password, image_b, src_tag) #4. Trigger scan all event; self.system.scan_now(**ADMIN_CLIENT) diff --git a/tests/apitests/python/test_tag_crud.py b/tests/apitests/python/test_tag_crud.py index d73d620a9..f697eca17 100644 --- a/tests/apitests/python/test_tag_crud.py +++ b/tests/apitests/python/test_tag_crud.py @@ -13,7 +13,7 @@ from library.project import Project from library.user import User from library.repository import Repository from library.artifact import Artifact -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from library.repository import pull_harbor_image class TestProjects(unittest.TestCase): @@ -67,7 +67,7 @@ class TestProjects(unittest.TestCase): TestProjects.project_id, TestProjects.project_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_CLIENT) #3. Push an image(IA) to Harbor by docker successfully; - repo_name, tag = push_image_to_project(TestProjects.project_name, harbor_server, 'admin', 'Harbor12345', self.repo_name, "latest") + repo_name, tag = push_self_build_image_to_project(TestProjects.project_name, harbor_server, 'admin', 'Harbor12345', self.repo_name, "latest") #4. Create a tag(1.0) for the image(IA) self.artifact.create_tag(TestProjects.project_name, self.repo_name, tag, "1.0",**TestProjects.USER_CLIENT) diff --git a/tests/apitests/python/test_tag_immutability.py b/tests/apitests/python/test_tag_immutability.py index 5619d2b26..ff09ccd07 100644 --- a/tests/apitests/python/test_tag_immutability.py +++ b/tests/apitests/python/test_tag_immutability.py @@ -9,7 +9,6 @@ from testutils import harbor_server from library.project import Project from library.user import User from library.repository import Repository -from library.repository import push_image_to_project from library.registry import Registry from library.artifact import Artifact from library.tag_immutability import Tag_Immutability @@ -180,8 +179,8 @@ class TestTagImmutability(unittest.TestCase): self.check_tag_immutability(artifact_a, image_a["tag2"], status = False) #5. Can not push image with the same image name and with the same tag name. - push_image_to_project(project_name, harbor_server, self.user_name, self.user_password, "tomcat", image_a["tag1"], - new_image = image_a["name"], expected_error_message = "configured as immutable") + push_special_image_to_project(project_name, harbor_server, self.user_name, self.user_password, image_a["name"], [image_a["tag1"]], size=10 + , expected_error_message = "configured as immutable") def test_copy_disability(self): """ diff --git a/tests/apitests/python/test_user_view_logs.py b/tests/apitests/python/test_user_view_logs.py index 64f09807d..d1b70b75d 100644 --- a/tests/apitests/python/test_user_view_logs.py +++ b/tests/apitests/python/test_user_view_logs.py @@ -10,7 +10,7 @@ from library.user import User from library.projectV2 import ProjectV2 from library.project import Project from library.repository import Repository -from library.repository import push_image_to_project +from library.repository import push_self_build_image_to_project from testutils import harbor_server class TestProjects(unittest.TestCase): @@ -66,7 +66,7 @@ class TestProjects(unittest.TestCase): format(user_user_view_logs_name, project_user_view_logs_name, "project", operation, log_count)) #3.1 Push a new image(IA) in project(PA) by admin; - repo_name, tag = push_image_to_project(project_user_view_logs_name, harbor_server, admin_name, admin_password, "tomcat", "latest") + repo_name, tag = push_self_build_image_to_project(project_user_view_logs_name, harbor_server, admin_name, admin_password, "tomcat", "latest") time.sleep(2) #3.2 In project(PA), there should be 1 'push' log record; diff --git a/tests/resources/Harbor-Util.robot b/tests/resources/Harbor-Util.robot index da3779cb4..cdb207aeb 100644 --- a/tests/resources/Harbor-Util.robot +++ b/tests/resources/Harbor-Util.robot @@ -93,7 +93,7 @@ Switch To LDAP Enable Notary Client ${rc} ${output}= Run And Return Rc And Output rm -rf ~/.docker/ Log ${rc} - ${rc} ${output}= Run And Return Rc and Output curl -o /notary_ca.crt -s -k -X GET --header 'Accept: application/json' -u 'admin:Harbor12345' 'https://${ip}/api/v2.0/systeminfo/getcert' + ${rc} ${output}= Run And Return Rc and Output curl -o /notary_ca.crt -s -k -X GET -u 'admin:Harbor12345' 'https://${ip}/api/v2.0/systeminfo/getcert' Log ${output} Should Be Equal As Integers ${rc} 0 diff --git a/tests/resources/Nightly-Util.robot b/tests/resources/Nightly-Util.robot index 9cb467d92..f17607bf9 100644 --- a/tests/resources/Nightly-Util.robot +++ b/tests/resources/Nightly-Util.robot @@ -32,6 +32,9 @@ Nightly Test Setup Run Keyword Start Containerd Daemon Locally Log To Console wget mariadb ... Run wget ${prometheus_chart_file_url} + #Prepare docker image for push special image keyword in replication test + Run Keyword If '${DOCKER_USER}' != '${EMPTY}' Docker Login "" ${DOCKER_USER} ${DOCKER_PWD} + Docker Pull busybox:latest CA Setup [Arguments] ${ip} ${HARBOR_PASSWORD} ${cert}=/ca/ca.crt diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 9eef9be60..dd28042e6 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -587,16 +587,17 @@ Test Case - Tag Immutability Delete Success busybox Close Browser -Test Case - Robot Account - Init Chrome Driver - ${d}= Get Current Date result_format=%m%s - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project And Go Into Project project${d} - ${token}= Create A Robot Account And Return Token project${d} robot${d} - Log To Console ${token} - Log ${token} - Push image ${ip} robot${d} ${token} project${d} hello-world:latest is_robot=${true} - Pull image ${ip} robot${d} ${token} project${d} hello-world:latest is_robot=${true} +#TODO in 2.2: Modify this case when new robot account feature is ready. +#Test Case - Robot Account +# Init Chrome Driver +# ${d}= Get Current Date result_format=%m%s +# Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} +# Create An New Project And Go Into Project project${d} +# ${token}= Create A Robot Account And Return Token project${d} robot${d} +# Log To Console ${token} +# Log ${token} +# Push image ${ip} robot${d} ${token} project${d} hello-world:latest is_robot=${true} +# Pull image ${ip} robot${d} ${token} project${d} hello-world:latest is_robot=${true} Test Case - Push Docker Manifest Index and Display Init Chrome Driver diff --git a/tests/robot-cases/Group3-Upgrade/prepare.py b/tests/robot-cases/Group3-Upgrade/prepare.py index 95d9ad3df..7fb27c5fa 100644 --- a/tests/robot-cases/Group3-Upgrade/prepare.py +++ b/tests/robot-cases/Group3-Upgrade/prepare.py @@ -19,8 +19,8 @@ from os import path sys.path.append(args.libpath) sys.path.append(args.libpath + "/library") from library.docker_api import docker_manifest_push_to_harbor -from library.repository import push_image_to_project from library.repository import Repository +from library.repository import push_self_build_image_to_project url = "https://"+args.endpoint+"/api/" endpoint_url = "https://"+args.endpoint @@ -577,8 +577,8 @@ class HarborAPI: def push_artifact_index(self, project, name, tag, **kwargs): image_a = "alpine" image_b = "busybox" - repo_name_a, tag_a = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_a, "latest") - repo_name_b, tag_b = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_b, "latest") + repo_name_a, tag_a = push_self_build_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_a, "latest") + repo_name_b, tag_b = push_self_build_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_b, "latest") manifests = [args.endpoint+"/"+repo_name_a+":"+tag_a, args.endpoint+"/"+repo_name_b+":"+tag_b] index = args.endpoint+"/"+project+"/"+name+":"+tag docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh") @@ -647,7 +647,7 @@ def do_data_creation(): harborAPI.push_artifact_index(data["projects"][0]["name"], data["projects"][0]["artifact_index"]["name"], data["projects"][0]["artifact_index"]["tag"], version=args.version) #pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2") - push_image_to_project(data["projects"][0]["name"], args.endpoint, 'admin', 'Harbor12345', "busybox", "latest") + push_self_build_image_to_project(data["projects"][0]["name"], args.endpoint, 'admin', 'Harbor12345', "busybox", "latest") push_signed_image("alpine", data["projects"][0]["name"], "latest") for replicationrule in data["replicationrule"]: diff --git a/tests/robot-cases/Group3-Upgrade/run.sh b/tests/robot-cases/Group3-Upgrade/run.sh index 142f0274c..7b1127a2a 100755 --- a/tests/robot-cases/Group3-Upgrade/run.sh +++ b/tests/robot-cases/Group3-Upgrade/run.sh @@ -4,6 +4,6 @@ HARBOR_VERSION=$2 DOCKER_USER=$3 DOCKER_PWD=$4 -robot -v ip:$IP -v ip1: -v HARBOR_PASSWORD:Harbor12345 /drone/tests/robot-cases/Group1-Nightly/Setup.robot +robot -v ip:$IP -v ip1: -v HARBOR_PASSWORD:Harbor12345 -v DOCKER_USER:$DOCKER_USER -v DOCKER_PWD:$DOCKER_PWD /drone/tests/robot-cases/Group1-Nightly/Setup.robot cd /drone/tests/robot-cases/Group3-Upgrade DOCKER_USER=$DOCKER_USER DOCKER_PWD=$DOCKER_PWD python ./prepare.py -e $IP -v $HARBOR_VERSION -l /drone/tests/apitests/python/ diff --git a/tests/robot-cases/Group3-Upgrade/verify.robot b/tests/robot-cases/Group3-Upgrade/verify.robot index fa28db9cb..52b48a110 100644 --- a/tests/robot-cases/Group3-Upgrade/verify.robot +++ b/tests/robot-cases/Group3-Upgrade/verify.robot @@ -73,7 +73,8 @@ Test Case - Upgrade Verify Run Keyword Verify Project Metadata ${data} check_content_trust=${false} #Run Keyword Verify Project Label ${data} Run Keyword Verify Member Exist ${data} - Run Keyword Verify Robot Account Exist ${data} + #TODO in 2.2: Modify this case when new robot account feature is ready. + #Run Keyword Verify Robot Account Exist ${data} Run Keyword Verify Project-level Allowlist ${data} Run Keyword Verify Webhook For 2.0 ${data} Run Keyword Verify Tag Retention Rule ${data} @@ -96,7 +97,8 @@ Test Case - Upgrade Verify Run Keyword Verify Project Metadata ${data} check_content_trust=${false} verify_registry_name=${true} #Run Keyword Verify Project Label ${data} verify_registry_name=${true} Run Keyword Verify Member Exist ${data} verify_registry_name=${true} - Run Keyword Verify Robot Account Exist ${data} verify_registry_name=${true} + ##TODO in 2.2: Modify this case when new robot account feature is ready. + #Run Keyword Verify Robot Account Exist ${data} verify_registry_name=${true} Run Keyword Verify Project-level Allowlist ${data} verify_registry_name=${true} Run Keyword Verify Webhook For 2.0 ${data} verify_registry_name=${true} Run Keyword Verify Tag Retention Rule ${data} verify_registry_name=${true}