diff --git a/tests/apitests/python/library/cnab.py b/tests/apitests/python/library/cnab.py index d93f516b0..835d002a8 100644 --- a/tests/apitests/python/library/cnab.py +++ b/tests/apitests/python/library/cnab.py @@ -43,6 +43,7 @@ def cnab_push_bundle(bundle_file, target): raise Exception(r"Fail to get sha256 in returned data: {}".format(ret)) def push_cnab_bundle(harbor_server, user, password, service_image, invocation_image, target, auto_update_bundle = True): + docker_api.docker_info_display() docker_api.docker_login_cmd(harbor_server, user, password, enable_manifest = False) bundle_file = load_bundle(service_image, invocation_image) fixed_bundle_file = cnab_fixup_bundle(bundle_file, target, auto_update_bundle = auto_update_bundle) diff --git a/tests/apitests/python/library/containerd.py b/tests/apitests/python/library/containerd.py new file mode 100644 index 000000000..1517f169f --- /dev/null +++ b/tests/apitests/python/library/containerd.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +import base +import json +import docker_api + +def ctr_images_pull(username, password, oci): + command = ["sudo", "ctr", "images", "pull", "-u", username+":"+password, oci] + print "Command: ", command + ret = base.run_command(command) + print "Command return: ", ret + +def ctr_images_list(oci_ref = None): + command = ["sudo", "ctr", "images", "list", "--q"] + print "Command: ", command + ret = base.run_command(command) + print "Command return: ", ret + if oci_ref is not None and oci_ref not in ret.split("\n"): + raise Exception(r" Get OCI ref failed, expected ref is [{}], but return ref list is [{}]".format (ret)) + + diff --git a/tests/apitests/python/library/docker_api.py b/tests/apitests/python/library/docker_api.py index b60dfc6cf..252791375 100644 --- a/tests/apitests/python/library/docker_api.py +++ b/tests/apitests/python/library/docker_api.py @@ -11,6 +11,12 @@ except ImportError: pip.main(['install', 'docker']) import docker +def docker_info_display(): + command = ["docker", "info", "-f", "'{{.OSType}}/{{.Architecture}}'"] + print "Docker Info: ", command + ret = base.run_command(command) + print "Command return: ", ret + def docker_login_cmd(harbor_host, user, password, enable_manifest = True): command = ["sudo", "docker", "login", harbor_host, "-u", user, "-p", password] print "Docker Login Command: ", command diff --git a/tests/apitests/python/library/repository.py b/tests/apitests/python/library/repository.py index 2ab8ae9e9..6df04a660 100644 --- a/tests/apitests/python/library/repository.py +++ b/tests/apitests/python/library/repository.py @@ -15,7 +15,7 @@ def pull_harbor_image(registry, username, password, image, tag, expected_login_e ret = _docker_api.docker_image_pull(r'{}/{}'.format(registry, image), tag = tag, expected_error_message = expected_error_message) print ret -def push_image_to_project(project_name, registry, username, password, image, tag, expected_login_error_message = None, expected_error_message = None): +def push_image_to_project(project_name, registry, username, password, image, tag, expected_login_error_message = None, expected_error_message = None, profix_for_image = None): _docker_api = DockerAPI() _docker_api.docker_login(registry, username, password, expected_error_message = expected_login_error_message) time.sleep(2) @@ -24,7 +24,10 @@ def push_image_to_project(project_name, registry, username, password, image, tag _docker_api.docker_image_pull(image, tag = tag) time.sleep(2) - new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(image, tag), r'{}/{}/{}'.format(registry, project_name, image)) + if profix_for_image == None: + new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(image, tag), r'{}/{}/{}'.format(registry, project_name, image)) + else: + new_harbor_registry, new_tag = _docker_api.docker_image_tag(r'{}:{}'.format(image, tag), r'{}/{}/{}/{}'.format(registry, project_name, profix_for_image, image)) time.sleep(2) _docker_api.docker_image_push(new_harbor_registry, new_tag, expected_error_message = expected_error_message) diff --git a/tests/apitests/python/library/retention.py b/tests/apitests/python/library/retention.py index 90f9f19d7..8c3bb7eaa 100644 --- a/tests/apitests/python/library/retention.py +++ b/tests/apitests/python/library/retention.py @@ -134,6 +134,7 @@ class Retention(base.Base): { "kind": "doublestar", "decoration": "matches", + "extras":'["untagged":True]', "pattern": selector_tag } ] diff --git a/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py b/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py index 2ba5a6ed9..2e7c653f8 100644 --- a/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py +++ b/tests/apitests/python/test_push_chart_by_helm3_chart_cli.py @@ -76,11 +76,16 @@ class TestProjects(unittest.TestCase): self.assertEqual(artifacts[0].type, 'CHART') self.assertEqual(artifacts[0].tags[0].name, self.verion) - #5. Get chart(CA) by reference successfully; + #5.1 Get chart(CA) by reference successfully; artifact = self.artifact.get_reference_info(TestProjects.project_push_chart_name, self.repo_name, self.verion, **TestProjects.USER_CLIENT) self.assertEqual(artifact[0].type, 'CHART') self.assertEqual(artifact[0].tags[0].name, self.verion) + #5.2 Chart bundle can be pulled by ctr successfully; + #oci_ref = harbor_server+"/"+TestProjects.project_push_chart_name+"/"+self.repo_name+":"+self.verion + #library.containerd.ctr_images_pull(user_name, self.user_push_chart_password, oci_ref) + #library.containerd.ctr_images_list(oci_ref = oci_ref) + #6. Get addtion successfully; addition_r = self.artifact.get_addition(TestProjects.project_push_chart_name, self.repo_name, self.verion, "readme.md", **TestProjects.USER_CLIENT) self.assertIn("Helm Chart for Harbor", addition_r[0]) diff --git a/tests/apitests/python/test_push_cnab_bundle.py b/tests/apitests/python/test_push_cnab_bundle.py index 3f815779d..3a28c006a 100644 --- a/tests/apitests/python/test_push_cnab_bundle.py +++ b/tests/apitests/python/test_push_cnab_bundle.py @@ -5,6 +5,7 @@ import unittest import library.repository import library.cnab + from testutils import ADMIN_CLIENT from testutils import harbor_server @@ -23,8 +24,9 @@ class TestProjects(unittest.TestCase): self.artifact = Artifact() self.repo= Repository() self.url = ADMIN_CLIENT["endpoint"] - self.user_push_chart_password = "Aa123456" + self.user_push_cnab_password = "Aa123456" self.cnab_repo_name = "test_cnab" + self.cnab_tag = "test_cnab_tag" @classmethod def tearDownClass(self): @@ -60,8 +62,8 @@ class TestProjects(unittest.TestCase): 3. Delete user(UA). """ #1. Create a new user(UA); - TestProjects.user_id, user_name = self.user.create_user(user_password = self.user_push_chart_password, **ADMIN_CLIENT) - TestProjects.USER_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_chart_password) + TestProjects.user_id, user_name = self.user.create_user(user_password = self.user_push_cnab_password, **ADMIN_CLIENT) + TestProjects.USER_CLIENT=dict(endpoint = self.url, username = user_name, password = self.user_push_cnab_password) #2. Create a new project(PA) by user(UA); @@ -69,17 +71,23 @@ class TestProjects(unittest.TestCase): #3. Pull images for bundle; _docker_api = DockerAPI() - _docker_api.docker_image_pull("hello-world", tag = "latest") - _docker_api.docker_image_pull("busybox", tag = "latest") + _docker_api.docker_image_pull("alpine", tag = "latest") + _docker_api.docker_image_pull("haproxy", tag = "latest") #4. Push bundle to harbor as repository(RA); - target = harbor_server + "/" + TestProjects.project_push_bundle_name + "/" + self.cnab_repo_name - reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, user_name, self.user_push_chart_password, "hello-world:latest", "busybox:latest", target) + target = harbor_server + "/" + TestProjects.project_push_bundle_name + "/" + self.cnab_repo_name + ":" + self.cnab_tag + reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, user_name, self.user_push_cnab_password, "alpine:latest", "haproxy:latest", target) #5. Get repository from Harbor successfully; index_data = self.repo.get_repository(TestProjects.project_push_bundle_name, self.cnab_repo_name, **TestProjects.USER_CLIENT) print "index_data:", index_data + #5.2 Cnab bundle can be pulled by ctr successfully; + # This step might not successful since ctr does't support cnab fully, it might be uncomment sometime in future. + # Please keep them in comment! + #library.containerd.ctr_images_pull(user_name, self.user_push_cnab_password, target) + #library.containerd.ctr_images_list(oci_ref = target) + #6. Verfiy bundle name; self.assertEqual(index_data.name, TestProjects.project_push_bundle_name + "/" + self.cnab_repo_name) diff --git a/tests/apitests/python/test_push_image_with_special_name.py b/tests/apitests/python/test_push_image_with_special_name.py new file mode 100644 index 000000000..3d5955c9b --- /dev/null +++ b/tests/apitests/python/test_push_image_with_special_name.py @@ -0,0 +1,87 @@ +from __future__ import absolute_import +import unittest +import urllib + +from library.sign import sign_image +from testutils import ADMIN_CLIENT +from testutils import harbor_server +from testutils import TEARDOWN +from library.artifact import Artifact +from library.project import Project +from library.user import User +from library.repository import Repository +from library.repository import push_image_to_project + +class TestProjects(unittest.TestCase): + @classmethod + def setUp(self): + self.project = Project() + self.user = User() + self.artifact = Artifact() + self.repo = Repository() + + @classmethod + def tearDown(self): + print "Case completed" + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def test_ClearData(self): + # remove the deletion as the signed image cannot be deleted. + #1. Delete repository(RA) by user(UA); + #self.repo.delete_repoitory(TestProjects.project_sign_image_name, TestProjects.repo_name.split('/')[1], **TestProjects.USER_sign_image_CLIENT) + + #2. Delete project(PA); + #self.project.delete_project(TestProjects.project_sign_image_id, **TestProjects.USER_sign_image_CLIENT) + + #3. Delete user(UA); + self.user.delete_user(TestProjects.user_sign_image_id, **ADMIN_CLIENT) + + def testSignImage(self): + """ + Test case: + Push Image With Special Name + Test step and expected result: + 1. Create a new user(UA); + 2. Create a new private project(PA) by user(UA); + 3. Add user(UA) as a member of project(PA) with project-admin role; + 4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + 5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + 6. Sign image with tag(TA) which was tagged by step #5; + 7. Get signature of image with tag(TA), it should be exist. + Tear down: + NA + """ + url = ADMIN_CLIENT["endpoint"] + user_001_password = "Aa123456" + + #1. Create user-001 + TestProjects.user_sign_image_id, user_sign_image_name = self.user.create_user(user_password = user_001_password, **ADMIN_CLIENT) + + TestProjects.USER_sign_image_CLIENT=dict(with_signature = True, endpoint = url, username = user_sign_image_name, password = user_001_password) + + #2. Create a new private project(PA) by user(UA); + TestProjects.project_sign_image_id, TestProjects.project_sign_image_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) + + #3. Add user(UA) as a member of project(PA) with project-admin role; + self.project.add_project_members(TestProjects.project_sign_image_id, TestProjects.user_sign_image_id, **ADMIN_CLIENT) + + #4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + self.project.projects_should_exist(dict(public=False), expected_count = 1, + expected_project_id = TestProjects.project_sign_image_id, **TestProjects.USER_sign_image_CLIENT) + + image = "hello-world" + src_tag = "latest" + profix = "aaa/bbb" + + #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + TestProjects.repo_name, tag = push_image_to_project(TestProjects.project_sign_image_name, harbor_server, user_sign_image_name, user_001_password, image, src_tag, profix_for_image=profix) + + #7. Get signature of image with tag(TA), it should be exist. + full_name = urllib.quote(profix+"/"+image,'utf-8') + artifact = self.artifact.get_reference_info(TestProjects.project_sign_image_name, (str(full_name)).encode(), tag, **TestProjects.USER_sign_image_CLIENT) + + print artifact + self.assertEqual(artifact[0].type, 'IMAGE') + +if __name__ == '__main__': + unittest.main() diff --git a/tests/apitests/python/test_push_index_by_docker_manifest.py b/tests/apitests/python/test_push_index_by_docker_manifest.py index a8a98903a..8e371bb86 100644 --- a/tests/apitests/python/test_push_index_by_docker_manifest.py +++ b/tests/apitests/python/test_push_index_by_docker_manifest.py @@ -5,6 +5,7 @@ import unittest import library.repository import library.docker_api +import library.containerd from library.base import _assert_status_code from testutils import ADMIN_CLIENT from testutils import harbor_server @@ -60,7 +61,8 @@ class TestProjects(unittest.TestCase): 5. Get Artifacts successfully; 6. Get index(IA) by reference successfully; 7. Verify harbor index is index(IA) pushed by docker manifest CLI; - 8. Verify harbor index(IA) can be pulled by docker CLI successfully; + 8.1 Verify harbor index(IA) can be pulled by docker CLI successfully; + 8.2 Verify harbor index(IA) can be pulled by docker CLI successfully; 9. Get addition successfully; 10. Unable to Delete artifact in manifest list; 11. Delete index successfully. @@ -101,9 +103,14 @@ class TestProjects(unittest.TestCase): self.assertEqual(manifests_sha256_harbor_ret.count(manifests_sha256_cli_ret[0]), 1) self.assertEqual(manifests_sha256_harbor_ret.count(manifests_sha256_cli_ret[1]), 1) - #8. Verify harbor index(IA) can be pulled by docker CLI successfully; + #8.1 Verify harbor index(IA) can be pulled by docker CLI successfully; pull_harbor_image(harbor_server, user_name, self.user_push_index_password, TestProjects.project_push_index_name+"/"+self.index_name, self.index_tag) + #8.2 Verify harbor index(IA) can be pulled by ctr successfully; + oci_ref = harbor_server+"/"+TestProjects.project_push_index_name+"/"+self.index_name+":"+self.index_tag + library.containerd.ctr_images_pull(user_name, self.user_push_index_password, oci_ref) + library.containerd.ctr_images_list(oci_ref = oci_ref) + #9. Get addition successfully; addition_v = self.artifact.get_addition(TestProjects.project_push_index_name, self.index_name, self.index_tag, "vulnerabilities", **TestProjects.USER_CLIENT) self.assertEqual(addition_v[0], '{}') diff --git a/tests/apitests/python/test_retention.py b/tests/apitests/python/test_retention.py index b8b502ffd..de452e1c2 100644 --- a/tests/apitests/python/test_retention.py +++ b/tests/apitests/python/test_retention.py @@ -7,13 +7,14 @@ from testutils import ADMIN_CLIENT from testutils import TEARDOWN from testutils import harbor_server from library.repository import push_special_image_to_project +from library.docker_api import list_image_tags from library.retention import Retention from library.project import Project from library.repository import Repository from library.user import User from library.system import System - +from library.artifact import Artifact class TestProjects(unittest.TestCase): """ @@ -38,6 +39,8 @@ class TestProjects(unittest.TestCase): self.repo = Repository() self.project = Project() self.retention = Retention() + self.artifact = Artifact() + self.repo_name_1 = "test1" def testTagRetention(self): user_ra_password = "Aa123456" @@ -51,14 +54,20 @@ class TestProjects(unittest.TestCase): TestProjects.project_src_repo_id, TestProjects.project_src_repo_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_RA_CLIENT) # Push image test1:1.0, test1:2.0, test1:3.0,latest, test2:1.0, test2:latest, test3:1.0 - push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['1.0']) - push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['2.0']) - push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test1", ['3.0','latest']) + push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['1.0']) + push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['2.0']) + push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['3.0','latest']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['1.0']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['latest']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test3", ['1.0']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test4", ['1.0']) + tags = list_image_tags(harbor_server, TestProjects.project_src_repo_name+"/"+self.repo_name_1, user_ra_name, user_ra_password) + #Delete all tags of "artifact3" in repostory "image1"; + self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "latest",**TestProjects.USER_RA_CLIENT) + self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "3.0",**TestProjects.USER_RA_CLIENT) + tags = list_image_tags(harbor_server, TestProjects.project_src_repo_name+"/"+self.repo_name_1, user_ra_name, user_ra_password) + resp=self.repo.list_repositories(TestProjects.project_src_repo_name, **TestProjects.USER_RA_CLIENT) self.assertEqual(len(resp), 4) @@ -77,7 +86,13 @@ class TestProjects(unittest.TestCase): resp=self.retention.get_retention_exec_tasks(retention_id,execution.id, **TestProjects.USER_RA_CLIENT) self.assertEqual(len(resp), 4) resp=self.retention.get_retention_exec_task_log(retention_id,execution.id,resp[0].id, **TestProjects.USER_RA_CLIENT) - print(resp) + #For Debug: + print("Task 0 log begin:-----------------------------") + i=0 + for line in resp.split("\n"): + print("Line"+str(i)+": "+line) + i=i+1 + print("Task 0 log end:-----------------------------") # Real run self.retention.trigger_retention_policy(retention_id, dry_run=False, **TestProjects.USER_RA_CLIENT) @@ -94,6 +109,13 @@ class TestProjects(unittest.TestCase): # resp=self.repo.list_repositories(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) # self.assertEqual(len(resp), 3) + #List artifacts successfully; + artifacts = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_1, **TestProjects.USER_RA_CLIENT) + print artifacts + # 'test1' has 3 artifacts, artifact1 with tag '1.0' and artifact2 with tag '2.0' should be deleted because they doesn't match 'latest' + # artifact3 should be retained because it has no tag, so count of artifacts should be 1. + # TODO: This verfication should be enhanced by verify sha256 at the same time; + self.assertTrue(len(artifacts)==1) @classmethod def tearDownClass(self): diff --git a/tests/resources/Harbor-Pages/Administration-Users.robot b/tests/resources/Harbor-Pages/Administration-Users.robot index c89858296..86ba32f51 100644 --- a/tests/resources/Harbor-Pages/Administration-Users.robot +++ b/tests/resources/Harbor-Pages/Administration-Users.robot @@ -56,5 +56,5 @@ Add A New User Retry Text Input xpath=${newPassword_xpath} ${newPassword} Retry Text Input xpath=${confirmPassword_xpath} ${newPassword} Retry Text Input xpath=${comment_xpath} ${comment} - Retry Element Click xpath=${save_new_user_button} + Retry Double Keywords When Error Retry Element Click xpath=${save_new_user_button} Retry Wait Until Page Not Contains Element xpath=${save_new_user_button} Retry Wait Until Page Contains Element xpath=//harbor-user//clr-dg-row//clr-dg-cell[contains(., '${username}')] diff --git a/tests/resources/Harbor-Pages/Project-Repository.robot b/tests/resources/Harbor-Pages/Project-Repository.robot index 23d4ad569..8a4a525ad 100644 --- a/tests/resources/Harbor-Pages/Project-Repository.robot +++ b/tests/resources/Harbor-Pages/Project-Repository.robot @@ -19,7 +19,7 @@ Resource ../../resources/Util.robot *** Keywords *** View Repo Scan Details Retry Element Click xpath=${first_repo_xpath} - Capture Page Screenshot viewcve1.png + Capture Page Screenshot Retry Wait Until Page Contains unknown Retry Wait Until Page Contains high Retry Wait Until Page Contains medium diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index 59d28b876..e0f2548e7 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -95,3 +95,6 @@ Test Case - Scan All Images Test Case - Registry API [Tags] reg_api Harbor API Test ./tests/apitests/python/test_registry_api.py +Test Case - Push Image With Special Name + [Tags] special_repo_name + Harbor API Test ./tests/apitests/python/test_push_image_with_special_name.py diff --git a/tests/robot-cases/Group1-Nightly/Clair.robot b/tests/robot-cases/Group1-Nightly/Clair.robot index c33e65b42..edd44fc70 100644 --- a/tests/robot-cases/Group1-Nightly/Clair.robot +++ b/tests/robot-cases/Group1-Nightly/Clair.robot @@ -105,6 +105,7 @@ Test Case - Scan Image On Push Go Into Project library Go Into Repo memcached Summary Chart Should Display latest + View Repo Scan Details Close Browser Test Case - View Scan Results diff --git a/tests/robot-cases/Group1-Nightly/Trivy.robot b/tests/robot-cases/Group1-Nightly/Trivy.robot index 90642f6ac..0bd5327c7 100644 --- a/tests/robot-cases/Group1-Nightly/Trivy.robot +++ b/tests/robot-cases/Group1-Nightly/Trivy.robot @@ -107,6 +107,7 @@ Test Case - Scan Image On Push Go Into Project library Go Into Repo memcached Summary Chart Should Display latest + View Repo Scan Details Close Browser Test Case - View Scan Results @@ -122,7 +123,7 @@ Test Case - View Scan Results Scan Repo latest Succeed Summary Chart Should Display latest View Repo Scan Details - Close Browser + Close Browser Test Case - Project Level Image Serverity Policy [Tags] run-once Init Chrome Driver