From cdbef9d4d3a6b2ace9ebd04f8f651413cdcacd0c Mon Sep 17 00:00:00 2001 From: Daniel Jiang Date: Fri, 30 Aug 2024 10:17:34 +0800 Subject: [PATCH] [Cherrypick to v2.11]add Test Case for SBOM feature (#20797) (#20886) add Test Case for SBOM feature (#20797) Signed-off-by: Shengwen Yu Co-authored-by: Shengwen YU --- tests/apitests/python/library/artifact.py | 25 +++++ tests/apitests/python/library/scan.py | 15 +++ tests/apitests/python/library/scan_stop.py | 15 +++ .../python/test_project_permission.py | 15 ++- .../test_sbom_generation_of_image_artifact.py | 87 ++++++++++++++++++ ..._stop_sbom_generation_of_image_artifact.py | 91 +++++++++++++++++++ .../Harbor-Pages/Project-Repository.robot | 17 +++- .../Project-Repository_Elements.robot | 5 +- .../Harbor-Pages/Vulnerability.robot | 35 +++++++ tests/resources/TestCaseBody.robot | 43 +++++++++ tests/robot-cases/Group0-BAT/API_DB.robot | 8 ++ tests/robot-cases/Group1-Nightly/Trivy.robot | 12 +++ 12 files changed, 359 insertions(+), 9 deletions(-) create mode 100644 tests/apitests/python/test_sbom_generation_of_image_artifact.py create mode 100644 tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py diff --git a/tests/apitests/python/library/artifact.py b/tests/apitests/python/library/artifact.py index d8b6905b9..eca6a37c8 100644 --- a/tests/apitests/python/library/artifact.py +++ b/tests/apitests/python/library/artifact.py @@ -30,6 +30,8 @@ class Artifact(base.Base, object): if "with_scan_overview" in kwargs: params["with_scan_overview"] = kwargs["with_scan_overview"] params["x_accept_vulnerabilities"] = ",".join(report_mime_types) + if "with_sbom_overview" in kwargs: + params["with_sbom_overview"] = kwargs["with_sbom_overview"] if "with_immutable_status" in kwargs: params["with_immutable_status"] = kwargs["with_immutable_status"] if "with_accessory" in kwargs: @@ -140,6 +142,29 @@ class Artifact(base.Base, object): return raise Exception("Scan image result is {}, not as expected {}.".format(scan_status, expected_scan_status)) + def check_image_sbom_generation_result(self, project_name, repo_name, reference, expected_scan_status = "Success", **kwargs): + timeout_count = 30 + scan_status="" + while True: + time.sleep(5) + timeout_count = timeout_count - 1 + if (timeout_count == 0): + break + artifact = self.get_reference_info(project_name, repo_name, reference, **kwargs) + if expected_scan_status in ["Not Scanned", "No SBOM Overview"]: + if artifact.sbom_overview is None: + if (timeout_count > 24): + continue + print("artifact SBOM is not generated.") + return + else: + raise Exception("Artifact SBOM should not be generated {}.".format(artifact.sbom_overview)) + + scan_status = artifact.sbom_overview.scan_status + if scan_status == expected_scan_status: + return + raise Exception("Generate image SBOM result is {}, not as expected {}.".format(scan_status, expected_scan_status)) + def check_reference_exist(self, project_name, repo_name, reference, ignore_not_found = False, **kwargs): artifact = self.get_reference_info( project_name, repo_name, reference, ignore_not_found=ignore_not_found, **kwargs) return { diff --git a/tests/apitests/python/library/scan.py b/tests/apitests/python/library/scan.py index a9260e2d6..6af92200f 100644 --- a/tests/apitests/python/library/scan.py +++ b/tests/apitests/python/library/scan.py @@ -21,3 +21,18 @@ class Scan(base.Base, object): base._assert_status_code(expect_status_code, status_code) return data + + def sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs): + try: + req_param = dict(scan_type = {"scan_type":"sbom"}) + data, status_code, _ = self._get_client(**kwargs).scan_artifact_with_http_info(project_name, repo_name, reference, **req_param) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + return + + base._assert_status_code(expect_status_code, status_code) + + return data + diff --git a/tests/apitests/python/library/scan_stop.py b/tests/apitests/python/library/scan_stop.py index e002239b2..80507ebbb 100644 --- a/tests/apitests/python/library/scan_stop.py +++ b/tests/apitests/python/library/scan_stop.py @@ -22,4 +22,19 @@ class StopScan(base.Base, object): base._assert_status_code(expect_status_code, status_code) + return data + + def stop_sbom_generation_of_artifact(self, project_name, repo_name, reference, expect_status_code = 202, expect_response_body = None, **kwargs): + try: + scanType = v2_swagger_client.ScanType() + scanType.scan_type = "sbom" + data, status_code, _ = self._get_client(**kwargs).stop_scan_artifact_with_http_info(project_name, repo_name, reference, scanType) + except ApiException as e: + base._assert_status_code(expect_status_code, e.status) + if expect_response_body is not None: + base._assert_status_body(expect_response_body, e.body) + return + + base._assert_status_code(expect_status_code, status_code) + return data \ No newline at end of file diff --git a/tests/apitests/python/test_project_permission.py b/tests/apitests/python/test_project_permission.py index 491eba2dc..a7533c722 100644 --- a/tests/apitests/python/test_project_permission.py +++ b/tests/apitests/python/test_project_permission.py @@ -73,7 +73,7 @@ list_metadata = Permission("{}/projects/{}/metadatas".format(harbor_base_url, pr read_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "GET", 200, metadata_payload) metadata_payload_for_update = { "auto_scan": "false" } update_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "PUT", 200, metadata_payload_for_update) -delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload) +delete_metadata = Permission("{}/projects/{}/metadatas/auto_scan".format(harbor_base_url, project_id), "DELETE", 200, metadata_payload_for_update) # 4. Resource: repository actions: ['read', 'list', 'update', 'delete', 'pull', 'push'] # note: pull and push are for docker cli, no API needs them @@ -89,12 +89,17 @@ copy_artifact = Permission("{}/projects/{}/repositories/target_repo/artifacts?fr delete_artifact = Permission("{}/projects/{}/repositories/target_repo/artifacts/{}".format(harbor_base_url, project_name, source_artifact_tag), "DELETE", 200) # 6. Resource scan actions: ['read', 'create', 'stop'] -stop_scan_payload = { +vulnerability_scan_payload = { "scan_type": "vulnerability" } -create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202) -stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, stop_scan_payload) +create_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload) +stop_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, vulnerability_scan_payload) read_scan = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/83be44fd-1234-5678-b49f-4b6d6e8f5730/log".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "get", 404) +sbom_gen_payload = { + "scan_type": "sbom" +} +create_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload) +stop_sbom_generation = Permission("{}/projects/{}/repositories/{}/artifacts/{}/scan/stop".format(harbor_base_url, project_name, source_artifact_name, source_artifact_tag), "POST", 202, sbom_gen_payload) # 7. Resource tag actions: ['list', 'create', 'delete'] tag_payload = { "name": "test-{}".format(int(random.randint(1000, 9999))) } @@ -240,7 +245,7 @@ resource_permissions = { "metadata": [create_metadata, list_metadata, read_metadata, update_metadata, delete_metadata], "repository": [list_repo, read_repo, update_repo, delete_repo], "artifact": [list_artifact, read_artifact, copy_artifact, delete_artifact], - "scan": [create_scan, stop_scan, read_scan], + "scan": [create_scan, stop_scan, read_scan, create_sbom_generation, stop_sbom_generation], "tag": [create_tag, list_tag, delete_tag], "accessory": [list_accessory], "artifact-addition": [read_artifact_addition_vul, read_artifact_addition_dependencies], diff --git a/tests/apitests/python/test_sbom_generation_of_image_artifact.py b/tests/apitests/python/test_sbom_generation_of_image_artifact.py new file mode 100644 index 000000000..8025fafe3 --- /dev/null +++ b/tests/apitests/python/test_sbom_generation_of_image_artifact.py @@ -0,0 +1,87 @@ +from __future__ import absolute_import +import unittest +import sys + +from testutils import harbor_server, suppress_urllib3_warning +from testutils import TEARDOWN +from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME +from library.project import Project +from library.user import User +from library.repository import Repository +from library.repository import push_self_build_image_to_project +from library.artifact import Artifact +from library.scan import Scan + +class TestSBOMGeneration(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.project= Project() + self.user= User() + self.artifact = Artifact() + self.repo = Repository() + self.scan = Scan() + + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5 + self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True) + + + #2. Create a new private project(PA) by user(UA); + self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) + + #3. Add user(UA) as a member of project(PA) with project-admin role; + self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT) + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def do_tearDown(self): + #1. Delete repository(RA) by user(UA); + self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT) + + #2. Delete project(PA); + self.project.delete_project(self.project_id, **self.USER_CLIENT) + + #3. Delete user(UA); + self.user.delete_user(self.user_id, **ADMIN_CLIENT) + + def testGenerateSBOMOfImageArtifact(self): + """ + Test case: + Generate an SBOM of An Image Artifact + Test step and expected result: + 1. Create a new user(UA); + 2. Create a new private project(PA) by user(UA); + 3. Add user(UA) as a member of project(PA) with project-admin role; + 4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + 5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + 6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished; + Tear down: + 1. Delete repository(RA) by user(UA); + 2. Delete project(PA); + 3. Delete user(UA); + """ + + #4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + self.project.projects_should_exist(dict(public=False), expected_count = 1, + expected_project_id = self.project_id, **self.USER_CLIENT) + + #Note: Please make sure that this Image has never been pulled before by any other cases, + # so it is a not-scanned image right after repository creation. + image = "docker" + src_tag = "1.13" + #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag) + + #6. Send sbom generation of an image command and get tag(TA) information to check sbom generation result, it should be finished; + self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) + self.artifact.check_image_sbom_generation_result(self.project_name, image, tag, **self.USER_CLIENT) + + self.do_tearDown() + + +if __name__ == '__main__': + suite = unittest.TestSuite(unittest.makeSuite(TestSBOMGeneration)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + if not result.wasSuccessful(): + raise Exception(r"SBOM generation test failed: {}".format(result)) diff --git a/tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py b/tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py new file mode 100644 index 000000000..232e5e85d --- /dev/null +++ b/tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py @@ -0,0 +1,91 @@ +from __future__ import absolute_import +import unittest +import sys + +from testutils import harbor_server, suppress_urllib3_warning +from testutils import TEARDOWN +from testutils import ADMIN_CLIENT, BASE_IMAGE, BASE_IMAGE_ABS_PATH_NAME +from library.project import Project +from library.user import User +from library.repository import Repository +from library.repository import push_self_build_image_to_project +from library.artifact import Artifact +from library.scan import Scan +from library.scan_stop import StopScan + +class TestStopSBOMGeneration(unittest.TestCase): + @suppress_urllib3_warning + def setUp(self): + self.project= Project() + self.user= User() + self.artifact = Artifact() + self.repo = Repository() + self.scan = Scan() + self.stop_scan = StopScan() + + self.url = ADMIN_CLIENT["endpoint"] + self.user_password = "Aa123456" + self.project_id, self.project_name, self.user_id, self.user_name, self.repo_name1 = [None] * 5 + self.user_id, self.user_name = self.user.create_user(user_password = self.user_password, **ADMIN_CLIENT) + self.USER_CLIENT = dict(with_signature = True, with_immutable_status = True, endpoint = self.url, username = self.user_name, password = self.user_password, with_sbom_overview = True) + + + #2. Create a new private project(PA) by user(UA); + self.project_id, self.project_name = self.project.create_project(metadata = {"public": "false"}, **ADMIN_CLIENT) + + #3. Add user(UA) as a member of project(PA) with project-admin role; + self.project.add_project_members(self.project_id, user_id = self.user_id, **ADMIN_CLIENT) + + @unittest.skipIf(TEARDOWN == False, "Test data won't be erased.") + def do_tearDown(self): + #1. Delete repository(RA) by user(UA); + self.repo.delete_repository(self.project_name, self.repo_name1.split('/')[1], **self.USER_CLIENT) + + #2. Delete project(PA); + self.project.delete_project(self.project_id, **self.USER_CLIENT) + + #3. Delete user(UA); + self.user.delete_user(self.user_id, **ADMIN_CLIENT) + + def testStopSBOMGenerationOfImageArtifact(self): + """ + Test case: + Stop SBOM Generation Of An Image Artifact + Test step and expected result: + 1. Create a new user(UA); + 2. Create a new private project(PA) by user(UA); + 3. Add user(UA) as a member of project(PA) with project-admin role; + 4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + 5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + 6. Send SBOM generation of an image command; + 7. Send stop SBOM generation of an image command. + Tear down: + 1. Delete repository(RA) by user(UA); + 2. Delete project(PA); + 3. Delete user(UA); + """ + + #4. Get private project of user(UA), user(UA) can see only one private project which is project(PA); + self.project.projects_should_exist(dict(public=False), expected_count = 1, + expected_project_id = self.project_id, **self.USER_CLIENT) + + #Note: Please make sure that this Image has never been pulled before by any other cases, + # so it is a not-scanned image right after repository creation. + image = "docker" + src_tag = "1.13" + #5. Create a new repository(RA) and tag(TA) in project(PA) by user(UA); + self.repo_name1, tag = push_self_build_image_to_project(self.project_name, harbor_server, self.user_name, self.user_password, image, src_tag) + + #6. Send SBOM generation of an image command; + self.scan.sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) + + #7. Send stop SBOM generation of an image command. + self.stop_scan.stop_sbom_generation_of_artifact(self.project_name, self.repo_name1.split('/')[1], tag, **self.USER_CLIENT) + + self.do_tearDown() + +if __name__ == '__main__': + suite = unittest.TestSuite(unittest.makeSuite(TestStopSBOMGeneration)) + result = unittest.TextTestRunner(sys.stdout, verbosity=2, failfast=True).run(suite) + if not result.wasSuccessful(): + raise Exception(r"Stop SBOM generation test failed: {}".format(result)) diff --git a/tests/resources/Harbor-Pages/Project-Repository.robot b/tests/resources/Harbor-Pages/Project-Repository.robot index 04c0b355b..1eef8e068 100644 --- a/tests/resources/Harbor-Pages/Project-Repository.robot +++ b/tests/resources/Harbor-Pages/Project-Repository.robot @@ -41,9 +41,20 @@ Stop Scan Artifact Retry Element Click ${stop_scan_artifact_btn} Check Scan Artifact Job Status Is Stopped - Wait Until Element Is Visible ${stopped_label} - ${job_status}= Get Text ${stopped_label} - Should Be Equal As Strings '${job_status}' 'Scan stopped' + Wait Until Element Is Visible ${scan_stopped_label} + +Generate Artifact SBOM + [Arguments] ${project} ${repo} ${label_xpath}=//clr-dg-row//label[contains(@class,'clr-control-label')][1] + Go Into Repo ${project} ${repo} + Retry Element Click ${label_xpath} + Retry Element Click ${gen_artifact_sbom_btn} + +Stop Gen Artifact SBOM + Retry Element Click ${artifact_action_xpath} + Retry Element Click ${stop_gen_artifact_sbom_btn} + +Check Gen Artifact SBOM Job Status Is Stopped + Wait Until Element Is Visible ${gen_sbom_stopped_label} Refresh Repositories Retry Element Click ${refresh_repositories_xpath} diff --git a/tests/resources/Harbor-Pages/Project-Repository_Elements.robot b/tests/resources/Harbor-Pages/Project-Repository_Elements.robot index 6cb344058..66c6aa7ff 100644 --- a/tests/resources/Harbor-Pages/Project-Repository_Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Repository_Elements.robot @@ -24,5 +24,8 @@ ${build_history_data} //clr-dg-row ${push_image_command_btn} //hbr-push-image-button//button ${scan_artifact_btn} //button[@id='scan-btn'] ${stop_scan_artifact_btn} //button[@id='stop-scan'] -${stopped_label} //span[@class='label stopped'] +${scan_stopped_label} //span[normalize-space()='Scan stopped'] +${gen_sbom_stopped_label} //span[normalize-space()='Generation stopped'] +${gen_artifact_sbom_btn} //button[@id='generate-sbom-btn'] +${stop_gen_artifact_sbom_btn} //button[@id='stop-sbom-btn'] ${refresh_repositories_xpath} //hbr-repository-gridview//span[contains(@class,'refresh-btn')] \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Vulnerability.robot b/tests/resources/Harbor-Pages/Vulnerability.robot index b7fb6d430..03ed48eef 100644 --- a/tests/resources/Harbor-Pages/Vulnerability.robot +++ b/tests/resources/Harbor-Pages/Vulnerability.robot @@ -59,6 +59,41 @@ Enable Scan On Push Checkbox Should Be Selected //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input Retry Element Click ${project_config_save_btn} +Generate Repo SBOM + [Arguments] ${tagname} ${status} + Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//label[contains(@class,'clr-control-label')] + Retry Element Click //button[@id='generate-sbom-btn'] + Run Keyword If '${status}' == 'Succeed' Wait Until Element Is Visible //a[@title='SBOM details'] 300 + +Checkout And Review SBOM Details + [Arguments] ${tagname} + Retry Element Click //clr-dg-row[contains(.,'${tagname}')]//a[@title='SBOM details'] + # Download SBOM file + Retry Element Click //button[@id='sbom-btn'] + ${sbom_artifact_short_sha256}= Get Text //span[@class='margin-left-10px'] + ${sbom_filename_raw}= Get Text //clr-dg-cell[contains(text(),'${sbom_artifact_short_sha256}')] + ${sbom_filename}= Replace String ${sbom_filename_raw} : _ count=-1 + ${sbom_filename}= Replace String ${sbom_filename} / _ count=-1 + ${sbom_json_path}= Set Variable ${download_directory}/${sbom_filename}.json + Retry File Should Exist ${sbom_json_path} + # Load the downloaded SBOM json file and verify the first N package records + ${sbom_json_content}= Load Json From File ${sbom_json_path} + ${items}= Get Value From JSON ${sbom_json_content} packages + ${items_length}= Get Length ${items} + ${first_n_records}= Evaluate min(5, ${items_length}) + FOR ${idx} IN RANGE 1 ${first_n_records} + ${item}= Get From List ${items} ${idx} + Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.name}'] + Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.versionInfo}'] + Wait Until Element Is Visible //clr-dg-cell[normalize-space()='${item.licenseConcluded}'] + END + +Enable Generating SBOM On Push + Checkbox Should Not Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input + Retry Element Click //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//label[contains(@class,'clr-control-label')] + Checkbox Should Be Selected //clr-checkbox-wrapper[@id='generate-sbom-on-push-wrapper']//input + Retry Element Click ${project_config_save_btn} + Vulnerability Not Ready Project Hint Sleep 2 ${element}= Set Variable xpath=//span[contains(@class, 'db-status-warning')] diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index fc785a185..9c721d8de 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -417,6 +417,49 @@ Stop Scan All Stop Scan All Artifact Retry Action Keyword Check Scan All Artifact Job Status Is Stopped +Body Of Generate SBOM of An Image In The Repo + [Arguments] ${image_argument} ${tag_argument} + Init Chrome Driver + + ${d}= get current date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project And Go Into Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_argument}:${tag_argument} + Go Into Repo project${d} ${image_argument} + Generate Repo SBOM ${tag_argument} Succeed + Checkout And Review SBOM Details ${tag_argument} + Close Browser + +Body Of Generate Image SBOM On Push + Init Chrome Driver + ${d}= get current date result_format=%m%s + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project And Go Into Project project${d} + Goto Project Config + Enable Generating SBOM On Push + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} memcached + Go Into Repo project${d} memcached + Checkout And Review SBOM Details latest + Close Browser + +Body Of Stop SBOM Manual Generation + Init Chrome Driver + ${d}= get current date result_format=%m%s + ${repo}= Set Variable goharbor/harbor-e2e-engine + ${tag}= Set Variable test-ui + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project And Go Into Project project${d} + Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${repo} ${tag} ${tag} + # stop generate sbom of an artifact + Retry Action Keyword Stop SBOM Generation project${d} ${repo} + Close Browser + +Stop SBOM Generation + [Arguments] ${project_name} ${repo} + Generate Artifact SBOM ${project_name} ${repo} + Stop Gen Artifact SBOM + Retry Action Keyword Check Gen Artifact SBOM Job Status Is Stopped + Prepare Image Package Test Files [Arguments] ${files_path} ${rc} ${output}= Run And Return Rc And Output bash tests/robot-cases/Group0-Util/prepare_imgpkg_test_files.sh ${files_path} diff --git a/tests/robot-cases/Group0-BAT/API_DB.robot b/tests/robot-cases/Group0-BAT/API_DB.robot index f1a92b720..7e5bae3c2 100644 --- a/tests/robot-cases/Group0-BAT/API_DB.robot +++ b/tests/robot-cases/Group0-BAT/API_DB.robot @@ -120,6 +120,14 @@ Test Case - Stop Scan All Images [Tags] stop_scan_all Harbor API Test ./tests/apitests/python/test_system_level_stop_scan_all.py +Test Case - Generate SBOM Of An Image + [Tags] generate_sbom + Harbor API Test ./tests/apitests/python/test_sbom_generation_of_image_artifact.py + +Test Case - Stop Generating SBOM Of An Image + [Tags] stop_generating_sbom + Harbor API Test ./tests/apitests/python/test_stop_sbom_generation_of_image_artifact.py + Test Case - Registry API [Tags] reg_api Harbor API Test ./tests/apitests/python/test_registry_api.py diff --git a/tests/robot-cases/Group1-Nightly/Trivy.robot b/tests/robot-cases/Group1-Nightly/Trivy.robot index bcfff07d2..4d976ace3 100644 --- a/tests/robot-cases/Group1-Nightly/Trivy.robot +++ b/tests/robot-cases/Group1-Nightly/Trivy.robot @@ -164,6 +164,18 @@ Test Case - Stop Scan And Stop Scan All [Tags] stop_scan_job Body Of Stop Scan And Stop Scan All +Test Case - Verify SBOM Manual Generation + [Tags] sbom_manual_gen + Body Of Generate SBOM of An Image In The Repo alpine 3.10 + +Test Case - Generate Image SBOM On Push + [Tags] run-once + Body Of Generate Image SBOM On Push + +Test Case - Stop SBOM Manual Generation + [Tags] stop_sbom_gen + Body Of Stop SBOM Manual Generation + Test Case - External Scanner CRUD [Tags] external_scanner_crud need_scanner_endpoint ${SCANNER_ENDPOINT_VALUE}= Get Variable Value ${SCANNER_ENDPOINT} ${EMPTY}