From 79051140b2fe479c2725a2e6464650f073e46e82 Mon Sep 17 00:00:00 2001 From: Yang Jiao <72076317+YangJiao0817@users.noreply.github.com> Date: Mon, 23 Oct 2023 14:40:55 +0800 Subject: [PATCH] [cherry-pick]Add GC accessory UI test case (#19471) Add GC accessory UI test case Fix #19274 Signed-off-by: Yang Jiao --- .../python/test_garbage_collection.py | 25 +++--- tests/resources/Cosign_Util.robot | 4 + tests/resources/Harbor-Pages/GC.robot | 16 ++++ .../Project-Artifact-Elements.robot | 14 ++++ .../Harbor-Pages/Project-Artifact.robot | 8 ++ tests/resources/TestCaseBody.robot | 35 ++++++++ .../Group1-Nightly/Common_GC.robot | 82 +++++++++++++++++-- 7 files changed, 164 insertions(+), 20 deletions(-) diff --git a/tests/apitests/python/test_garbage_collection.py b/tests/apitests/python/test_garbage_collection.py index a76d24248..8747f1d13 100644 --- a/tests/apitests/python/test_garbage_collection.py +++ b/tests/apitests/python/test_garbage_collection.py @@ -140,6 +140,7 @@ class TestProjects(unittest.TestCase): """ url = ADMIN_CLIENT["endpoint"] user_password = "Aa123456" + deleted_prefix = "delete blob from storage: " # 1. Create user(UA) _, user_name = self.user.create_user(user_password = user_password, **ADMIN_CLIENT) user_client = dict(endpoint = url, username = user_name, password = user_password, with_accessory = True) @@ -188,7 +189,7 @@ class TestProjects(unittest.TestCase): # 10. Get the GC log and check that the image(IA) Signature of signature is deleted gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT) - self.assertIn(signature_signature_digest, gc_log) + self.assertIn(deleted_prefix + signature_signature_digest, gc_log) # 11. Delete image(IA) Signature by user(UA) self.artifact.delete_artifact(project_name, self.image, image_signature_digest, **user_client) @@ -199,7 +200,7 @@ class TestProjects(unittest.TestCase): # 13. Get the GC log and check that the image(IA) Signature is deleted gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT) - self.assertIn(image_signature_digest, gc_log) + self.assertIn(deleted_prefix + image_signature_digest, gc_log) # 14. Delete image(IA) SBOM by user(UA) self.artifact.delete_artifact(project_name, self.image, sbom_digest, **user_client) @@ -210,8 +211,8 @@ class TestProjects(unittest.TestCase): # 16. Get the GC log and check that the image(IA) SBOM and Signature of SBOM is deleted gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT) - self.assertIn(sbom_digest, gc_log) - self.assertIn(sbom_signature_digest, gc_log) + self.assertIn(deleted_prefix + sbom_digest, gc_log) + self.assertIn(deleted_prefix + sbom_signature_digest, gc_log) # 17. Push image(IA) SBOM to project(PA) by user(UA) self.sbom_path = files_directory + "sbom_test.json" @@ -246,10 +247,10 @@ class TestProjects(unittest.TestCase): # 22. Get the GC log and check that it is not deleted gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT) - self.assertNotIn(sbom_digest, gc_log) - self.assertNotIn(sbom_signature_digest, gc_log) - self.assertNotIn(image_signature_digest, gc_log) - self.assertNotIn(signature_signature_digest, gc_log) + self.assertNotIn(deleted_prefix + sbom_digest, gc_log) + self.assertNotIn(deleted_prefix + sbom_signature_digest, gc_log) + self.assertNotIn(deleted_prefix + image_signature_digest, gc_log) + self.assertNotIn(deleted_prefix + signature_signature_digest, gc_log) # 23. Delete tag of image(IA) by user(UA) self.artifact.delete_tag(project_name, self.image, self.tag, self.tag, **user_client) @@ -261,10 +262,10 @@ class TestProjects(unittest.TestCase): # 25. Get the GC log and check that the image(IA) and all aeecssory is deleted gc_log = self.gc.get_gc_log_by_id(gc_id, **ADMIN_CLIENT) self.assertIn(self.image, gc_log) - self.assertIn(sbom_digest, gc_log) - self.assertIn(sbom_signature_digest, gc_log) - self.assertIn(image_signature_digest, gc_log) - self.assertIn(signature_signature_digest, gc_log) + self.assertIn(deleted_prefix + sbom_digest, gc_log) + self.assertIn(deleted_prefix + sbom_signature_digest, gc_log) + self.assertIn(deleted_prefix + image_signature_digest, gc_log) + self.assertIn(deleted_prefix + signature_signature_digest, gc_log) if __name__ == '__main__': diff --git a/tests/resources/Cosign_Util.robot b/tests/resources/Cosign_Util.robot index ed87e19a6..223e8d22a 100644 --- a/tests/resources/Cosign_Util.robot +++ b/tests/resources/Cosign_Util.robot @@ -30,3 +30,7 @@ Cosign Verify [Arguments] ${artifact} ${signed} Run Keyword If ${signed}==${true} Wait Unitl Command Success cosign verify --key cosign.pub ${artifact} ... ELSE Command Should be Failed cosign verify --key cosign.pub ${artifact} + +Cosign Push Sbom + [Arguments] ${artifact} ${sbom_path}=${files_directory}/sbom_test.json ${type}=spdx + Wait Unitl Command Success cosign attach sbom --allow-insecure-registry --registry-referrers-mode oci-1-1 --type ${type} --sbom ${sbom_path} ${artifact} diff --git a/tests/resources/Harbor-Pages/GC.robot b/tests/resources/Harbor-Pages/GC.robot index c74879363..25a3ff21f 100644 --- a/tests/resources/Harbor-Pages/GC.robot +++ b/tests/resources/Harbor-Pages/GC.robot @@ -58,3 +58,19 @@ Set GC Schedule Run Keyword If '${type}'=='custom' Run Keywords Retry Element Click ${vulnerability_dropdown_list_item_custom} AND Retry Text Input ${targetCron_id} ${value} ... ELSE Retry Element Click ${vulnerability_dropdown_list_item_none} Retry Double Keywords When Error Retry Element Click ${GC_schedule_save_btn} Retry Wait Until Page Not Contains Element ${gc_schedule_save_btn} + +Go To GC Log + [Arguments] ${gc_job_id} + Retry Link Click //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//a + Switch Window locator=NEW + +Check GC Log + [Arguments] ${gc_job_id} ${log_containing} ${log_excluding} + Go To GC Log ${gc_job_id} + FOR ${log} IN @{log_containing} + Wait Until Page Contains ${log} + END + FOR ${log} IN @{log_excluding} + Wait Until Page Does Not Contain ${log} + END + Switch Window locator=MAIN diff --git a/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot b/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot index 2cbaeadac..fd260d3c9 100644 --- a/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Artifact-Elements.robot @@ -36,3 +36,17 @@ ${harbor_helm_name} harbor-helm-1.7.3 ${harbor_helm_filename} harbor-helm-1.7.3.tar.gz ${harbor_helm_version} 1.7.3 ${harbor_helm_package} harbor-1.7.3.tgz + +${artifact_list_accessory_btn} (//clr-dg-row//button)[1] +${artifact_cosign_accessory} //clr-dg-row//clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div[text()=' signature.cosign ']] +${artifact_sbom_accessory} //clr-dg-row//clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div[text()=' subject.accessory ']] +${artifact_cosign_accessory_action_btn} (//clr-dg-row//clr-dg-row[.//div[text()=' signature.cosign ']]//button)[1] +${artifact_sbom_accessory_action_btn} (//clr-dg-row//clr-dg-row[.//div[text()=' subject.accessory ']]//button)[1] +${artifact_cosign_cosign_accessory_action_btn} ${artifact_cosign_accessory}//clr-dg-row//button +${artifact_sbom_cosign_accessory_action_btn} ${artifact_sbom_accessory}//clr-dg-row//button +${artifact_list_cosign_accessory_btn} (${artifact_cosign_accessory}//button)[2] +${artifact_list_sbom_accessory_btn} (${artifact_sbom_accessory}//button)[2] +${copy_digest_btn} //button[text()=' Copy Digest '] +${delete_accessory_btn} //button[text()=' Delete '] +${copy_btn} //button[text()=' COPY '] +${artifact_digest} //textarea diff --git a/tests/resources/Harbor-Pages/Project-Artifact.robot b/tests/resources/Harbor-Pages/Project-Artifact.robot index c91f32fa2..731abfd99 100644 --- a/tests/resources/Harbor-Pages/Project-Artifact.robot +++ b/tests/resources/Harbor-Pages/Project-Artifact.robot @@ -55,3 +55,11 @@ Get The Specific Artifact ${cmd}= Set Variable curl -u ${HARBOR_ADMIN}:${HARBOR_PASSWORD} -s --insecure -H "Content-Type: application/json" -X GET "${HARBOR_URL}/api/v2.0/projects/${project_name}/repositories/${repo_name}/artifacts/${reference}?page=1&page_size=10&with_tag=true&with_label=true&with_scan_overview=true&with_accessory=true&with_signature=true&with_immutable_status=true" ${rc} ${output}= Run And Return Rc And Output ${cmd} [Return] ${output} + +Refresh Artifacts + Retry Element Click ${artifact_list_refresh_btn} + +Delete Accessory By Aeecssory XPath + [Arguments] ${aeecssory_action_xpath} + Retry Double Keywords When Error Retry Button Click ${aeecssory_action_xpath} Retry Button Click ${delete_accessory_btn} + Retry Button Click //button[text()=' DELETE '] diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index 55c018671..fc785a185 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -667,3 +667,38 @@ Reset Schedules For Job Service Dashboard Schedules # Reset the schedule of log rotation Switch To Log Rotation Set Log Rotation Schedule 2 Days None + +Prepare Accessory + [Arguments] ${project_name} ${image} ${tag} ${user} ${pwd} + Docker Login ${ip} ${user} ${pwd} + Cosign Generate Key Pair + Cosign Sign ${ip}/${project_name}/${image}:${tag} + Cosign Push Sbom ${ip}/${project_name}/${image}:${tag} + Go Into Repo ${project_name} ${image} + Retry Button Click ${artifact_list_accessory_btn} + # Get SBOM digest + Retry Double Keywords When Error Retry Button Click ${artifact_sbom_accessory_action_btn} Retry Button Click ${copy_digest_btn} + Wait Until Element Is Visible And Enabled ${artifact_digest} + ${sbom_digest}= Get Text ${artifact_digest} + Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn} + # Get Signature digest + Retry Double Keywords When Error Retry Button Click ${artifact_cosign_accessory_action_btn} Retry Button Click ${copy_digest_btn} + Wait Until Element Is Visible And Enabled ${artifact_digest} + ${signature_digest}= Get Text ${artifact_digest} + Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn} + Cosign Sign ${ip}/${project_name}/${image}@${sbom_digest} + Cosign Sign ${ip}/${project_name}/${image}@${signature_digest} + Refresh Artifacts + Retry Button Click ${artifact_list_accessory_btn} + # Get Signature of SBOM digest + Retry Double Keywords When Error Retry Button Click ${artifact_list_sbom_accessory_btn} Retry Button Click ${artifact_sbom_cosign_accessory_action_btn} + Retry Double Keywords When Error Retry Button Click ${copy_digest_btn} Wait Until Element Is Visible And Enabled ${artifact_digest} + ${signature_of_sbom_digest}= Get Text ${artifact_digest} + Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn} + # Get Signature of Signature digest + Retry Double Keywords When Error Retry Button Click ${artifact_list_cosign_accessory_btn} Retry Button Click ${artifact_cosign_cosign_accessory_action_btn} + Retry Double Keywords When Error Retry Button Click ${copy_digest_btn} Wait Until Element Is Visible And Enabled ${artifact_digest} + ${signature_of_signature_digest}= Get Text ${artifact_digest} + Retry Double Keywords When Error Retry Button Click ${copy_btn} Retry Wait Element Not Visible ${copy_btn} + Docker Logout ${ip} + [Return] ${sbom_digest} ${signature_digest} ${signature_of_sbom_digest} ${signature_of_signature_digest} diff --git a/tests/robot-cases/Group1-Nightly/Common_GC.robot b/tests/robot-cases/Group1-Nightly/Common_GC.robot index 0fcd98bb0..c94d59c3f 100644 --- a/tests/robot-cases/Group1-Nightly/Common_GC.robot +++ b/tests/robot-cases/Group1-Nightly/Common_GC.robot @@ -25,7 +25,7 @@ ${HARBOR_ADMIN} admin *** Test Cases *** Test Case - Garbage Collection Init Chrome Driver - ${d}= Get Current Date result_format=%m%s + ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} GC Now Create An New Project And Go Into Project project${d} @@ -39,22 +39,22 @@ Test Case - Garbage Collection Test Case - GC Untagged Images Init Chrome Driver - ${d}= Get Current Date result_format=%m%s + ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} GC Now Create An New Project And Go Into Project project${d} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world latest # make hello-world untagged Go Into Repo project${d} hello-world - Go Into Artifact latest - Should Contain Tag latest - Delete A Tag latest - Should Not Contain Tag latest + Go Into Artifact latest + Should Contain Tag latest + Delete A Tag latest + Should Not Contain Tag latest # run gc without param delete untagged artifacts checked, should not delete hello-world:latest GC Now ${latest_job_id}= Get Text ${latest_job_id_xpath} Retry GC Should Be Successful ${latest_job_id} ${null} - Go Into Repo project${d} hello-world + Go Into Repo project${d} hello-world Should Contain Artifact # run gc with param delete untagged artifacts checked, should delete hello-world Switch To Garbage Collection @@ -72,7 +72,7 @@ Test Case - Project Quotas Control Under GC ${storage_quota}= Set Variable 200 ${storage_quota_unit}= Set Variable MiB ${image_a}= Set Variable logstash - ${image_a_size}= Set Variable 321.03MiB + ${image_a_size}= Set Variable 321.03MiB ${image_a_ver}= Set Variable 6.8.3 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} GC Now @@ -89,3 +89,69 @@ Test Case - Project Quotas Control Under GC END Should Be Equal As Strings '${out2[0]}' 'PASS' Close Browser + +Test Case - Garbage Collection Accessory + Init Chrome Driver + ${d}= Get Current Date result_format=%m%s + ${image}= Set Variable hello-world + ${tag}= Set Variable latest + ${deleted_prefix}= Set Variable delete blob from storage: + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + GC Now + Create An New Project And Go Into Project project${d} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} + + ${sbom_digest} ${signature_digest} ${signature_of_sbom_digest} ${signature_of_signature_digest}= Prepare Accessory project${d} ${image} ${tag} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + # Delete the Signature of Signature + Delete Accessory By Aeecssory XPath ${artifact_cosign_cosign_accessory_action_btn} + ${gc_job_id}= GC Now + Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS'] + ${log_containing}= Create List ${deleted_prefix} ${signature_of_signature_digest} + ${log_excluding}= Create List ${deleted_prefix} ${sbom_digest} + ... ${deleted_prefix} ${signature_of_sbom_digest} + ... ${deleted_prefix} ${signature_digest} + Check GC Log ${gc_job_id} ${log_containing} ${log_excluding} + Go Into Repo project${d} ${image} + Retry Button Click ${artifact_list_accessory_btn} + # Delete the Signature + Delete Accessory By Aeecssory XPath ${artifact_cosign_accessory_action_btn} + ${gc_job_id}= GC Now + Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS'] + ${log_containing}= Create List ${deleted_prefix} ${signature_digest} + ${log_excluding}= Create List ${deleted_prefix} ${sbom_digest} + ... ${deleted_prefix} ${signature_of_sbom_digest} + Check GC Log ${gc_job_id} ${log_containing} ${log_excluding} + Go Into Repo project${d} ${image} + Retry Button Click ${artifact_list_accessory_btn} + # Delete the SBOM + Delete Accessory By Aeecssory XPath ${artifact_sbom_accessory_action_btn} + ${gc_job_id}= GC Now + Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS'] + ${log_containing}= Create List ${deleted_prefix} ${sbom_digest} + ... ${deleted_prefix} ${signature_of_sbom_digest} + ${log_excluding}= Create List + Check GC Log ${gc_job_id} ${log_containing} ${log_excluding} + + ${sbom_digest} ${signature_digest} ${signature_of_sbom_digest} ${signature_of_signature_digest}= Prepare Accessory project${d} ${image} ${tag} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + # Delete image tags + Go Into Repo project${d} ${image} + Go Into Artifact ${tag} + Should Contain Tag ${tag} + Delete A Tag ${tag} + ${gc_job_id}= GC Now + Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS'] + ${log_containing}= Create List + ${log_excluding}= Create List ${deleted_prefix} ${sbom_digest} + ... ${deleted_prefix} ${signature_digest} + ... ${deleted_prefix} ${signature_of_sbom_digest} + ... ${deleted_prefix} ${signature_of_signature_digest} + Check GC Log ${gc_job_id} ${log_containing} ${log_excluding} + ${gc_job_id}= GC Now untag=${true} + Wait Until Element Is Visible And Enabled //clr-dg-row[.//clr-dg-cell[text()='${gc_job_id}']]//clr-dg-cell[text()='SUCCESS'] + ${log_containing}= Create List ${deleted_prefix} ${sbom_digest} + ... ${deleted_prefix} ${signature_digest} + ... ${deleted_prefix} ${signature_of_sbom_digest} + ... ${deleted_prefix} ${signature_of_signature_digest} + ${log_excluding}= Create List + Check GC Log ${gc_job_id} ${log_containing} ${log_excluding} + Close Browser