From 969dd1be66ea53e8cf284e0769a895c0fb777601 Mon Sep 17 00:00:00 2001 From: Yang Jiao <72076317+YangJiao0817@users.noreply.github.com> Date: Wed, 22 Nov 2023 18:13:44 +0800 Subject: [PATCH] Add notation accessory copy test case (#19605) Fix #19546 Signed-off-by: Yang Jiao --- tests/resources/Harbor-Pages/Project.robot | 22 +++++- tests/resources/Notation_Util.robot | 26 +++++++ tests/resources/Util.robot | 1 + tests/robot-cases/Group1-Nightly/Common.robot | 17 +++-- .../Group1-Nightly/Replication.robot | 72 +++++++++---------- 5 files changed, 95 insertions(+), 43 deletions(-) create mode 100644 tests/resources/Notation_Util.robot diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index fe3a88586..62d5c8420 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -375,13 +375,29 @@ Back Project Home [Arguments] ${project_name} Retry Link Click //a[contains(.,'${project_name}')] -Should Not Be Signed By Cosign +Should Be Signed + [Arguments] ${tag} + Retry Wait Element Visible //clr-dg-row[contains(.,'${tag}')]//clr-icon[contains(@class,'signed')] + +Should Not Be Signed [Arguments] ${tag} Retry Wait Element Visible //clr-dg-row[contains(.,'${tag}')]//clr-icon[contains(@class,'color-red')] Should Be Signed By Cosign - [Arguments] ${tag} - Retry Wait Element Visible //clr-dg-row[contains(.,'${tag}')]//clr-icon[contains(@class,'signed')] + [Arguments] ${tag}=${null} ${digest}=${null} + IF '${tag}' != '${null}' + Retry Wait Element Visible //clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div/clr-tooltip/div/div/span[contains(.,'${tag}')] and .//clr-dg-row[.//img[@title='signature.cosign']]] + ELSE + Retry Wait Element Visible //clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div/a[contains(.,'${digest}')] and .//clr-dg-row[.//img[@title='signature.cosign']]] + END + +Should Be Signed By Notation + [Arguments] ${tag}=${null} ${digest}=${null} + IF '${tag}' != '${null}' + Retry Wait Element Visible //clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div/clr-tooltip/div/div/span[contains(.,'${tag}')] and .//clr-dg-row[.//img[@title='signature.notation']]] + ELSE + Retry Wait Element Visible //clr-dg-row[./clr-expandable-animation/div/div/div/clr-dg-cell/div/a[contains(.,'${digest}')] and .//clr-dg-row[.//img[@title='signature.notation']]] + END Delete Accessory [Arguments] ${tag} diff --git a/tests/resources/Notation_Util.robot b/tests/resources/Notation_Util.robot new file mode 100644 index 000000000..ac5f6e6c1 --- /dev/null +++ b/tests/resources/Notation_Util.robot @@ -0,0 +1,26 @@ +# Copyright Project Harbor Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +*** Settings *** +Documentation This resource provides helper functions for docker operations +Library OperatingSystem +Library Process + +*** Keywords *** +Notation Generate Cert + Run And Return Rc And Output notation cert generate-test --default wabbit-networks.io + +Notation Sign + [Arguments] ${artifact} + Wait Unitl Command Success notation sign -d --allow-referrers-api ${artifact} diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index 17c86a9d2..b368e9dbb 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -82,6 +82,7 @@ Resource SeleniumUtil.robot Resource Nightly-Util.robot Resource APITest-Util.robot Resource Cosign_Util.robot +Resource Notation_Util.robot Resource Imgpkg-Util.robot Resource Webhook-Util.robot Resource TestCaseBody.robot diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 6b5246db4..63e1c9e51 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -474,19 +474,28 @@ Test Case - Copy A Image And Accessory Create An New Project And Go Into Project ${source_project} Push Image With Tag ${ip} ${user} ${pwd} ${source_project} ${image} ${tag} - Cosign Generate Key Pair Docker Login ${ip} ${user} ${pwd} + Cosign Generate Key Pair Cosign Sign ${ip}/${source_project}/${image}:${tag} - Docker Logout ${ip} + Notation Generate Cert + Notation Sign ${ip}/${source_project}/${image}:${tag} + Go Into Repo ${source_project} ${image} + Should Be Signed ${tag} + Retry Button Click ${artifact_list_accessory_btn} Should Be Signed By Cosign ${tag} + Should Be Signed By Notation ${tag} Copy Image ${tag} ${target_project} ${image} Retry Double Keywords When Error Go Into Project ${target_project} Retry Wait Until Page Contains ${image} Go Into Repo ${target_project} ${image} Retry Wait Until Page Contains Element //clr-dg-row[contains(.,${tag})] + Should Be Signed ${tag} + Retry Button Click ${artifact_list_accessory_btn} Should Be Signed By Cosign ${tag} + Should Be Signed By Notation ${tag} + Docker Logout ${ip} Close Browser Test Case - Create An New Project With Quotas Set @@ -772,14 +781,14 @@ Test Case - Cosign And Cosign Deployment Security Policy Push Image With Tag ${ip} ${user} ${pwd} project${d} ${image} ${tag} Go Into Project project${d} Go Into Repo project${d} ${image} - Should Not Be Signed By Cosign ${tag} + Should Not Be Signed ${tag} Cannot Pull Image ${ip} ${user} ${pwd} project${d} ${image}:${tag} err_msg=The image is not signed by cosign. Cosign Generate Key Pair Cosign Verify ${ip}/project${d}/${image}:${tag} ${false} Cosign Sign ${ip}/project${d}/${image}:${tag} Cosign Verify ${ip}/project${d}/${image}:${tag} ${true} - Retry Double Keywords When Error Retry Element Click ${artifact_list_refresh_btn} Should Be Signed By Cosign ${tag} + Retry Double Keywords When Error Retry Element Click ${artifact_list_refresh_btn} Should Be Signed ${tag} Pull image ${ip} ${user} ${pwd} project${d} ${image}:${tag} Retry Double Keywords When Error Delete Accessory ${tag} Should be Accessory deleted ${tag} diff --git a/tests/robot-cases/Group1-Nightly/Replication.robot b/tests/robot-cases/Group1-Nightly/Replication.robot index 1b1ec7e10..0380432b7 100644 --- a/tests/robot-cases/Group1-Nightly/Replication.robot +++ b/tests/robot-cases/Group1-Nightly/Replication.robot @@ -389,16 +389,16 @@ Test Case - Robot Account Do Replication Logout Harbor Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Image Should Be Replicated To Project project_dest${d} ${image1} - Should Be Signed By Cosign ${tag1} + Should Be Signed ${tag1} Image Should Be Replicated To Project project_dest${d} ${image2} - Should Be Signed By Cosign ${tag2} + Should Be Signed ${tag2} Back Project Home project_dest${d} Go Into Repo project_dest${d} ${index} - Should Be Signed By Cosign ${index_tag} + Should Be Signed ${index_tag} Go Into Repo project_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image1_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image1_short_sha256} Go Into Repo project_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed By Cosign ${image2_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed ${image2_short_sha256} # pull mode Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -409,16 +409,16 @@ Test Case - Robot Account Do Replication Check Latest Replication Job Status Succeeded Check Latest Replication Enabled Copy By Chunk Image Should Be Replicated To Project project_dest${d} ${image1} - Should Be Signed By Cosign ${tag1} + Should Be Signed ${tag1} Image Should Be Replicated To Project project_dest${d} ${image2} - Should Be Signed By Cosign ${tag2} + Should Be Signed ${tag2} Back Project Home project_dest${d} Go Into Repo project_dest${d} ${index} - Should Be Signed By Cosign ${index_tag} + Should Be Signed ${index_tag} Go Into Repo project_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image1_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image1_short_sha256} Go Into Repo project_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed By Cosign ${image2_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed ${image2_short_sha256} Close Browser Test Case - Replication Triggered By Events @@ -468,28 +468,28 @@ Test Case - Replication Triggered By Events Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Go Into Repo project${d} ${image1} - Should Be Signed By Cosign ${tag1} + Should Be Signed ${tag1} Go Into Repo project${d} ${index} - Should Be Signed By Cosign ${index_tag} + Should Be Signed ${index_tag} Go Into Repo project${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image1_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image1_short_sha256} Go Into Repo project${d} ${image2} - Should Not Be Signed By Cosign ${tag2} + Should Not Be Signed ${tag2} Go Into Repo project${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed By Cosign ${image2_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed ${image2_short_sha256} Logout Harbor Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Go Into Repo project_dest${d} ${image1} - Should Be Signed By Cosign ${tag1} + Should Be Signed ${tag1} Go Into Repo project_dest${d} ${index} - Should Be Signed By Cosign ${index_tag} + Should Be Signed ${index_tag} Go Into Repo project_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image1_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image1_short_sha256} Go Into Repo project_dest${d} ${image2} - Should Not Be Signed By Cosign ${tag2} + Should Not Be Signed ${tag2} Go Into Repo project_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed By Cosign ${image2_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Not Be Signed ${image2_short_sha256} Logout Harbor # delete Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -498,13 +498,13 @@ Test Case - Replication Triggered By Events Repo Not Exist project${d} ${image2} Go Into Repo project${d} ${image1} Retry Double Keywords When Error Delete Accessory ${tag1} Should be Accessory deleted ${tag1} - Should Not Be Signed By Cosign ${tag1} + Should Not Be Signed ${tag1} Go Into Repo project${d} ${index} Retry Double Keywords When Error Delete Accessory ${index_tag} Should be Accessory deleted ${index_tag} - Should Not Be Signed By Cosign ${index_tag} + Should Not Be Signed ${index_tag} Click Index Achieve ${index_tag} Retry Double Keywords When Error Delete Accessory ${image1_short_sha256} Should be Accessory deleted ${image1_short_sha256} - Should Not Be Signed By Cosign ${image1_short_sha256} + Should Not Be Signed ${image1_short_sha256} Logout Harbor Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -512,12 +512,12 @@ Test Case - Replication Triggered By Events Wait Until Page Contains We couldn't find any artifacts! Go Into Repo project_dest${d} ${image1} Should be Accessory deleted ${tag1} - Should Not Be Signed By Cosign ${tag1} + Should Not Be Signed ${tag1} Go Into Repo project_dest${d} ${index} Should be Accessory deleted ${index_tag} - Should Not Be Signed By Cosign ${index_tag} + Should Not Be Signed ${index_tag} Retry Double Keywords When Error Click Index Achieve ${index_tag} Should be Accessory deleted ${image1_short_sha256} - Should Not Be Signed By Cosign ${image1_short_sha256} + Should Not Be Signed ${image1_short_sha256} Close Browser Test Case - Enable Replication Of Cosign Deployment Security Policy @@ -595,15 +595,15 @@ Test Case - Enable Replication Of Cosign Deployment Security Policy Repo Exist project_pull_dest${d} ${image2} Repo Exist project_pull_dest${d} ${index} Go Into Repo project_pull_dest${d} ${image1} - Should Be Signed By Cosign ${tag1} + Should Be Signed ${tag1} Go Into Repo project_pull_dest${d} ${image2} - Should Be Signed By Cosign ${tag2} + Should Be Signed ${tag2} Go Into Repo project_pull_dest${d} ${index} - Should Be Signed By Cosign ${index_tag} + Should Be Signed ${index_tag} Go Into Repo project_pull_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image1_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image1_short_sha256} Go Into Repo project_pull_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image2_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image2_short_sha256} # check project_push_dest Go Into Project project_push_dest${d} Switch To Project Repo @@ -611,15 +611,15 @@ Test Case - Enable Replication Of Cosign Deployment Security Policy Repo Exist project_push_dest${d} ${image2} Repo Exist project_push_dest${d} ${index} Go Into Repo project_push_dest${d} ${image1} - Should Be Signed By Cosign ${tag1} + Should Be Signed ${tag1} Go Into Repo project_push_dest${d} ${image2} - Should Be Signed By Cosign ${tag2} + Should Be Signed ${tag2} Go Into Repo project_push_dest${d} ${index} - Should Be Signed By Cosign ${index_tag} + Should Be Signed ${index_tag} Go Into Repo project_push_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image1_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image1_short_sha256} Go Into Repo project_push_dest${d} ${index} - Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed By Cosign ${image2_short_sha256} + Retry Double Keywords When Error Click Index Achieve ${index_tag} Should Be Signed ${image2_short_sha256} Close Browser Test Case - Carvel Imgpkg Copy To Harbor