From b4a54adfeb17a2dc1305505cc8c773ddfb9d6c22 Mon Sep 17 00:00:00 2001 From: danfengliu Date: Mon, 18 May 2020 17:59:01 +0800 Subject: [PATCH] Add Scan schedule test in nightly 1. Add Scan schedule test in nightly; 2. Remove redundancy steps when create a new project; Signed-off-by: danfengliu --- tests/resources/Harbor-Pages/Project.robot | 4 +- .../Harbor-Pages/Vulnerability.robot | 7 ++ .../Harbor-Pages/Vulnerability_Elements.robot | 2 + tests/resources/TestCaseBody.robot | 26 +++--- tests/robot-cases/Group1-Nightly/Clair.robot | 4 +- tests/robot-cases/Group1-Nightly/Common.robot | 79 +++++++++---------- .../Group1-Nightly/Common_GC.robot | 6 +- tests/robot-cases/Group1-Nightly/GC.robot | 0 tests/robot-cases/Group1-Nightly/LDAP.robot | 10 +-- .../robot-cases/Group1-Nightly/Nightly.robot | 2 +- tests/robot-cases/Group1-Nightly/Notary.robot | 2 +- tests/robot-cases/Group1-Nightly/OIDC.robot | 4 +- .../Group1-Nightly/Replication.robot | 19 +++-- .../robot-cases/Group1-Nightly/Schedule.robot | 58 ++++++++++++++ tests/robot-cases/Group1-Nightly/Trivy.robot | 4 +- tests/robot-cases/Group1-Nightly/UAA.robot | 4 +- .../Group1-Nightly/multi_scanners.robot | 5 +- 17 files changed, 144 insertions(+), 92 deletions(-) delete mode 100644 tests/robot-cases/Group1-Nightly/GC.robot create mode 100644 tests/robot-cases/Group1-Nightly/Schedule.robot diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index 8aa41db8d..06bb77aad 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -19,7 +19,7 @@ Resource ../../resources/Util.robot *** Variables *** *** Keywords *** -Create An New Project +Create An New Project And Go Into Project [Arguments] ${projectname} ${public}=false ${count_quota}=${null} ${storage_quota}=${null} ${storage_quota_unit}=${null} Navigate To Projects Retry Button Click xpath=${create_project_button_xpath} @@ -40,7 +40,7 @@ Create An New Project With New User Create An New User url=${url} username=${username} email=${email} realname=${realname} newPassword=${newPassword} comment=${comment} Logout Harbor Sign In Harbor ${url} ${username} ${newPassword} - Create An New Project ${projectname} ${public} + Create An New Project And Go Into Project ${projectname} ${public} Sleep 1 #It's the log of project. diff --git a/tests/resources/Harbor-Pages/Vulnerability.robot b/tests/resources/Harbor-Pages/Vulnerability.robot index 5fde022ec..3f99a45a8 100644 --- a/tests/resources/Harbor-Pages/Vulnerability.robot +++ b/tests/resources/Harbor-Pages/Vulnerability.robot @@ -11,6 +11,13 @@ Disable Scan Schedule Retry Element Click ${vulnerability_dropdown_list_item_none} Retry Double Keywords When Error Retry Element Click ${vulnerability_save_btn} Retry Wait Until Page Not Contains Element ${vulnerability_save_btn} +Set Scan Schedule + [Arguments] ${type} ${value}=${null} + Retry Double Keywords When Error Retry Element Click ${vulnerability_edit_btn} Retry Wait Until Page Not Contains Element ${vulnerability_edit_btn} + Retry Element Click ${vulnerability_dropdown_list} + Run Keyword If '${type}'=='custom' Run Keywords Retry Element Click ${vulnerability_dropdown_list_item_custom} AND Retry Text Input ${targetCron_id} ${value} + Retry Double Keywords When Error Retry Element Click ${vulnerability_save_btn} Retry Wait Until Page Not Contains Element ${vulnerability_save_btn} + Trigger Scan Now And Wait Until The Result Appears Retry Element Click xpath=${scan_now_button} Sleep 60 diff --git a/tests/resources/Harbor-Pages/Vulnerability_Elements.robot b/tests/resources/Harbor-Pages/Vulnerability_Elements.robot index 35765de44..815ddb616 100644 --- a/tests/resources/Harbor-Pages/Vulnerability_Elements.robot +++ b/tests/resources/Harbor-Pages/Vulnerability_Elements.robot @@ -19,6 +19,7 @@ Documentation This resource provides any keywords related to the Harbor private ${vulnerability_edit_btn} xpath=//vulnerability-config//cron-selection//button[contains(.,'EDIT')] ${vulnerability_dropdown_list} xpath=//vulnerability-config//cron-selection//select[@id='selectPolicy'] ${vulnerability_dropdown_list_item_none} xpath=//vulnerability-config//cron-selection//select[@id='selectPolicy']//option[contains(.,'None')] +${vulnerability_dropdown_list_item_custom} xpath=//vulnerability-config//cron-selection//select[@id='selectPolicy']//option[contains(.,'Custom')] ${vulnerability_save_btn} xpath=//cron-selection//button[contains(.,'SAVE')] ${scan_now_button} //vulnerability-config//button[contains(.,'NOW')] ${vulnerability_page} //clr-vertical-nav-group-children/a[contains(.,'Vulnerability')] @@ -31,5 +32,6 @@ ${delete_scanner_confirm_btn} xpath=//clr-modal//button[contains(.,'DELETE')] ${scan_now_result} xpath=//div[@id="scan-result-container"] ${scanner_set_default} xpath=//button[@id="set-default"] ${scanner_set_default_success_xpath} //span[contains(.,'Successfully updated')] +${not_scanned_icon} xpath=//span[@class[contains(.,'not-scan')]] diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index 473afcafa..383eb459f 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -23,7 +23,7 @@ Body Of Manage project publicity ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user007 Test1@34 - Create An New Project project${d} public=true + Create An New Project And Go Into Project project${d} public=true Push image ${ip} user007 Test1@34 project${d} hello-world:latest Pull image ${ip} user008 Test1@34 project${d} hello-world:latest @@ -58,8 +58,7 @@ Body Of Scan A Tag In The Repo ${d}= get current date result_format=%m%s Sign In Harbor ${HARBOR_URL} user023 Test1@34 - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Push Image ${ip} user023 Test1@34 project${d} ${image_argument}:${tag_argument} Go Into Project project${d} Go Into Repo project${d}/${image_argument} @@ -102,7 +101,7 @@ Body Of View Scan Results ${d}= get current date result_format=%m%s Sign In Harbor ${HARBOR_URL} user025 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} user025 Test1@34 project${d} tomcat Go Into Project project${d} Go Into Repo project${d}/tomcat @@ -131,8 +130,7 @@ Body Of List Helm Charts ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user027 Test1@34 - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Switch To Project Charts Upload Chart files @@ -176,7 +174,7 @@ Delete A Project Without Sign In Harbor [Arguments] ${harbor_ip}=${ip} ${username}=${HARBOR_ADMIN} ${password}=${HARBOR_PASSWORD} ${d}= Get Current Date result_format=%m%s ${project_name}= Set Variable 000${d} - Create An New Project ${project_name} + Create An New Project And Go Into Project ${project_name} Push Image ${harbor_ip} ${username} ${password} ${project_name} hello-world Project Should Not Be Deleted ${project_name} Go Into Project ${project_name} @@ -187,7 +185,7 @@ Delete A Project Without Sign In Harbor Manage Project Member Without Sign In Harbor [Arguments] ${sign_in_user} ${sign_in_pwd} ${test_user1}=user005 ${test_user2}=user006 ${is_oidc_mode}=${false} ${d}= Get current Date result_format=%m%s - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push image ip=${ip} user=${sign_in_user} pwd=${sign_in_pwd} project=project${d} image=hello-world Logout Harbor @@ -208,10 +206,9 @@ Manage Project Member Without Sign In Harbor Helm CLI Push Without Sign In Harbor [Arguments] ${sign_in_user} ${sign_in_pwd} ${d}= Get Current Date result_format=%m%s - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Helm Repo Add ${HARBOR_URL} ${sign_in_user} ${sign_in_pwd} project_name=project${d} Helm Repo Push ${sign_in_user} ${sign_in_pwd} ${harbor_chart_filename} - Go Into Project project${d} has_image=${false} Switch To Project Charts Go Into Chart Version ${harbor_chart_name} Retry Wait Until Page Contains ${harbor_chart_version} @@ -220,9 +217,8 @@ Helm CLI Push Without Sign In Harbor Helm3 CLI Push Without Sign In Harbor [Arguments] ${sign_in_user} ${sign_in_pwd} ${d}= Get Current Date result_format=%m%s - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Helm Repo Push ${sign_in_user} ${sign_in_pwd} ${harbor_chart_filename} helm_repo_name=${HARBOR_URL}/chartrepo/project${d} helm_cmd=helm3 - Go Into Project project${d} has_image=${false} Switch To Project Charts Retry Double Keywords When Error Go Into Chart Version ${harbor_chart_name} Retry Wait Until Page Contains ${harbor_chart_version} Capture Page Screenshot @@ -240,7 +236,7 @@ Body Of Verfiy System Level CVE Whitelist ${signin_user}= Set Variable user025 ${signin_pwd}= Set Variable Test1@34 Sign In Harbor ${HARBOR_URL} ${signin_user} ${signin_pwd} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} sha256=${sha256} Go Into Project project${d} Set Vulnerabilty Serverity 2 @@ -272,7 +268,7 @@ Body Of Verfiy Project Level CVE Whitelist ${signin_user}= Set Variable user025 ${signin_pwd}= Set Variable Test1@34 Sign In Harbor ${HARBOR_URL} ${signin_user} ${signin_pwd} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} sha256=${sha256} Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256} Go Into Project project${d} @@ -305,7 +301,7 @@ Body Of Verfiy Project Level CVE Whitelist By Quick Way of Add System Add Items To System CVE Whitelist ${cve_list} Logout Harbor Sign In Harbor ${HARBOR_URL} ${signin_user} ${signin_pwd} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} sha256=${sha256} Go Into Project project${d} Set Vulnerabilty Serverity 2 diff --git a/tests/robot-cases/Group1-Nightly/Clair.robot b/tests/robot-cases/Group1-Nightly/Clair.robot index 1509fe473..41f2d519b 100644 --- a/tests/robot-cases/Group1-Nightly/Clair.robot +++ b/tests/robot-cases/Group1-Nightly/Clair.robot @@ -68,7 +68,7 @@ Test Case - View Scan Error ${d}= get current date result_format=%m%s Sign In Harbor ${HARBOR_URL} user026 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} user026 Test1@34 project${d} vmware/photon:1.0 Go Into Project project${d} Go Into Repo project${d}/vmware/photon @@ -94,7 +94,7 @@ Test Case - Project Level Image Serverity Policy #For internal CPE harbor registry ${sha256}= Set Variable 0e67625224c1da47cb3270e7a861a83e332f708d3d89dde0cbed432c94824d9a ${image}= Set Variable redis - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} sha256=${sha256} Go Into Project project${d} Go Into Repo ${image} diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index f261f876f..9ea89efb8 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -33,7 +33,7 @@ Test Case - Create An New Project Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project test${d} + Create An New Project And Go Into Project test${d} Close Browser Test Case - Delete A Project @@ -64,8 +64,8 @@ Test Case - Staticsinfo ${publicprojcount1}= Get Statics Public Project ${totalrepocount1}= Get Statics Total Repo ${totalprojcount1}= Get Statics Total Project - Create An New Project private${d} - Create An New Project public${d} true + Create An New Project And Go Into Project private${d} + Create An New Project And Go Into Project public${d} true Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} private${d} hello-world Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} public${d} hello-world Reload Page @@ -96,7 +96,7 @@ Test Case - Push Image Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project test${d} + Create An New Project And Go Into Project test${d} Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} test${d} hello-world:latest Go Into Project test${d} @@ -106,8 +106,7 @@ Test Case - Project Level Policy Public Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Goto Project Config Click Project Public Save Project Config @@ -191,9 +190,9 @@ Test Case - User View Projects Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user001 Test1@34 - Create An New Project test${d}1 - Create An New Project test${d}2 - Create An New Project test${d}3 + Create An New Project And Go Into Project test${d}1 + Create An New Project And Go Into Project test${d}2 + Create An New Project And Go Into Project test${d}3 Switch To Log Wait Until Page Contains test${d}1 Wait Until Page Contains test${d}2 @@ -205,7 +204,7 @@ Test Case - User View Logs ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user002 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push image ${ip} user002 Test1@34 project${d} busybox:latest Pull image ${ip} user002 Test1@34 project${d} busybox:latest @@ -271,7 +270,7 @@ Test Case - Edit Repo Info ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user011 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} user011 Test1@34 project${d} hello-world Go Into Project project${d} Go Into Repo project${d}/hello-world @@ -283,8 +282,8 @@ Test Case - Delete Multi Project ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user012 Test1@34 - Create An New Project projecta${d} - Create An New Project projectb${d} + Create An New Project And Go Into Project projecta${d} + Create An New Project And Go Into Project projectb${d} Push Image ${ip} user012 Test1@34 projecta${d} hello-world Navigate To Projects Filter Object project @@ -301,7 +300,7 @@ Test Case - Delete Multi Repo ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user013 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} user013 Test1@34 project${d} hello-world Push Image ${ip} user013 Test1@34 project${d} busybox Sleep 2 @@ -316,7 +315,7 @@ Test Case - Delete Multi Artifacts Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user014 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image With Tag ${ip} user014 Test1@34 project${d} redis 3.2.10-alpine 3.2.10-alpine Push Image With Tag ${ip} user014 Test1@34 project${d} redis 4.0.7-alpine 4.0.7-alpine Go Into Project project${d} @@ -331,7 +330,7 @@ Test Case - Delete Repo on CardView Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user015 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} user015 Test1@34 project${d} hello-world Push Image ${ip} user015 Test1@34 project${d} busybox Go Into Project project${d} @@ -345,8 +344,7 @@ Test Case - Delete Multi Member Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user016 Test1@34 - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Switch To Member Add Guest Member To Project user017 Add Guest Member To Project user018 @@ -358,8 +356,7 @@ Test Case - Project Admin Operate Labels Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user019 Test1@34 - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Sleep 2 # Add labels Switch To Project Label @@ -374,7 +371,7 @@ Test Case - Project Admin Add Labels To Repo Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user020 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image With Tag ${ip} user020 Test1@34 project${d} redis 3.2.10-alpine 3.2.10-alpine Push Image With Tag ${ip} user020 Test1@34 project${d} redis 4.0.7-alpine 4.0.7-alpine Go Into Project project${d} @@ -396,7 +393,7 @@ Test Case - Developer Operate Labels ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user021 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Logout Harbor Manage Project Member user021 Test1@34 project${d} user022 Add ${false} @@ -414,10 +411,9 @@ Test Case - Copy A Image ${random_num2}= Evaluate str(random.randint(1000,9999)) modules=random Sign In Harbor ${HARBOR_URL} user028 Test1@34 - Create An New Project project${random_num1} - Create An New Project project${random_num1}${random_num2} + Create An New Project And Go Into Project project${random_num1}${random_num2} + Create An New Project And Go Into Project project${random_num1} - Go Into Project project${random_num1} has_image=${false} Sleep 1 Push Image With Tag ${ip} user028 Test1@34 project${random_num1} redis ${image_tag} Sleep 1 @@ -439,7 +435,7 @@ Test Case - Create An New Project With Quotas Set ${storage_quota}= Set Variable 600 ${storage_quota_unit}= Set Variable GB Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} + Create An New Project And Go Into Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} ${storage_quota_ret}= Get Project Storage Quota Text From Project Quotas List project${d} Should Be Equal As Strings ${storage_quota_ret} 0Byte of ${storage_quota}${storage_quota_unit} Close Browser @@ -456,7 +452,7 @@ Test Case - Project Storage Quotas Dispaly And Control ${image_a_ver}= Set Variable donotremove5.0 ${image_b_ver}= Set Variable do_not_remove_6.8.3 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} + Create An New Project And Go Into Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_b} tag=${image_b_ver} tag1=${image_b_ver} ${storage_quota_ret}= Get Project Storage Quota Text From Project Quotas List project${d} Should Be Equal As Strings ${storage_quota_ret} ${image_b_size} of ${storage_quota}${storage_quota_unit} @@ -483,8 +479,8 @@ Test Case - Project Quotas Control Under Copy ${storage_quota}= Set Variable 330 ${storage_quota_unit}= Set Variable MB Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project_a_${d} - Create An New Project project_b_${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} + Create An New Project And Go Into Project project_a_${d} + Create An New Project And Go Into Project project_b_${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project_a_${d} ${image_a} tag=${image_a_ver} tag1=${image_a_ver} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project_a_${d} ${image_b} tag=${image_b_ver} tag1=${image_b_ver} Go Into Project project_a_${d} @@ -508,8 +504,7 @@ Test Case - Webhook CRUD Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Switch To Project Webhooks # create more than one webhooks Create A New Webhook webhook${d} https://test.com @@ -523,7 +518,7 @@ Test Case - Tag CRUD Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${d}= Get Current Date result_format=%m%s - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world latest Switch To Project Repo Go Into Repo hello-world @@ -542,7 +537,7 @@ Test Case - Tag Retention Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${d}= Get Current Date result_format=%m%s - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Switch To Tag Retention Add A Tag Retention Rule Delete A Tag Retention Rule @@ -559,8 +554,7 @@ Test Case - Tag Immutability Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${d}= Get Current Date result_format=%m%s - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Switch To Tag Immutability Add A Tag Immutability Rule 1212 3434 Delete A Tag Immutability Rule @@ -580,7 +574,7 @@ Test Case - Robot Account Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} ${token}= Create A Robot Account And Return Token project${d} robot${d} Log To Console ${token} Log ${token} @@ -596,7 +590,7 @@ Test Case - Push Docker Manifest Index and Display ${image_b_ver}= Set Variable latest Sign In Harbor ${HARBOR_URL} user010 Test1@34 - Create An New Project test${d} + Create An New Project And Go Into Project test${d} Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} test${d} ${image_a}:${image_a_ver} Go Into Project test${d} @@ -626,7 +620,7 @@ Test Case - Push CNAB Bundle and Display ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user010 Test1@34 - Create An New Project test${d} + Create An New Project And Go Into Project test${d} ${target}= Set Variable ${ip}/test${d}/cnab${d}:cnab_tag${d} CNAB Push Bundle ${ip} user010 Test1@34 ${target} ./tests/robot-cases/Group0-Util/bundle.json @@ -652,7 +646,7 @@ Test Case - Push Helm Chart and Display ${repo_name}= Set Variable harbor_chart_test Sign In Harbor ${HARBOR_URL} user010 Test1@34 - Create An New Project test${d} + Create An New Project And Go Into Project test${d} Helm Chart Push ${ip} user010 Test1@34 ${chart_file} ${archive} test${d} ${repo_name} ${verion} @@ -672,10 +666,9 @@ Test Case - Can Not Copy Image In ReadOnly Mode ${random_num2}= Evaluate str(random.randint(1000,9999)) modules=random Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${random_num1} - Create An New Project project${random_num1}${random_num2} + Create An New Project And Go Into Project project${random_num1}${random_num2} + Create An New Project And Go Into Project project${random_num1} - Go Into Project project${random_num1} has_image=${false} Sleep 1 Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${random_num1} redis ${image_tag} Sleep 1 @@ -695,7 +688,7 @@ Test Case - Read Only Mode Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Enable Read Only Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest diff --git a/tests/robot-cases/Group1-Nightly/Common_GC.robot b/tests/robot-cases/Group1-Nightly/Common_GC.robot index abece76b8..28f3ba0b2 100644 --- a/tests/robot-cases/Group1-Nightly/Common_GC.robot +++ b/tests/robot-cases/Group1-Nightly/Common_GC.robot @@ -29,7 +29,7 @@ Test Case - Garbage Collection ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world Sleep 2 Go Into Project project${d} @@ -43,7 +43,7 @@ Test Case - GC Untagged Images Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${d}= Get Current Date result_format=%m%s - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world latest # make hello-world untagged Go Into Project project${d} @@ -80,7 +80,7 @@ Test Case - Project Quotas Control Under GC ${image_a_ver}= Set Variable 6.8.3 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Capture Page Screenshot - Create An New Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} + Create An New Project And Go Into Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} Capture Page Screenshot Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_a}:${image_a_ver} err_msg=will exceed the configured upper limit of 200.0 MiB Capture Page Screenshot diff --git a/tests/robot-cases/Group1-Nightly/GC.robot b/tests/robot-cases/Group1-Nightly/GC.robot deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/robot-cases/Group1-Nightly/LDAP.robot b/tests/robot-cases/Group1-Nightly/LDAP.robot index 746c003de..7b7511692 100644 --- a/tests/robot-cases/Group1-Nightly/LDAP.robot +++ b/tests/robot-cases/Group1-Nightly/LDAP.robot @@ -47,8 +47,7 @@ Test Case - System Admin On-board New Member Sleep 2 Page Should Not Contain mike02 Navigate To Projects - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Switch To Member Add Guest Member To Project mike02 Page Should Contain mike02 @@ -58,8 +57,7 @@ Test Case - LDAP User On-borad New Member Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} mike03 zhu88jie - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Switch To Member Sleep 2 Page Should Not Contain mike04 @@ -94,7 +92,7 @@ Test Case - Ldap User Create Project Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} mike zhu88jie - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Logout Harbor Manage Project Member ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} mike02 Add has_image=${false} Close Browser @@ -103,7 +101,7 @@ Test Case - Ldap User Push An Image Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} mike zhu88jie - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} mike zhu88jie project${d} hello-world:latest Go Into Project project${d} diff --git a/tests/robot-cases/Group1-Nightly/Nightly.robot b/tests/robot-cases/Group1-Nightly/Nightly.robot index a4bb5ca3b..9589bf5e7 100644 --- a/tests/robot-cases/Group1-Nightly/Nightly.robot +++ b/tests/robot-cases/Group1-Nightly/Nightly.robot @@ -54,7 +54,7 @@ Test Case - Repo Size Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library alpine 2.6 2.6 Go Into Project library Go Into Repo alpine - Page Should Contain 1.92MB + Page Should Contain 1.92MB Close Browser Test Case - Staticsinfo diff --git a/tests/robot-cases/Group1-Nightly/Notary.robot b/tests/robot-cases/Group1-Nightly/Notary.robot index 189052460..3de07e6e5 100644 --- a/tests/robot-cases/Group1-Nightly/Notary.robot +++ b/tests/robot-cases/Group1-Nightly/Notary.robot @@ -28,7 +28,7 @@ Test Case - Project Level Policy Content Trust Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest Go Into Project project${d} Goto Project Config diff --git a/tests/robot-cases/Group1-Nightly/OIDC.robot b/tests/robot-cases/Group1-Nightly/OIDC.robot index a426e65aa..d98d4bb0b 100644 --- a/tests/robot-cases/Group1-Nightly/OIDC.robot +++ b/tests/robot-cases/Group1-Nightly/OIDC.robot @@ -51,7 +51,7 @@ Test Case - OIDC User Sign In Test Case - Create An New Project Sign In Harbor With OIDC User ${HARBOR_URL} ${d}= Get Current Date result_format=%m%s - Create An New Project test${d} + Create An New Project And Go Into Project test${d} Close Browser Test Case - Delete A Project @@ -73,7 +73,7 @@ Test Case - Generate User CLI Secret ${d}= Get current Date result_format=%m%s ${image}= Set Variable hello-world Sign In Harbor With OIDC User ${HARBOR_URL} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} ${secret_old}= Get Secrete By API ${HARBOR_URL} Push image ip=${ip} user=${OIDC_USERNAME} pwd=${secret_old} project=project${d} image=${image} ${secret_new}= Generate And Return Secret ${HARBOR_URL} diff --git a/tests/robot-cases/Group1-Nightly/Replication.robot b/tests/robot-cases/Group1-Nightly/Replication.robot index 96f8e1eab..fad0e7f6c 100644 --- a/tests/robot-cases/Group1-Nightly/Replication.robot +++ b/tests/robot-cases/Group1-Nightly/Replication.robot @@ -152,7 +152,7 @@ Test Case - Replication Of Pull Images from DockerHub To Self ${d}= Get Current Date result_format=%m%s #login source Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Switch To Registries Create A New Endpoint docker-hub e${d} https://hub.docker.com/ danfengliu Aa123456 Y Switch To Replication Manage @@ -168,7 +168,7 @@ Test Case - Replication Of Push Images from Self To Harbor ${d}= Get Current Date result_format=%m%s #login source Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world v1 @@ -179,7 +179,7 @@ Test Case - Replication Of Push Images from Self To Harbor #logout and login target Logout Harbor Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project_dest${d} + Create An New Project And Go Into Project project_dest${d} #logout and login source Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -197,8 +197,7 @@ Test Case - Replication Of Push Chart from Self To Harbor ${d}= Get Current Date result_format=%m%s #login source Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Switch To Project Charts Upload Chart files Switch To Registries @@ -208,7 +207,7 @@ Test Case - Replication Of Push Chart from Self To Harbor #logout and login target Logout Harbor Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project_dest${d} + Create An New Project And Go Into Project project_dest${d} #logout and login source Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -229,7 +228,7 @@ Test Case - Replication Of Push Images from Self To Harbor By Push Event ${d}= Get Current Date result_format=%m%s #login source Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Switch To Registries Create A New Endpoint harbor e${d} https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Replication Manage @@ -238,7 +237,7 @@ Test Case - Replication Of Push Images from Self To Harbor By Push Event #logout and login target Logout Harbor Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project_dest${d} + Create An New Project And Go Into Project project_dest${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} centos Image Should Be Replicated To Project project_dest${d} centos Close Browser @@ -248,7 +247,7 @@ Test Case - Replication Of Pull Images from AWS-ECR To Self ${d}= Get Current Date result_format=%m%s #login source Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Switch To Registries Create A New Endpoint aws-ecr e${d} us-east-2 ${ecr_ac_id} ${ecr_ac_key} Y Switch To Replication Manage @@ -264,7 +263,7 @@ Test Case - Replication Of Pull Images from Google-GCR To Self ${d}= Get Current Date result_format=%m%s #login source Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Switch To Registries Create A New Endpoint google-gcr e${d} asia.gcr.io ${null} ${gcr_ac_key} Y Switch To Replication Manage diff --git a/tests/robot-cases/Group1-Nightly/Schedule.robot b/tests/robot-cases/Group1-Nightly/Schedule.robot new file mode 100644 index 000000000..07a62ba04 --- /dev/null +++ b/tests/robot-cases/Group1-Nightly/Schedule.robot @@ -0,0 +1,58 @@ + +// Copyright (c) 2017 VMware, Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +*** Settings *** +Documentation Harbor BATs +Resource ../../resources/Util.robot +Default Tags Nightly + +*** Variables *** +${HARBOR_URL} https://${ip} +${SSH_USER} root +${HARBOR_ADMIN} admin + +*** Test Cases *** +Test Case - Scan Schedule Job + Init Chrome Driver + ${d}= Get Current Date result_format=%M + Log To Console ${d} + ${project_name}= Set Variable scan_schedule_proj + ${image}= Set Variable redis + ${tag}= Set Variable latest + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Create An New Project And Go Into Project ${project_name} + Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_name} ${image}:${tag} + Go Into Repo ${project_name}/${image} + Retry Wait Until Page Contains Element ${not_scanned_icon} + Switch To Vulnerability Page + ${flag}= Set Variable ${false} + :FOR ${i} IN RANGE 999999 + \ ${minite}= Get Current Date result_format=%M + \ ${left} = Evaluate ${minite}%10 + \ ${d} = Convert To Integer ${left} + \ Log To Console ${i}/${d} + \ Run Keyword If ${d} <= 3 Run Keywords Set Scan Schedule custom value=* */10 * * * * AND Set Suite Variable ${flag} ${true} + \ Sleep 55 + \ Exit For Loop If '${flag}' == '${true}' + # After scan custom schedule is set, image should stay in unscanned status. + Sleep 360 + Go Into Project ${project_name} + Go Into Repo ${project_name}/${image} + Retry Wait Until Page Contains Element ${not_scanned_icon} + Sleep 360 + Scan Result Should Display In List Row ${tag} + View Repo Scan Details Critical High Medium + + diff --git a/tests/robot-cases/Group1-Nightly/Trivy.robot b/tests/robot-cases/Group1-Nightly/Trivy.robot index a4b936f62..8e8ec8fce 100644 --- a/tests/robot-cases/Group1-Nightly/Trivy.robot +++ b/tests/robot-cases/Group1-Nightly/Trivy.robot @@ -72,7 +72,7 @@ Test Case - View Scan Error ${d}= get current date result_format=%m%s Sign In Harbor ${HARBOR_URL} user026 Test1@34 - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} user026 Test1@34 project${d} busybox:latest Go Into Project project${d} Go Into Repo project${d}/busybox @@ -98,7 +98,7 @@ Test Case - Project Level Image Serverity Policy #For internal CPE harbor registry ${sha256}= Set Variable 0e67625224c1da47cb3270e7a861a83e332f708d3d89dde0cbed432c94824d9a ${image}= Set Variable redis - Create An New Project project${d} + Create An New Project And Go Into Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} sha256=${sha256} Go Into Project project${d} Go Into Repo ${image} diff --git a/tests/robot-cases/Group1-Nightly/UAA.robot b/tests/robot-cases/Group1-Nightly/UAA.robot index 11c38d901..3124d5d29 100644 --- a/tests/robot-cases/Group1-Nightly/UAA.robot +++ b/tests/robot-cases/Group1-Nightly/UAA.robot @@ -57,8 +57,8 @@ Test Case - UAA User Push An Image Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} mike zhu88jie - Create An New Project project${d} - + Create An New Project And Go Into Project project${d} + Push Image ${ip} mike zhu88jie project${d} hello-world:latest Go Into Project project${d} Wait Until Page Contains project${d}/hello-world diff --git a/tests/robot-cases/Group1-Nightly/multi_scanners.robot b/tests/robot-cases/Group1-Nightly/multi_scanners.robot index 96523cd31..af53d0da5 100644 --- a/tests/robot-cases/Group1-Nightly/multi_scanners.robot +++ b/tests/robot-cases/Group1-Nightly/multi_scanners.robot @@ -37,8 +37,7 @@ Test Case - Switch Scanner Should Display The Default Trivy Scanner - Create An New Project project${d} - Go Into Project project${d} has_image=${false} + Create An New Project And Go Into Project project${d} Push Image ${ip} admin Harbor12345 project${d} hello-world:latest Go Into Project project${d} Go Into Repo project${d}/hello-world @@ -50,7 +49,7 @@ Test Case - Switch Scanner Set Default Scanner Clair Should Display The Default Clair Scanner - Go Into Project project${d} + Go Into Project project${d} Go Into Repo project${d}/hello-world Scan Repo latest Succeed Move To Summary Chart