From 9e42f6d6d1d4a4d34a96af13be1f51419d430d71 Mon Sep 17 00:00:00 2001 From: danfengliu Date: Fri, 27 Dec 2019 14:11:01 +0800 Subject: [PATCH] Notary test failed in Helm Pipeline In Helm pipeline, harbor access address is by domain name instead of IP, so cert directory should be created by domain name. Signed-off-by: danfengliu --- tests/resources/Docker-Util.robot | 9 +- .../Harbor-Pages/Administration-Users.robot | 14 +- .../Harbor-Pages/Configuration.robot | 6 +- tests/resources/Harbor-Pages/HomePage.robot | 5 +- .../Harbor-Pages/HomePage_Elements.robot | 1 + tests/resources/Harbor-Pages/LDAP-Mode.robot | 4 +- .../Harbor-Pages/Project-Config.robot | 2 +- .../Harbor-Pages/Project-Members.robot | 51 ++-- .../Project-Members_Elements.robot | 2 +- .../Project-Repository_Elements.robot | 3 + .../Harbor-Pages/Project-Webhooks.robot | 1 + tests/resources/Harbor-Pages/Project.robot | 6 +- .../Harbor-Pages/Project_Elements.robot | 8 +- .../resources/Harbor-Pages/Replication.robot | 28 +- tests/resources/Harbor-Pages/ToolKit.robot | 10 +- .../resources/Harbor-Pages/UserProfile.robot | 2 +- tests/resources/Harbor-Pages/Verify.robot | 48 ++- tests/resources/Helm-Util.robot | 2 +- tests/resources/TestCaseBody.robot | 13 +- tests/resources/Util.robot | 11 +- tests/robot-cases/Group0-BAT/BAT.robot | 8 +- .../Group0-Util/notary-push-image.sh | 9 +- .../Group1-Nightly/Chartmuseum.robot | 9 +- tests/robot-cases/Group1-Nightly/Common.robot | 14 +- tests/robot-cases/Group1-Nightly/LDAP.robot | 3 + .../robot-cases/Group1-Nightly/Nightly.robot | 2 +- tests/robot-cases/Group1-Nightly/Notary.robot | 2 +- .../Group1-Nightly/Replication.robot | 66 ++-- tests/robot-cases/Group1-Nightly/UAA.robot | 2 +- tests/robot-cases/Group3-Upgrade/data.json | 162 +++++++++- .../Group3-Upgrade/feature_map.json | 192 ++++++++++++ tests/robot-cases/Group3-Upgrade/prepare.py | 286 +++++++++++++++--- tests/robot-cases/Group3-Upgrade/verify.robot | 28 ++ 33 files changed, 812 insertions(+), 197 deletions(-) create mode 100644 tests/robot-cases/Group3-Upgrade/feature_map.json diff --git a/tests/resources/Docker-Util.robot b/tests/resources/Docker-Util.robot index 038246c6b..dc76ceb1b 100644 --- a/tests/resources/Docker-Util.robot +++ b/tests/resources/Docker-Util.robot @@ -28,7 +28,9 @@ Pull image ${image_with_tag}= Set Variable If '${tag}'=='${null}' ${image} ${image}:${tag} Run Keyword If ${is_robot}==${false} Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip} ... ELSE Wait Unitl Command Success docker login -u robot\\\$${user} -p ${pwd} ${ip} - ${output}= Wait Unitl Command Success docker pull ${ip}/${project}/${image_with_tag} + ${output}= Docker Pull ${ip}/${project}/${image_with_tag} + Log ${output} + Log To Console ${output} Should Contain ${output} Digest: Should Contain ${output} Status: Should Not Contain ${output} No such image: @@ -140,7 +142,10 @@ Docker Login Docker Pull [Arguments] ${image} - Wait Unitl Command Success docker pull ${image} + ${output}= Retry Keyword When Error Wait Unitl Command Success docker pull ${image} + Log ${output} + Log To Console Docker Pull: \n ${output} + [Return] ${output} Docker Tag [Arguments] ${src_image} ${dst_image} diff --git a/tests/resources/Harbor-Pages/Administration-Users.robot b/tests/resources/Harbor-Pages/Administration-Users.robot index cd15d0af5..be3a67571 100644 --- a/tests/resources/Harbor-Pages/Administration-Users.robot +++ b/tests/resources/Harbor-Pages/Administration-Users.robot @@ -21,28 +21,28 @@ Resource ../../resources/Util.robot *** Keywords *** Assign User Admin [Arguments] ${user} - Click Element xpath=//harbor-user//hbr-filter//clr-icon + Retry Element Click xpath=//harbor-user//hbr-filter//clr-icon Input Text xpath=//harbor-user//hbr-filter//input ${user} Sleep 2 #select checkbox - Click Element //clr-dg-row[contains(.,'${user}')]//label + Retry Element Click //clr-dg-row[contains(.,'${user}')]//label #click assign admin - Click Element //*[@id='set-admin'] + Retry Element Click //*[@id='set-admin'] Sleep 1 Switch to User Tag - Click Element xpath=${administration_user_tag_xpath} + Retry Element Click xpath=${administration_user_tag_xpath} Sleep 1 Administration Tag Should Display - Page Should Contain Element xpath=${administration_tag_xpath} + Retry Wait Until Page Contains Element xpath=${administration_tag_xpath} User Email Should Exist [Arguments] ${email} Sign In Harbor ${HARBOR_URL} %{HARBOR_ADMIN} %{HARBOR_PASSWORD} Switch to User Tag - Page Should Contain Element xpath=//clr-dg-cell[contains(., '${email}')] + Retry Wait Until Page Contains Element xpath=//clr-dg-cell[contains(., '${email}')] Add User Button Should Be Disabled Sleep 1 - Page Should Contain Element //button[contains(.,'New') and @disabled=''] + Retry Wait Until Page Contains Element //button[contains(.,'New') and @disabled=''] diff --git a/tests/resources/Harbor-Pages/Configuration.robot b/tests/resources/Harbor-Pages/Configuration.robot index d619698a1..49dbf5a22 100644 --- a/tests/resources/Harbor-Pages/Configuration.robot +++ b/tests/resources/Harbor-Pages/Configuration.robot @@ -140,10 +140,10 @@ Self Reg Should Be Enabled Checkbox Should Be Selected xpath=${self_reg_xpath} Project Creation Should Display - Page Should Contain Element xpath=${project_create_xpath} + Retry Wait Until Page Contains Element xpath=${project_create_xpath} Project Creation Should Not Display - Page Should Not Contain Element xpath=${project_create_xpath} + Retry Wait Until Page Not Contains Element xpath=${project_create_xpath} ## System settings Switch To System Settings @@ -153,6 +153,8 @@ Switch To System Settings Sleep 1 Switch To Project Quotas + Sleep 1 + Retry Element Click xpath=${configuration_xpath} Sleep 1 Retry Element Click xpath=//clr-main-container//clr-vertical-nav//a[contains(.,'Project Quotas')] Sleep 1 diff --git a/tests/resources/Harbor-Pages/HomePage.robot b/tests/resources/Harbor-Pages/HomePage.robot index c5f8ad83c..b5bffd495 100644 --- a/tests/resources/Harbor-Pages/HomePage.robot +++ b/tests/resources/Harbor-Pages/HomePage.robot @@ -20,7 +20,7 @@ Resource ../../resources/Util.robot *** Keywords *** Sign In Harbor - [Arguments] ${url} ${user} ${pw} + [Arguments] ${url} ${user} ${pw} ${is_close_scan_plugin_mesg}=${false} Go To ${url} Retry Wait Element ${harbor_span_title} Retry Wait Element ${login_name} @@ -31,13 +31,14 @@ Sign In Harbor Retry Button Click ${login_btn} Log To Console ${user} Retry Wait Element xpath=//span[contains(., '${user}')] + Run Keyword If ${is_close_scan_plugin_mesg}==${true} Run Keyword And Ignore Error Retry Element Click ${close_scan_plugin_mesg} Capture Screenshot And Source Capture Page Screenshot Log Source Sign Up Should Not Display - Page Should Not Contain Element xpath=${sign_up_button_xpath} + Retry Wait Until Page Not Contains Element xpath=${sign_up_button_xpath} Create An New User [Arguments] ${url} ${username} ${email} ${realname} ${newPassword} ${comment} diff --git a/tests/resources/Harbor-Pages/HomePage_Elements.robot b/tests/resources/Harbor-Pages/HomePage_Elements.robot index 15e54c186..daabf851c 100644 --- a/tests/resources/Harbor-Pages/HomePage_Elements.robot +++ b/tests/resources/Harbor-Pages/HomePage_Elements.robot @@ -30,4 +30,5 @@ ${login_btn} //*[@id='log_in'] ${harbor_span_title} //span[contains(., 'Harbor')] ${login_name} //*[@id='login_username'] ${login_pwd} //*[@id='login_password'] +${close_scan_plugin_mesg} //clr-icon[@class='close-icon'] diff --git a/tests/resources/Harbor-Pages/LDAP-Mode.robot b/tests/resources/Harbor-Pages/LDAP-Mode.robot index d14b53430..fa0cf4c43 100644 --- a/tests/resources/Harbor-Pages/LDAP-Mode.robot +++ b/tests/resources/Harbor-Pages/LDAP-Mode.robot @@ -21,8 +21,8 @@ Resource ../../resources/Util.robot *** Keywords *** Ldap User Should Not See Change Password - Click Element //clr-header//clr-dropdown[2]//button - Sleep 1 + Retry Element Click //clr-header//clr-dropdown[2]//button + Sleep 2 Page Should Not Contain Password diff --git a/tests/resources/Harbor-Pages/Project-Config.robot b/tests/resources/Harbor-Pages/Project-Config.robot index 96eb835a9..c0e6c4871 100644 --- a/tests/resources/Harbor-Pages/Project-Config.robot +++ b/tests/resources/Harbor-Pages/Project-Config.robot @@ -42,7 +42,7 @@ Public Should Be Selected Project Should Be Public [Arguments] ${projectName} - Page Should Contain Element //clr-dg-row[contains(.,'${projectName}')]//clr-dg-cell[contains(.,'Public')] + Retry Wait Until Page Contains Element //clr-dg-row[contains(.,'${projectName}')]//clr-dg-cell[contains(.,'Public')] Content Trust Should Be Selected Checkbox Should Be Selected //hbr-project-policy-config//input[@name='content-trust'] diff --git a/tests/resources/Harbor-Pages/Project-Members.robot b/tests/resources/Harbor-Pages/Project-Members.robot index 5dd7a6a0f..2dd20ccc3 100644 --- a/tests/resources/Harbor-Pages/Project-Members.robot +++ b/tests/resources/Harbor-Pages/Project-Members.robot @@ -21,28 +21,24 @@ Resource ../../resources/Util.robot *** Keywords *** Go Into Project [Arguments] ${project} ${has_image}=${true} - Sleep 2 - Retry Wait Element ${search_input} - Input Text ${search_input} ${project} - Retry Wait Until Page Contains ${project} - Retry Element Click xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a - #To prevent waiting for a fixed-period of time for page loading and failure caused by exception, we add loop to re-run when - # exception was caught. :For ${n} IN RANGE 1 5 - \ ${out} Run Keyword If ${has_image}==${false} Run Keyword And Ignore Error Wait Until Element Is Visible And Enabled xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any repositories!\")] - \ ... ELSE Run Keyword And Ignore Error Wait Until Element Is Visible And Enabled xpath=//clr-dg-cell[contains(.,'${project}/')] - \ Log To Console ${out[0]} - \ ${result} Set Variable If '${out[0]}'=='PASS' ${true} ${false} - \ Run Keyword If ${result} == ${true} Exit For Loop + \ Sleep 2 + \ Retry Wait Element ${search_input} + \ Retry Clear Element Text ${search_input} + \ Input Text ${search_input} ${project} + \ ${out} Run Keyword If ${has_image}==${false} Retry Double Keywords When Error Retry Element Click xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a Wait Until Element Is Visible And Enabled xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any repositories!\")] DoAssert=${false} + \ ... ELSE Retry Double Keywords When Error Retry Element Click xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a Wait Until Element Is Visible And Enabled xpath=//project-detail//hbr-repository-gridview//clr-dg-cell[contains(.,'${project}/')] DoAssert=${false} + \ Log To Console ${out} + \ Run Keyword If ${out} == 'PASS' Exit For Loop \ Sleep 1 - Should Be Equal ${result} ${true} + Should Be Equal ${out} 'PASS' Sleep 1 Add User To Project Admin [Arguments] ${project} ${user} # *** this keyword has not been used *** Go Into Project - Retry Element Click xpath=${project_member_tag_xpath} + Retry Element Click xpath=${project_member_xpath} Retry Element Click xpath=${project_member_add_button_xpath} Retry Text Input xpath=${project_member_add_username_xpath} ${user} Retry Element Click xpath=${project_member_add_admin_xpath} @@ -61,7 +57,7 @@ Search Project Member Change Project Member Role [Arguments] ${project} ${user} ${role} Retry Element Click xpath=//clr-dg-cell//a[contains(.,'${project}')] - Retry Element Click xpath=${project_member_tag_xpath} + Retry Element Click xpath=${project_member_xpath} Retry Element Click xpath=//project-detail//clr-dg-row[contains(.,'${user}')]//clr-checkbox-wrapper #change role Retry Element Click ${project_member_action_xpath} @@ -74,13 +70,13 @@ User Can Change Role [arguments] ${username} Retry Element Click xpath=//clr-dg-row[contains(.,'${username}')]//input/../label Retry Element Click xpath=//*[@id='member-action'] - Page Should Not Contain Element xpath=//button[@disabled='' and contains(.,'Admin')] + Retry Wait Until Page Not Contains Element xpath=//button[@disabled='' and contains(.,'Admin')] User Can Not Change Role [arguments] ${username} Retry Element Click xpath=//clr-dg-row[contains(.,'${username}')]//input/../label Retry Element Click xpath=//*[@id='member-action'] - Page Should Contain Element xpath=//button[@disabled='' and contains(.,'Admin')] + Retry Wait Until Page Contains Element xpath=//button[@disabled='' and contains(.,'Admin')] #this keyworkd seems will not use any more, will delete in the future Non-admin View Member Account @@ -88,11 +84,11 @@ Non-admin View Member Account Xpath Should Match X Times //clr-dg-row-master ${times} User Can Not Add Member - Page Should Contain Element xpath=//button[@disabled='' and contains(.,'User')] + Retry Wait Until Page Contains Element xpath=//button[@disabled='' and contains(.,'User')] Add Guest Member To Project [arguments] ${member} - Retry Element Click xpath=${project_member_add_button_xpath} + Retry Double Keywords When Error Retry Element Click xpath=${project_member_add_button_xpath} Retry Wait Until Page Contains Element xpath=${project_member_add_button_xpath} Retry Text Input xpath=${project_member_add_username_xpath} ${member} #select guest Mouse Down xpath=${project_member_guest_radio_checkbox} @@ -102,10 +98,9 @@ Add Guest Member To Project Delete Project Member [arguments] ${member} Retry Element Click xpath=//clr-dg-row[contains(.,'${member}')]//input/../label - Retry Element Click ${member_action_xpath} - Retry Element Click ${delete_action_xpath} - Retry Element Click ${repo_delete_on_card_view_btn} - Retry Wait Element xpath=${project_member_xpath} + Retry Double Keywords When Error Retry Element Click ${member_action_xpath} Retry Wait Until Page Contains Element ${delete_action_xpath} + Retry Double Keywords When Error Retry Element Click ${delete_action_xpath} Retry Wait Until Page Contains Element ${repo_delete_on_card_view_btn} + Retry Double Keywords When Error Retry Element Click ${repo_delete_on_card_view_btn} Retry Wait Element xpath=${project_member_xpath} Sleep 1 User Should Be Owner Of Project @@ -157,7 +152,7 @@ User Should Be Guest Go Into Project ${project} Switch To Member User Can Not Add Member - Page Should Contain Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Guest')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Guest')] Logout Harbor Pull image ${ip} ${user} ${password} ${project} hello-world Cannot Push image ${ip} ${user} ${password} ${project} hello-world @@ -172,7 +167,7 @@ User Should Be Developer Go Into Project ${project} Switch To Member User Can Not Add Member - Page Should Contain Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Developer')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Developer')] Logout Harbor Push Image With Tag ${ip} ${user} ${password} ${project} hello-world v1 @@ -187,7 +182,7 @@ User Should Be Admin Switch To Member Add Guest Member To Project ${guest} User Can Change Role ${guest} - Page Should Contain Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Admin')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Admin')] Logout Harbor Push Image With Tag ${ip} ${user} ${password} ${project} hello-world v2 @@ -201,7 +196,7 @@ User Should Be Master Go Into Project ${project} Delete Repo ${project} Switch To Member - Page Should Contain Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Master')] + Retry Wait Until Page Contains Element xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Master')] Logout Harbor Push Image With Tag ${ip} ${user} ${password} ${project} hello-world v3 @@ -210,5 +205,5 @@ Project Should Have Member Sign In Harbor ${HARBOR_URL} %{HARBOR_ADMIN} %{HARBOR_PASSWORD} Go Into Project ${project} Switch To Member - Page Should Contain Element xpath=//clr-dg-cell[contains(., '${user}')] + Retry Wait Until Page Contains Element xpath=//clr-dg-cell[contains(., '${user}')] Logout Harbor diff --git a/tests/resources/Harbor-Pages/Project-Members_Elements.robot b/tests/resources/Harbor-Pages/Project-Members_Elements.robot index c30342c49..e1b73850f 100644 --- a/tests/resources/Harbor-Pages/Project-Members_Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Members_Elements.robot @@ -16,7 +16,7 @@ Documentation This resource provides any keywords related to the Harbor private registry appliance *** Variables *** -${project_member_tag_xpath} //clr-main-container//project-detail/nav/ul//a[contains(.,'Members')] +${project_member_tag_xpath} //clr-main-container//project-detail/clr-tabs//a[contains(.,'Members')] ${project_member_add_button_xpath} //project-detail//button[contains(.,'User')] ${project_member_add_username_xpath} //*[@id='member_name'] ${project_member_add_admin_xpath} /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/ng-component/div/div[1]/div/div[1]/add-member/clr-modal/div/div[1]/div/div[1]/div/div[2]/form/section/div[2]/div[1]/label diff --git a/tests/resources/Harbor-Pages/Project-Repository_Elements.robot b/tests/resources/Harbor-Pages/Project-Repository_Elements.robot index 52ab139d9..053133296 100644 --- a/tests/resources/Harbor-Pages/Project-Repository_Elements.robot +++ b/tests/resources/Harbor-Pages/Project-Repository_Elements.robot @@ -21,3 +21,6 @@ ${first_cve_xpath} //clr-dg-row[1]//clr-dg-cell//a ${view_log_xpath} //clr-dg-row[1]//clr-dg-cell[4]//a ${build_history_btn} //button[contains(.,'Build History')] ${build_history_data} //clr-dg-row +${push_image_command_btn} //hbr-push-image-button//button + + diff --git a/tests/resources/Harbor-Pages/Project-Webhooks.robot b/tests/resources/Harbor-Pages/Project-Webhooks.robot index 1d913498e..0329e1f71 100644 --- a/tests/resources/Harbor-Pages/Project-Webhooks.robot +++ b/tests/resources/Harbor-Pages/Project-Webhooks.robot @@ -6,6 +6,7 @@ Resource ../../resources/Util.robot *** Keywords *** Switch To Project Webhooks + #Switch To Project Tab Overflow Retry Element Click xpath=//project-detail//a[contains(.,'Webhooks')] Sleep 1 diff --git a/tests/resources/Harbor-Pages/Project.robot b/tests/resources/Harbor-Pages/Project.robot index 987e6d7d8..db9e61f18 100644 --- a/tests/resources/Harbor-Pages/Project.robot +++ b/tests/resources/Harbor-Pages/Project.robot @@ -27,7 +27,7 @@ Create An New Project Capture Page Screenshot Retry Text Input xpath=${project_name_xpath} ${projectname} ${element_project_public}= Set Variable xpath=${project_public_xpath} - Run Keyword If '${public}' == 'true' Run Keywords Wait Until Element Is Visible And Enabled ${element_project_public} AND Click Element ${element_project_public} + Run Keyword If '${public}' == 'true' Run Keywords Wait Until Element Is Visible And Enabled ${element_project_public} AND Retry Element Click ${element_project_public} Run Keyword If '${count_quota}'!='${null}' Input Count Quota ${count_quota} Run Keyword If '${storage_quota}'!='${null}' Input Storage Quota ${storage_quota} ${storage_quota_unit} Capture Page Screenshot @@ -45,6 +45,7 @@ Create An New Project With New User #It's the log of project. Go To Project Log + #Switch To Project Tab Overflow Retry Element Click xpath=${project_log_xpath} Sleep 2 @@ -183,7 +184,7 @@ Do Log Advanced Search Retry Text Input xpath=//audit-log//hbr-filter//input harbor Sleep 1 Capture Page Screenshot LogAdvancedSearch2.png - ${rc} = Get Matching Xpath Count //audit-log//clr-dg-row + ${rc} = Get Element Count //audit-log//clr-dg-row Should Be Equal As Integers ${rc} 0 Go Into Repo @@ -200,7 +201,6 @@ Go Into Repo \ Sleep 2 Capture Page Screenshot Retry Double Keywords When Error Retry Element Click ${repo_name_element} Retry Wait Until Page Not Contains Element ${repo_name_element} - Capture Page Screenshot Retry Wait Element ${tag_table_column_pull_command} Retry Wait Element ${tag_images_btn} Capture Page Screenshot diff --git a/tests/resources/Harbor-Pages/Project_Elements.robot b/tests/resources/Harbor-Pages/Project_Elements.robot index 72f7c1081..fe680b725 100644 --- a/tests/resources/Harbor-Pages/Project_Elements.robot +++ b/tests/resources/Harbor-Pages/Project_Elements.robot @@ -23,13 +23,13 @@ ${project_save_css} html body.no-scrolling harbor-app harbor-shell clr-main-con ${log_xpath} //clr-main-container//clr-vertical-nav//a[contains(.,'Logs')] ${projects_xpath} //clr-main-container//clr-vertical-nav//a[contains(.,'Projects')] ${project_replication_xpath} //project-detail//a[contains(.,'Replication')] -${project_log_xpath} //project-detail//li[contains(.,'Logs')] -${project_member_xpath} //project-detail//li[contains(.,'Members')] +${project_log_xpath} //project-detail//a[contains(.,'Logs')] +${project_member_xpath} //project-detail//a[contains(.,'Members')] ${project_config_tabsheet} xpath=//project-detail//a[contains(.,'Configuration')] -${project_tag_strategy_xpath} //clr-tabs//a[contains(.,'Tag')] +${project_tag_strategy_xpath} //project-detail//a[contains(.,'Tag Retention')] ${project_tab_overflow_btn} //clr-tabs//li//button[contains(@class,"dropdown-toggle")] -${project_tag_immutability_switch} //project-detail/app-tag-feature-integration//label/a[contains(.,'Tag Immutability')] +${project_tag_immutability_switch} //project-detail//a[contains(.,'Tag Immutability')] ${create_project_CANCEL_button_xpath} xpath=//button[contains(.,'CANCEL')] ${create_project_OK_button_xpath} xpath=//button[contains(.,'OK')] diff --git a/tests/resources/Harbor-Pages/Replication.robot b/tests/resources/Harbor-Pages/Replication.robot index 9efd93fa9..69244a1db 100644 --- a/tests/resources/Harbor-Pages/Replication.robot +++ b/tests/resources/Harbor-Pages/Replication.robot @@ -185,11 +185,14 @@ Find Item And Click Delete Button Retry Select Object ${name} Retry Element Click ${action_bar_delete} -Edit Replication Rule By Name +Switch To Replication Manage Page [Arguments] ${name} Switch To Registries Switch To Replication Manage - Find Item And Click Edit Button ${name} + +Edit Replication Rule By Name + [Arguments] ${name} + Retry Double Keywords When Error Switch To Replication Manage Page "NULL" Find Item And Click Edit Button ${name} Delete Replication Rule By Name [Arguments] ${name} @@ -226,6 +229,11 @@ Select Rule And Replicate Retry Element Click ${replication_exec_id} Retry Double Keywords When Error Retry Element Click xpath=${dialog_replicate} Retry Wait Until Page Not Contains Element xpath=${dialog_replicate} +Select Rule And Click Edit Button + [Arguments] ${rule_name} + Retry Element Click //clr-dg-row[contains(.,'${rule_name}')]//clr-radio-wrapper/label + Retry Element Click ${edit_replication_rule_id} + Delete Replication Rule [Arguments] ${name} Retry Element Click ${endpoint_filter_search} @@ -234,4 +242,18 @@ Delete Replication Rule Retry Element Click //clr-dg-row[contains(.,'${name}')]//label Retry Element Click ${action_bar_delete} Wait Until Page Contains Element ${dialog_delete} - Retry Element Click ${dialog_delete} \ No newline at end of file + Retry Element Click ${dialog_delete} + +Image Should Be Replicated To Project + [Arguments] ${project} ${image} ${period}=60 ${times}=10 + :For ${n} IN RANGE 1 ${times} + \ Sleep ${period} + \ Go Into Project ${project} + \ Switch To Project Repo + \ #In AWS-ECR, under repository a, there're only several images: httpd,alpine,hello-world. + \ ${out} Run Keyword And Ignore Error Retry Wait Until Page Contains ${project}/${image} + \ Log To Console Return value is ${out[0]} + \ Exit For Loop If '${out[0]}'=='PASS' + \ Sleep 5 + Run Keyword If '${out[0]}'=='FAIL' Capture Page Screenshot + Should Be Equal As Strings '${out[0]}' 'PASS' \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/ToolKit.robot b/tests/resources/Harbor-Pages/ToolKit.robot index 954251473..92ea04950 100644 --- a/tests/resources/Harbor-Pages/ToolKit.robot +++ b/tests/resources/Harbor-Pages/ToolKit.robot @@ -39,9 +39,17 @@ Filter Object Retry Element Click xpath=//hbr-filter//clr-icon ${element}= Set Variable xpath=//hbr-filter//input Wait Until Element Is Visible And Enabled ${element} - Input Text ${element} ${kw} + Retry Clear Element Text ${element} + Retry Text Input ${element} ${kw} Sleep 3 +Filter Project +#Filter project repo user tag. + [Arguments] ${kw} + Retry Element Click ${log_xpath} + Retry Element Click ${projects_xpath} + Filter Object ${kw} + Select Object #select single element such as user project repo tag [Arguments] ${obj} diff --git a/tests/resources/Harbor-Pages/UserProfile.robot b/tests/resources/Harbor-Pages/UserProfile.robot index fc7b3fca0..8525fcca7 100644 --- a/tests/resources/Harbor-Pages/UserProfile.robot +++ b/tests/resources/Harbor-Pages/UserProfile.robot @@ -43,4 +43,4 @@ Logout Harbor Retry Link Click Log Out Capture Page Screenshot Logout.png Sleep 2 - Wait Until Keyword Succeeds 5x 1 Page Should Contain Element ${sign_in_title_xpath} \ No newline at end of file + Wait Until Keyword Succeeds 5x 1 Retry Wait Until Page Contains Element ${sign_in_title_xpath} \ No newline at end of file diff --git a/tests/resources/Harbor-Pages/Verify.robot b/tests/resources/Harbor-Pages/Verify.robot index 00616e84a..f113f3ca7 100644 --- a/tests/resources/Harbor-Pages/Verify.robot +++ b/tests/resources/Harbor-Pages/Verify.robot @@ -26,7 +26,7 @@ Verify Project Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} :FOR ${project} IN @{project} - \ Page Should Contain ${project} + \ Retry Wait Until Page Contains ${project} Verify Project Metadata ${json} Close Browser @@ -56,10 +56,10 @@ Verify Project Metadata \ Switch To Project Configuration \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.public ${project_config_public_checkbox} \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.enable_content_trust ${project_config_content_trust_checkbox} - \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.automatically_scan_images_on_push ${project_config_scan_images_on_push_checkbox} - \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.prevent_vulnerable_images_from_running ${project_config_prevent_vulnerable_images_from_running_checkbox} + \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.auto_scan ${project_config_scan_images_on_push_checkbox} + \ Verify Checkbox ${json} $.projects[?(@.name=${project})].configuration.prevent_vul ${project_config_prevent_vulnerable_images_from_running_checkbox} \ ${ret} Get Selected List Value ${project_config_severity_select} - \ @{severity}= Get Value From Json ${json} $.projects[?(@.name=${project})].configuration.prevent_vlunerable_images_from_running_severity + \ @{severity}= Get Value From Json ${json} $.projects[?(@.name=${project})].configuration.severity \ Should Contain ${ret} @{severity}[0] \ Navigate To Projects Close Browser @@ -145,12 +145,40 @@ Verify Endpoint Verify Replicationrule [Arguments] ${json} - @{replicationrule}= Get Value From Json ${json} $.replicationrule..name - Init Chrome Driver - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Replication Manage - :For ${replicationrule} IN @{replicationrule} - \ Page Should Contain ${replicationrule} + @{replicationrules}= Get Value From Json ${json} $.replicationrule.[*].rulename + @{endpoints}= Get Value From Json ${json} $.endpoint.[*].name + : FOR ${replicationrule} IN @{replicationrules} + \ Init Chrome Driver + \ Log To Console -----replicationrule-----"${replicationrule}"------------ + \ Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + \ Switch To Replication Manage + \ Select Rule And Click Edit Button ${replicationrule} + \ @{is_src_registry}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].is_src_registry + \ @{trigger_type}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].trigger_type + \ @{name_filters}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].name_filters + \ @{tag_filters}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].tag_filters + \ @{dest_namespace}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].dest_namespace + \ @{cron}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].cron + \ @{is_src_registry}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].is_src_registry + \ Log To Console -----is_src_registry-----@{is_src_registry}[0]------------ + \ @{endpoint}= Get Value From Json ${json} $.replicationrule[?(@.rulename=${replicationrule})].endpoint + \ Log To Console -----endpoint-----@{endpoint}------------ + \ ${endpoint0}= Set Variable @{endpoint}[0] + \ Log To Console -----endpoint0-----${endpoint0}------------ + \ @{endpoint_type}= Get Value From Json ${json} $.endpoint[?(@.name=${endpoint0})].type + \ Retry Textfield Value Should Be ${source_project} @{name_filters}[0] + \ Retry Textfield Value Should Be ${filter_tag} @{tag_filters}[0] + \ Retry Textfield Value Should Be ${rule_name_input} ${replicationrule} + \ Retry Textfield Value Should Be ${dest_namespace_xpath} @{dest_namespace}[0] + \ Log To Console -----endpoint_type-----@{endpoint_type}[0]------------ + \ ${registry}= Set Variable If "@{endpoint_type}[0]"=="harbor" ${endpoint0}-https://${IP} ${endpoint0}-https://hub.docker.com + \ Log To Console -------registry---${registry}------------ + \ Run Keyword If '@{is_src_registry}[0]' == '${true}' Retry List Selection Should Be ${src_registry_dropdown_list} ${registry} + \ ... ELSE Retry List Selection Should Be ${dest_registry_dropdown_list} ${registry} + \ #\ Retry List Selection Should Be ${rule_resource_selector} ${resource_type} + \ Retry List Selection Should Be ${rule_trigger_select} @{trigger_type}[0] + \ Run Keyword If '@{trigger_type}[0]' == 'scheduled' Log To Console ----------@{trigger_type}[0]------------ + \ Run Keyword If '@{trigger_type}[0]' == 'scheduled' Retry Textfield Value Should Be ${targetCron_id} @{cron}[0] Close Browser Verify Project Setting diff --git a/tests/resources/Helm-Util.robot b/tests/resources/Helm-Util.robot index 06be47dbf..7f761c0d6 100644 --- a/tests/resources/Helm-Util.robot +++ b/tests/resources/Helm-Util.robot @@ -28,7 +28,7 @@ Helm Repo Add Wait Unitl Command Success helm repo add --ca-file /helm_ca/server.crt --username=${user} --password=${pwd} ${helm_repo_name} ${harbor_url}/chartrepo/${project_name} Helm Repo Push - [Arguments] ${harbor_url} ${user} ${pwd} ${chart_filename} ${project_name}=library ${helm_repo_name}=myrepo + [Arguments] ${user} ${pwd} ${chart_filename} ${helm_repo_name}=myrepo ${current_dir}= Run pwd Run cd ${current_dir} Run wget ${harbor_chart_file_url} diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index d35f29052..2e9f1ff98 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -150,7 +150,18 @@ Helm CLI Push Without Sign In Harbor ${d}= Get Current Date result_format=%m%s Create An New Project project${d} Helm Repo Add ${HARBOR_URL} ${sign_in_user} ${sign_in_pwd} project_name=project${d} - Helm Repo Push ${HARBOR_URL} ${sign_in_user} ${sign_in_pwd} ${harbor_chart_filename} project_name=project${d} + Helm Repo Push ${sign_in_user} ${sign_in_pwd} ${harbor_chart_filename} + Go Into Project project${d} has_image=${false} + Switch To Project Charts + Go Into Chart Version ${harbor_chart_name} + Retry Wait Until Page Contains ${harbor_chart_version} + Capture Page Screenshot + +Helm3 CLI Push Without Sign In Harbor + [Arguments] ${sign_in_user} ${sign_in_pwd} + ${d}= Get Current Date result_format=%m%s + Create An New Project project${d} + Helm Repo Push ${sign_in_user} ${sign_in_pwd} ${harbor_chart_filename} helm_repo_name=${HARBOR_URL}/chartrepo/project${d} Go Into Project project${d} has_image=${false} Switch To Project Charts Go Into Chart Version ${harbor_chart_name} diff --git a/tests/resources/Util.robot b/tests/resources/Util.robot index 89df63036..d357fd784 100644 --- a/tests/resources/Util.robot +++ b/tests/resources/Util.robot @@ -68,7 +68,6 @@ Resource Harbor-Pages/OIDC_Auth_Elements.robot Resource Harbor-Pages/Verify.robot Resource Docker-Util.robot Resource Helm-Util.robot -Resource Admiral-Util.robot Resource OVA-Util.robot Resource Cert-Util.robot Resource SeleniumUtil.robot @@ -205,7 +204,7 @@ Clear Field Of Characters [Arguments] ${field} ${character count} [Documentation] This keyword pushes the delete key (ascii: \8) a specified number of times in a specified field. : FOR ${index} IN RANGE ${character count} - \ Press Key ${field} \\8 + \ Press Keys ${field} \\8 Wait Unitl Command Success [Arguments] ${cmd} ${times}=8 @@ -227,13 +226,14 @@ Command Should be Failed Retry Keyword When Error [Arguments] ${keyword} @{elements} :For ${n} IN RANGE 1 6 - \ Log To Console Trying ${keyword} ${n} times ... + \ Log To Console Trying ${keyword} elements @{elements} ${n} times ... \ ${out} Run Keyword And Ignore Error ${keyword} @{elements} \ Log To Console Return value is ${out[0]} \ Exit For Loop If '${out[0]}'=='PASS' \ Sleep 2 Run Keyword If '${out[0]}'=='FAIL' Capture Page Screenshot Should Be Equal As Strings '${out[0]}' 'PASS' + [Return] ${out[1]} Retry Keyword When Return Value Mismatch [Arguments] ${keyword} ${expected_value} ${count} @{elements} @@ -248,8 +248,8 @@ Retry Keyword When Return Value Mismatch Should Be Equal As Strings ${status} 'PASS' Retry Double Keywords When Error - [Arguments] ${keyword1} ${element1} ${keyword2} ${element2} - :For ${n} IN RANGE 1 3 + [Arguments] ${keyword1} ${element1} ${keyword2} ${element2} ${DoAssert}=${true} + :For ${n} IN RANGE 1 5 \ Log To Console Trying ${keyword1} and ${keyword2} ${n} times ... \ ${out1} Run Keyword And Ignore Error ${keyword1} ${element1} \ Capture Page Screenshot @@ -259,6 +259,7 @@ Retry Double Keywords When Error \ Log To Console Return value is ${out1[0]} ${out2[0]} \ Exit For Loop If '${out2[0]}'=='PASS' \ Sleep 1 + Return From Keyword If ${DoAssert} == ${false} '${out2[0]}' Should Be Equal As Strings '${out2[0]}' 'PASS' Run Curl And Return Json diff --git a/tests/robot-cases/Group0-BAT/BAT.robot b/tests/robot-cases/Group0-BAT/BAT.robot index 473d74c65..2a4ecf838 100644 --- a/tests/robot-cases/Group0-BAT/BAT.robot +++ b/tests/robot-cases/Group0-BAT/BAT.robot @@ -26,13 +26,13 @@ ${HARBOR_URL} https://${ip} Test Case - Registry Basic Verfication Init Chrome Driver ${d}= Get Current Date result_format=%m%s - + Create An New Project With New User url=${HARBOR_URL} username=tester${d} email=tester${d}@vmware.com realname=tester${d} newPassword=Test1@34 comment=harbor projectname=project${d} public=true Push image ${ip} tester${d} Test1@34 project${d} busybox:latest - Pull image ${ip} tester${d} Test1@34 project${d} busybox:latest + Pull image ${ip} tester${d} Test1@34 project${d} busybox:latest Go Into Project project${d} Delete Repo project${d} - + Close Browser Test Case - Clair Basic Verfication @@ -52,7 +52,7 @@ Test Case - Notary Basic Verfication ${rc} ${output}= Run And Return Rc And Output docker pull hello-world:latest Log ${output} - + Push image ${ip} %{HARBOR_ADMIN} %{HARBOR_PASSWORD} library hello-world:latest ${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/notary-push-image.sh ${ip} ${notaryServerEndpoint} Log ${output} diff --git a/tests/robot-cases/Group0-Util/notary-push-image.sh b/tests/robot-cases/Group0-Util/notary-push-image.sh index 6cdf99c90..f3a9366bf 100755 --- a/tests/robot-cases/Group0-Util/notary-push-image.sh +++ b/tests/robot-cases/Group0-Util/notary-push-image.sh @@ -11,11 +11,14 @@ echo $IP mkdir -p /etc/docker/certs.d/$IP/ mkdir -p ~/.docker/tls/$IP:4443/ -cp /notary_ca/ca.crt /etc/docker/certs.d/$IP/ -cp /notary_ca/ca.crt ~/.docker/tls/$IP:4443/ +cp /notary_ca.crt /etc/docker/certs.d/$IP/ +cp /notary_ca.crt ~/.docker/tls/$IP:4443/ + +mkdir -p ~/.docker/tls/$notaryServerEndpoint/ +cp /notary_ca.crt ~/.docker/tls/$notaryServerEndpoint/ export DOCKER_CONTENT_TRUST=1 -export DOCKER_CONTENT_TRUST_SERVER=https://$IP:4443 +export DOCKER_CONTENT_TRUST_SERVER=https://$notaryServerEndpoint export NOTARY_ROOT_PASSPHRASE=$PASSHRASE export NOTARY_TARGETS_PASSPHRASE=$PASSHRASE diff --git a/tests/robot-cases/Group1-Nightly/Chartmuseum.robot b/tests/robot-cases/Group1-Nightly/Chartmuseum.robot index 82701d0c2..0b760a63d 100644 --- a/tests/robot-cases/Group1-Nightly/Chartmuseum.robot +++ b/tests/robot-cases/Group1-Nightly/Chartmuseum.robot @@ -32,4 +32,11 @@ Test Case - Helm CLI Push ${user}= Set Variable user004 ${pwd}= Set Variable Test1@34 Sign In Harbor ${HARBOR_URL} ${user} ${pwd} - Helm CLI Push Without Sign In Harbor ${user} ${pwd} \ No newline at end of file + Helm CLI Push Without Sign In Harbor ${user} ${pwd} + +Test Case - Helm3 CLI Push + Init Chrome Driver + ${user}= Set Variable user004 + ${pwd}= Set Variable Test1@34 + Sign In Harbor ${HARBOR_URL} ${user} ${pwd} + Helm3 CLI Push Without Sign In Harbor ${user} ${pwd} \ No newline at end of file diff --git a/tests/robot-cases/Group1-Nightly/Common.robot b/tests/robot-cases/Group1-Nightly/Common.robot index 24a7aed21..6acfb8d3f 100644 --- a/tests/robot-cases/Group1-Nightly/Common.robot +++ b/tests/robot-cases/Group1-Nightly/Common.robot @@ -117,8 +117,7 @@ Test Case - Project Level Policy Public # Here logout and login to try avoid a bug only in autotest Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Filter Object project${d} - Project Should Be Public project${d} + Retry Double Keywords When Error Filter Project project${d} Project Should Be Public project${d} Close Browser Test Case - Verify Download Ca Link @@ -406,7 +405,7 @@ Test Case - Developer Operate Labels Sign In Harbor ${HARBOR_URL} user022 Test1@34 Go Into Project project${d} has_image=${false} Sleep 3 - Page Should Not Contain Element xpath=//a[contains(.,'Labels')] + Retry Wait Until Page Not Contains Element xpath=//a[contains(.,'Labels')] Close Browser Test Case - Retag A Image Tag @@ -431,7 +430,7 @@ Test Case - Retag A Image Tag Page Should Contain ${target_image_name} Go Into Repo project${random_num1}${random_num2}/${target_image_name} Sleep 1 - Page Should Contain Element xpath=${tag_value_xpath} + Retry Wait Until Page Contains Element xpath=${tag_value_xpath} Close Browser Test Case - Create An New Project With Quotas Set @@ -550,8 +549,11 @@ Test Case - Project Quotas Control Under GC ${image_a_size}= Set Variable 321.03MB ${image_a_ver}= Set Variable 6.8.3 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Capture Page Screenshot Create An New Project project${d} storage_quota=${storage_quota} storage_quota_unit=${storage_quota_unit} - Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_a}:${image_a_ver} err_msg=Quota exceeded when processing the request of adding 82.5 MiB of storage resource, which when updated to current usage of 166.6 MiB will exceed the configured upper limit of 200.0 MiB + Capture Page Screenshot + Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image_a}:${image_a_ver} err_msg=will exceed the configured upper limit of 200.0 MiB + Capture Page Screenshot GC Now ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @{param} Create List project${d} Retry Keyword When Return Value Mismatch Get Project Storage Quota Text From Project Quotas List 0Byte of ${storage_quota}${storage_quota_unit} 60 @{param} @@ -574,7 +576,7 @@ Test Case - Update Webhook Create An New Project project${d} Go Into Project project${d} has_image=${false} Switch To Project Webhooks - Create A New Webhook ${HARBOR_URL} auth_header=auth_header${d} + Create A New Webhook ${HARBOR_URL} auth_header=auth_header${d} Sleep 3 ${d1}= Get Current Date Update A Webhook 101.17.109.20 auth_header=auth_header${d1} diff --git a/tests/robot-cases/Group1-Nightly/LDAP.robot b/tests/robot-cases/Group1-Nightly/LDAP.robot index c077fca48..307565a6d 100644 --- a/tests/robot-cases/Group1-Nightly/LDAP.robot +++ b/tests/robot-cases/Group1-Nightly/LDAP.robot @@ -44,6 +44,7 @@ Test Case - System Admin On-board New Member ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To User Tag + Sleep 2 Page Should Not Contain mike02 Navigate To Projects Create An New Project project${d} @@ -60,8 +61,10 @@ Test Case - LDAP User On-borad New Member Create An New Project project${d} Go Into Project project${d} has_image=${false} Switch To Member + Sleep 2 Page Should Not Contain mike04 Add Guest Member To Project mike04 + Sleep 2 Page Should Contain mike04 Close Browser diff --git a/tests/robot-cases/Group1-Nightly/Nightly.robot b/tests/robot-cases/Group1-Nightly/Nightly.robot index 8648101a7..a4bb5ca3b 100644 --- a/tests/robot-cases/Group1-Nightly/Nightly.robot +++ b/tests/robot-cases/Group1-Nightly/Nightly.robot @@ -393,7 +393,7 @@ TestCase - Developer Operate Labels Sign In Harbor ${HARBOR_URL} bob${d} Test1@34 Go Into Project project${d} has_image=${false} Sleep 3 - Page Should Not Contain Element xpath=//a[contains(.,'Labels')] + Retry Wait Until Page Not Contains Element xpath=//a[contains(.,'Labels')] Close Browser Test Case - Scan A Tag In The Repo diff --git a/tests/robot-cases/Group1-Nightly/Notary.robot b/tests/robot-cases/Group1-Nightly/Notary.robot index f0a16fa1c..ab210454f 100644 --- a/tests/robot-cases/Group1-Nightly/Notary.robot +++ b/tests/robot-cases/Group1-Nightly/Notary.robot @@ -40,7 +40,7 @@ Test Case - Project Level Policy Content Trust Close Browser Test Case - Admin Push Signed Image - Body Of Admin Push Signed Image tomcat + Body Of Admin Push Signed Image Test Case - Admin Push Signed Image And Remove Signature Body Of Admin Push Signed Image image=mariadb with_remove=${true} diff --git a/tests/robot-cases/Group1-Nightly/Replication.robot b/tests/robot-cases/Group1-Nightly/Replication.robot index d6d75bf5d..d00aa4751 100644 --- a/tests/robot-cases/Group1-Nightly/Replication.robot +++ b/tests/robot-cases/Group1-Nightly/Replication.robot @@ -37,7 +37,7 @@ Test Case - Get Harbor Version Test Case - Pro Replication Rules Add Init Chrome Driver - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Switch To Replication Manage Check New Rule UI Without Endpoint Close Browser @@ -156,21 +156,16 @@ Test Case - Replication Of Pull Images from DockerHub To Self Switch To Replication Manage Create A Rule With Existing Endpoint rule${d} pull danfengliu/* image e${d} project${d} Select Rule And Replicate rule${d} - Sleep 30 - Go Into Project project${d} - Switch To Project Repo #In docker-hub, under repository danfengliu, there're only 2 images: centos,mariadb. - Retry Wait Until Page Contains project${d}/centos - Go Into Project project${d} - Switch To Project Repo - Retry Wait Until Page Contains project${d}/mariadb + Image Should Be Replicated To Project project${d} centos + Image Should Be Replicated To Project project${d} mariadb Close Browser Test Case - Replication Of Push Images from Self To Harbor Init Chrome Driver ${d}= Get Current Date result_format=%m%s #login source - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Create An New Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest @@ -181,27 +176,25 @@ Test Case - Replication Of Push Images from Self To Harbor Create A Rule With Existing Endpoint rule${d} push project${d}/* image e${d} project_dest${d} #logout and login target Logout Harbor - Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Create An New Project project_dest${d} #logout and login source Logout Harbor - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Switch To Replication Manage Select Rule And Replicate rule${d} - Sleep 20 + Sleep 20 Logout Harbor - Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project project_dest${d} - Switch To Project Repo - Retry Wait Until Page Contains project_dest${d}/hello-world - Retry Wait Until Page Contains project_dest${d}/busybox + Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} + Image Should Be Replicated To Project project_dest${d} hello-world + Image Should Be Replicated To Project project_dest${d} busybox Close Browser Test Case - Replication Of Push Chart from Self To Harbor Init Chrome Driver ${d}= Get Current Date result_format=%m%s #login source - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Create An New Project project${d} Go Into Project project${d} has_image=${false} Switch To Project Charts @@ -212,16 +205,16 @@ Test Case - Replication Of Push Chart from Self To Harbor Create A Rule With Existing Endpoint rule${d} push project${d}/* chart e${d} project_dest${d} #logout and login target Logout Harbor - Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Create An New Project project_dest${d} #logout and login source Logout Harbor - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Switch To Replication Manage Select Rule And Replicate rule${d} Sleep 20 Logout Harbor - Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Go Into Project project_dest${d} has_image=${false} Switch To Project Charts Go Into Chart Version ${harbor_chart_name} @@ -233,7 +226,7 @@ Test Case - Replication Of Push Images from Self To Harbor By Push Event Init Chrome Driver ${d}= Get Current Date result_format=%m%s #login source - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Create An New Project project${d} Switch To Registries Create A New Endpoint harbor e${d} https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} @@ -242,13 +235,10 @@ Test Case - Replication Of Push Images from Self To Harbor By Push Event ... Event Based #logout and login target Logout Harbor - Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Sign In Harbor https://${ip1} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} is_close_scan_plugin_mesg=${true} Create An New Project project_dest${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} centos - Sleep 10 - Go Into Project project_dest${d} - Switch To Project Repo - Retry Wait Until Page Contains project_dest${d}/centos + Image Should Be Replicated To Project project_dest${d} centos Close Browser Test Case - Replication Of Pull Images from AWS-ECR To Self @@ -262,15 +252,9 @@ Test Case - Replication Of Pull Images from AWS-ECR To Self Switch To Replication Manage Create A Rule With Existing Endpoint rule${d} pull a/* image e${d} project${d} Select Rule And Replicate rule${d} - Sleep 60 - Go Into Project project${d} - Switch To Project Repo - #In AWS-ECR, under repository a, there're only several images: httpd,alpine,hello-world. - Retry Wait Until Page Contains project${d}/httpd - Go Into Project project${d} - Switch To Project Repo - Retry Wait Until Page Contains project${d}/alpine - Retry Wait Until Page Contains project${d}/hello-world + Image Should Be Replicated To Project project${d} httpd + Image Should Be Replicated To Project project${d} alpine + Image Should Be Replicated To Project project${d} hello-world Close Browser Test Case - Replication Of Pull Images from Google-GCR To Self @@ -285,12 +269,6 @@ Test Case - Replication Of Pull Images from Google-GCR To Self Create A Rule With Existing Endpoint rule${d} pull eminent-nation-87317/* image e${d} project${d} Filter Replicatin Rule rule${d} Select Rule And Replicate rule${d} - Sleep 30 - Go Into Project project${d} - Switch To Project Repo - #In Google-GCR, under repository a, there're only several images: httpd,tomcat. - Retry Wait Until Page Contains project${d}/httpd - Go Into Project project${d} - Switch To Project Repo - Retry Wait Until Page Contains project${d}/tomcat + Image Should Be Replicated To Project project${d} httpd + Image Should Be Replicated To Project project${d} tomcat Close Browser diff --git a/tests/robot-cases/Group1-Nightly/UAA.robot b/tests/robot-cases/Group1-Nightly/UAA.robot index 1d5ba63e3..11c38d901 100644 --- a/tests/robot-cases/Group1-Nightly/UAA.robot +++ b/tests/robot-cases/Group1-Nightly/UAA.robot @@ -36,7 +36,7 @@ Test Case - Home Page Differences With DB Mode Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Logout Harbor Sleep 2 - Page Should Not Contain Sign up + Page Should Not Contain Sign up Page Should Not Contain Forgot password Close Browser diff --git a/tests/robot-cases/Group3-Upgrade/data.json b/tests/robot-cases/Group3-Upgrade/data.json index 07317ee3a..42caffd73 100644 --- a/tests/robot-cases/Group3-Upgrade/data.json +++ b/tests/robot-cases/Group3-Upgrade/data.json @@ -31,24 +31,53 @@ "url":"http://url", "name":"endpoint1", "user":"admin", - "pass":"Harbor12345" - } + "pass":"Harbor12345", + "insecure":true, + "type":"harbor" + }, + { + "url":"https://hub.docker.com", + "name":"endpoint2", + "user":"", + "pass":"", + "insecure":false, + "type":"docker-hub" + } ], "replicationrule":[ { "project":"project1", "endpoint":"endpoint1", "trigger":"Manual", - "rulename":"rulename" + "rulename":"rulename", + "is_src_registry":false, + "dest_namespace":"rule1-namespace", + "trigger_type":"event_based", + "cron":"", + "deletion":true, + "enabled":true, + "override":true, + "name_filters":"namefilter1", + "tag_filters":"tagfilter1" }, { "project":"library", - "endpoint":"endpoint1", + "endpoint":"endpoint2", "trigger":"Manual", - "rulename":"rulelibrary" - } - ], - "configuration":{ + "rulename":"rulelibrary", + "endpoint_type":"harbor", + "is_src_registry":true, + "dest_namespace":"rule2-namespace", + "trigger_type":"scheduled", + "cron":"6 7 8 * * *", + "deletion":false, + "enabled":true, + "override":true, + "name_filters":"namefilter2", + "tag_filters":"tagfilter2" + } + ], + "configuration":{ "authmode":"Database", "projectcreation":"everyone", "selfreg":false, @@ -68,6 +97,25 @@ "daily":0 } }, + "deployment_security":{ + "cve":[ + { + "id":"CVE-2019-12904" + }, + { + "id":"CVE-2011-3389" + }, + { + "id":"CVE-2018-12886" + }, + { + "id":"CVE-2019-3844" + } + ], + "never_expires":false, + "expires_at":"2572969600" + }, + "webhooks_enabled":true, "syslabel":[ { "name":"label1" @@ -121,6 +169,10 @@ "role":3 } ], + "count_limit":1234, + "storage_limit":53687091200, + "storage_limit_for_verify":50, + "storage_unit_for_verify":"GB", "replications":{ "rulename":"ruleproject1", "endpointname":"endpoint1", @@ -143,12 +195,50 @@ "operation":"create" } ], + "robot_account":[ + { + "name":"robot0", + "access": [ + { + "action": "pull" + }, + { + "action": "push" + } + ] + } + ], + "webhook":{ + "address":"https://1.1.1.1", + "skip_cert_verify":true, + "auth_header":"aaa", + "enabled":true + }, "configuration":{ "public":"true", "enable_content_trust":"true", - "automatically_scan_images_on_push":"true", - "prevent_vulnerable_images_from_running":"true", - "prevent_vlunerable_images_from_running_severity":"high" + "auto_scan":"true", + "prevent_vul":"true", + "severity":"high", + "reuse_sys_cve_whitelist":"false", + "deployment_security":{ + "cve":[ + { + "id":"CVE-2019-111" + }, + { + "id":"CVE-2011-2222" + }, + { + "id":"CVE-2018-33333" + }, + { + "id":"CVE-2019-44444" + } + ], + "never_expires":false, + "expires_at":"2572969600" + } } }, { @@ -182,6 +272,10 @@ "role":3 } ], + "count_limit":-1, + "storage_limit":32985348833280, + "storage_limit_for_verify":30, + "storage_unit_for_verify":"TB", "replications":{ "rulename":"rulename1", "endpointname":"endpoint1", @@ -204,12 +298,52 @@ "operation":"create" } ], + "robot_account":[ + { + "name":"robot1", + "access": [ + { + "action": "pull" + }, + { + "action": "push" + } + ] + }, + { + "name":"robot2", + "access": [ + { + "action": "pull" + } + ] + }, + { + "name":"robot3", + "access": [ + { + "action": "push" + } + ] + } + ], + "webhook":{ + "address":"https://1.1.1.1", + "skip_cert_verify":true, + "auth_header":"aaa", + "enabled":true + }, "configuration":{ "public":"false", "enable_content_trust":"false", - "automatically_scan_images_on_push":"false", - "prevent_vulnerable_images_from_running":"true", - "prevent_vlunerable_images_from_running_severity":"medium" + "auto_scan":"false", + "prevent_vul":"true", + "severity":"medium", + "reuse_sys_cve_whitelist":"true", + "deployment_security":{ + "cve":[ + ] + } } } ] diff --git a/tests/robot-cases/Group3-Upgrade/feature_map.json b/tests/robot-cases/Group3-Upgrade/feature_map.json new file mode 100644 index 000000000..6e891c9ea --- /dev/null +++ b/tests/robot-cases/Group3-Upgrade/feature_map.json @@ -0,0 +1,192 @@ +{ + "create_project":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":1, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + } + ], + "add_member":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":2, + "version":"1.6" + }, + { + "branch":2, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + } + ], + "set_user_admin":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":2, + "version":"1.6" + }, + { + "branch":2, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + } + ], + "add_endpoint":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + } + ], + "add_replication_rule":[ + { + "branch":1, + "version":"1.4" + }, + { + "branch":1, + "version":"1.6" + }, + { + "branch":1, + "version":"1.7" + }, + { + "branch":2, + "version":"1.8" + }, + { + "branch":2, + "version":"1.9" + }, + { + "branch":2, + "version":"1.10" + } + ], + "add_sys_whitelist":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + } + ], + "update_project_setting_whitelist":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + } + ], + "add_project_robot_account":[ + { + "branch":1, + "version":"1.8" + }, + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + } + ], + "add_tag_retention_rule":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + } + ], + "add_tag_immutability_rule":[ + { + "branch":1, + "version":"1.10" + } + ], + "add_webhook":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + } + ] + } \ No newline at end of file diff --git a/tests/robot-cases/Group3-Upgrade/prepare.py b/tests/robot-cases/Group3-Upgrade/prepare.py index 852e1cb4d..1c63a1d91 100644 --- a/tests/robot-cases/Group3-Upgrade/prepare.py +++ b/tests/robot-cases/Group3-Upgrade/prepare.py @@ -1,8 +1,9 @@ import os +import sys import json import argparse import requests - +from functools import wraps from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -15,9 +16,43 @@ url = "https://"+args.endpoint+"/api/" endpoint_url = "https://"+args.endpoint print url +with open("feature_map.json") as f: + feature_map = json.load(f) + +def get_branch(func_name, version): + has_feature = False + for node in feature_map[func_name]: + has_feature = True + if node["version"] == version: + return node["branch"] + if has_feature is False: + return "No Restriction" + else: + return "Not Supported" + +def get_feature_branch(func): + @wraps(func) + def inner_func(*args,**kwargs): + branch=get_branch(inner_func.__name__, kwargs.get("version")) + if branch == "No Restriction": + func(*args,**kwargs) + elif branch == "Not Supported": + print("Feature {} is not supported in version {}".format(inner_func.__name__, kwargs.get("version"))) + else: + kwargs["branch"] = branch + func(*args,**kwargs) + return + return inner_func + class HarborAPI: - def create_project(self, project_name): - body=dict(body={"project_name": ""+project_name+"", "metadata": {"public": "true"}}) + @get_feature_branch + def create_project(self, project, **kwargs): + if kwargs["branch"] == 1: + body=dict(body={"project_name": ""+project["name"]+"", "metadata": {"public": "true"}}) + elif kwargs["branch"] == 2: + body=dict(body={"project_name": ""+project["name"]+"", "metadata": {"public": "true"},"count_limit":project["count_limit"],"storage_limit":project["storage_limit"]}) + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) request(url+"projects", 'post', **body) def create_user(self, username): @@ -25,65 +60,135 @@ class HarborAPI: body=dict(body=payload) request(url+"users", 'post', **body) - def set_user_admin(self, user): + @get_feature_branch + def set_user_admin(self, user, **kwargs): r = request(url+"users?username="+user+"", 'get') userid = str(r.json()[0]['user_id']) - if args.version == "1.6": + + if kwargs["branch"] == 1: + body=dict(body={"has_admin_role": 1}) + elif kwargs["branch"] == 2: body=dict(body={"has_admin_role": True}) else: - body=dict(body={"has_admin_role": 1}) + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) request(url+"users/"+userid+"/sysadmin", 'put', **body) - def add_member(self, project, user, role): + @get_feature_branch + def add_member(self, project, user, role, **kwargs): r = request(url+"projects?name="+project+"", 'get') projectid = str(r.json()[0]['project_id']) - if args.version == "1.6": + + if kwargs["branch"] == 1: + payload = {"roles": [role], "username":""+user+""} + elif kwargs["branch"] == 2: payload = {"member_user":{ "username": ""+user+""},"role_id": role} else: - payload = {"roles": [role], "username":""+user+""} - + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) body=dict(body=payload) request(url+"projects/"+projectid+"/members", 'post', **body) - def add_endpoint(self, endpointurl, endpointname, username, password, insecure): - payload = { - "credential":{ - "access_key":""+username+"", - "access_secret":""+password+"", - "type":"basic" - }, - "insecure":insecure, - "name":""+endpointname+"", - "type":"harbor", - "url":""+endpoint_url+"" - } - body=dict(body=payload) - print body - request(url+"/registries", 'post', **body) + @get_feature_branch + def add_endpoint(self, endpointurl, endpointname, username, password, insecure, registry_type, **kwargs): + if kwargs["branch"] == 1: + payload = {"endpoint": ""+endpointurl+"", "name": ""+endpointname+"", "username": ""+username+"", "password": ""+password+"", "insecure": insecure} + body=dict(body=payload) + request(url+"targets", 'post', **body) + elif kwargs["branch"] == 2: + if registry_type == "harbor": + endpointurl = endpoint_url + payload = { + "credential":{ + "access_key":""+username+"", + "access_secret":""+password+"", + "type":"basic" + }, + "insecure":insecure, + "name":""+endpointname+"", + "type":""+registry_type+"", + "url":""+endpointurl+"" + } + body=dict(body=payload) + print body + request(url+"/registries", 'post', **body) + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) - def add_replication_rule(self, project, target, trigger, rulename): - r = request(url+"registries?name="+target+"", 'get') - targetid = r.json()[0]['id'] - payload = {"name": ""+rulename+"", "deletion": False, "enabled": True, "description": "string", "dest_registry": {"id": targetid},"trigger": {"type": "manual"}} - body=dict(body=payload) - request(url+"replication/policies", 'post', **body) + @get_feature_branch + def add_replication_rule(self, replicationrule, **kwargs): + if kwargs["branch"] == 1: + r = request(url+"projects?name="+replicationrule["project"]+"", 'get') + projectid = r.json()[0]['project_id'] + r = request(url+"targets?name="+replicationrule["endpoint"]+"", 'get') + targetid = r.json()[0]['id'] + payload = {"name": ""+replicationrule["rulename"]+"", "description": "string", "projects": [{"project_id": projectid,}], "targets": [{"id": targetid,}], "trigger": {"kind": ""+replicationrule["trigger"]+"", "schedule_param": {"type": "weekly", "weekday": 1, "offtime": 0}}} + body=dict(body=payload) + request(url+"policies/replication", 'post', **body) + elif kwargs["branch"] == 2: + r = request(url+"registries?name="+replicationrule["endpoint"]+"", 'get') + targetid = r.json()[0]['id'] + if replicationrule["is_src_registry"] is True: + registry = r'"src_registry": { "id": '+str(targetid)+r'},' + else: + registry = r'"dest_registry": { "id": '+str(targetid)+r'},' - def update_project_setting(self, project, public, contenttrust, preventrunning, preventseverity, scanonpush): + body=dict(body=json.loads(r'{"name":"'+replicationrule["rulename"].encode('utf-8')+r'","dest_namespace":"'+replicationrule["dest_namespace"].encode('utf-8')+r'","deletion": '+str(replicationrule["deletion"]).lower()+r',"enabled": '+str(replicationrule["enabled"]).lower()+r',"override": '+str(replicationrule["override"]).lower()+r',"description": "string",'+ registry + r'"trigger":{"type": "'+replicationrule["trigger_type"]+r'", "trigger_settings":{"cron":"'+replicationrule["cron"]+r'"}},"filters":[ {"type":"name","value":"'+replicationrule["name_filters"]+r'"},{"type":"tag","value":"'+replicationrule["tag_filters"]+r'"}]}')) + print body + request(url+"replication/policies", 'post', **body) + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) + + #@get_feature_branch + def update_project_setting_metadata(self, project, public, contenttrust, preventrunning, preventseverity, scanonpush): r = request(url+"projects?name="+project+"", 'get') projectid = str(r.json()[0]['project_id']) payload = { - "project_name": ""+project+"", "metadata": { "public": public, "enable_content_trust": contenttrust, - "prevent_vulnerable_images_from_running": preventrunning, - "prevent_vulnerable_images_from_running_severity": preventseverity, - "automatically_scan_images_on_push": scanonpush + "prevent_vul": preventrunning, + "severity": preventseverity, + "auto_scan": scanonpush } } body=dict(body=payload) + print body request(url+"projects/"+projectid+"", 'put', **body) + @get_feature_branch + def add_sys_whitelist(self, cve_id_list, **kwargs): + cve_id_str = "" + if kwargs["branch"] == 1: + for index, cve_id in enumerate(cve_id_list["cve"]): + cve_id_str = cve_id_str + '{"cve_id":"' +cve_id["id"] + '"}' + if index != len(cve_id_list["cve"]) - 1: + cve_id_str = cve_id_str + "," + body=dict(body=json.loads(r'{"items":['+cve_id_str.encode('utf-8')+r'],"expires_at":'+cve_id_list["expires_at"]+'}')) + request(url+"system/CVEWhitelist", 'put', **body) + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) + + @get_feature_branch + def update_project_setting_whitelist(self, project, reuse_sys_cve_whitelist, cve_id_list, **kwargs): + r = request(url+"projects?name="+project+"", 'get') + projectid = str(r.json()[0]['project_id']) + cve_id_str = "" + if kwargs["branch"] == 1: + for index, cve_id in enumerate(cve_id_list["cve"]): + cve_id_str = cve_id_str + '{"cve_id":"' +cve_id["id"] + '"}' + if index != len(cve_id_list["cve"]) - 1: + cve_id_str = cve_id_str + "," + print cve_id_str + if reuse_sys_cve_whitelist == "true": + payload = r'{"metadata":{"reuse_sys_cve_whitelist":"true"}}' + else: + payload = r'{"metadata":{"reuse_sys_cve_whitelist":"false"},"cve_whitelist":{"project_id":'+projectid+',"items":['+cve_id_str.encode('utf-8')+r'],"expires_at":'+cve_id_list["expires_at"]+'}}' + print payload + body=dict(body=json.loads(payload)) + request(url+"projects/"+projectid+"", 'put', **body) + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) + + def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token): payload = { "auth_mode": "db_auth", @@ -108,6 +213,78 @@ class HarborAPI: body=dict(body=payload) request(url+"configurations", 'put', **body) + @get_feature_branch + def add_project_robot_account(self, project, robot_account, **kwargs): + r = request(url+"projects?name="+project+"", 'get') + projectid = str(r.json()[0]['project_id']) + + if kwargs["branch"] == 1: + if len(robot_account["access"]) == 1: + robot_account_ac = robot_account["access"][0] + payload = { + "name": robot_account["name"], + "access": [ + { + "resource": "/project/"+projectid+"/repository", + "action": robot_account_ac["action"] + } + ] + } + elif len(robot_account["access"]) == 2: + payload = { + "name": robot_account["name"], + "access": [ + { + "resource": "/project/"+projectid+"/repository", + "action": "pull" + }, + { + "resource": "/project/"+projectid+"/repository", + "action": "push" + } + ] + } + else: + raise Exception(r"Error: Robot account count {} is not legal!".format(len(robot_account["access"]))) + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) + print payload + body=dict(body=payload) + request(url+"projects/"+projectid+"/robots", 'post', **body) + + @get_feature_branch + def add_tag_retention_rule(self, project, robot_account, **kwargs): + return + + @get_feature_branch + def add_webhook(self, webhook, **kwargs): + if kwargs["branch"] == 1: + payload = { + "targets":[ + { + "type":"http", + "address":webhook["address"], + "skip_cert_verify":webhook["skip_cert_verify"], + "auth_header":webhook["auth_header"] + } + ], + "event_types":[ + "downloadChart", + "deleteChart", + "uploadChart", + "deleteImage", + "pullImage", + "pushImage", + "scanningFailed", + "scanningCompleted" + ], + "enabled":+webhook["enabled"] + } + body=dict(body=payload) + request(url+"system/CVEWhitelist", 'put', **body) + else: + raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"])) + def update_repoinfo(self, reponame): payload = {"description": "testdescription"} body=dict(body=payload) @@ -125,6 +302,7 @@ class HarborAPI: try: os.makedirs(ca_path) except Exception, e: + print str(e) pass open(target, 'wb').write(ca_content) @@ -155,7 +333,7 @@ def pull_image(*image): def push_image(image, project): os.system("docker tag "+image+" "+args.endpoint+"/"+project+"/"+image) - os.system("docker login "+args.endpoint+" -u Admin"+" -p Harbor12345") + os.system("docker login "+args.endpoint+" -u admin"+" -p Harbor12345") os.system("docker push "+args.endpoint+"/"+project+"/"+image) def push_signed_image(image, project, tag): @@ -169,30 +347,39 @@ def do_data_creation(): harborAPI.create_user(user["name"]) for user in data["admin"]: - harborAPI.set_user_admin(user["name"]) + harborAPI.set_user_admin(user["name"], version=args.version) for project in data["projects"]: - harborAPI.create_project(project["name"]) + harborAPI.create_project(project, version=args.version) for member in project["member"]: - harborAPI.add_member(project["name"], member["name"], member["role"]) + harborAPI.add_member(project["name"], member["name"], member["role"], version=args.version) + for robot_account in project["robot_account"]: + harborAPI.add_project_robot_account(project["name"], robot_account, version=args.version) + harborAPI.add_webhook(project["webhook"], version=args.version) pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2") push_image("busybox", data["projects"][0]["name"]) push_signed_image("alpine", data["projects"][0]["name"], "latest") for endpoint in data["endpoint"]: - harborAPI.add_endpoint(endpoint["url"], endpoint["name"], endpoint["user"], endpoint["pass"], True) + harborAPI.add_endpoint(endpoint["url"], endpoint["name"], endpoint["user"], endpoint["pass"], endpoint["insecure"], endpoint["type"], version=args.version) + for replicationrule in data["replicationrule"]: - harborAPI.add_replication_rule(replicationrule["project"], - replicationrule["endpoint"], replicationrule["trigger"], - replicationrule["rulename"]) + harborAPI.add_replication_rule(replicationrule, version=args.version) + for project in data["projects"]: - harborAPI.update_project_setting(project["name"], + harborAPI.update_project_setting_metadata(project["name"], project["configuration"]["public"], project["configuration"]["enable_content_trust"], - project["configuration"]["prevent_vulnerable_images_from_running"], - project["configuration"]["prevent_vlunerable_images_from_running_severity"], - project["configuration"]["automatically_scan_images_on_push"]) + project["configuration"]["prevent_vul"], + project["configuration"]["severity"], + project["configuration"]["auto_scan"]) + + for project in data["projects"]: + harborAPI.update_project_setting_whitelist(project["name"], + project["configuration"]["reuse_sys_cve_whitelist"], + project["configuration"]["deployment_security"],version=args.version) + harborAPI.update_systemsetting(data["configuration"]["emailsetting"]["emailfrom"], data["configuration"]["emailsetting"]["emailserver"], float(data["configuration"]["emailsetting"]["emailport"]), @@ -200,4 +387,7 @@ def do_data_creation(): data["configuration"]["projectcreation"], data["configuration"]["selfreg"], float(data["configuration"]["token"])) + + harborAPI.add_sys_whitelist(data["configuration"]["deployment_security"],version=args.version) + do_data_creation() \ No newline at end of file diff --git a/tests/robot-cases/Group3-Upgrade/verify.robot b/tests/robot-cases/Group3-Upgrade/verify.robot index 1a4d21556..3bfb32b5f 100644 --- a/tests/robot-cases/Group3-Upgrade/verify.robot +++ b/tests/robot-cases/Group3-Upgrade/verify.robot @@ -9,13 +9,41 @@ ${HARBOR_ADMIN} admin *** Test Cases *** Test Case - Upgrade Verify + [Tags] 1.8-latest + ${data}= Load Json From File ${CURDIR}${/}data.json + Log To Console "Verify User..." + Run Keyword Verify User ${data} + Log To Console "Verify Project..." + Run Keyword Verify Project ${data} + Log To Console "Verify Member Exist..." + Run Keyword Verify Member Exist ${data} + #Run Keyword Verify Robot Account Exist ${data} + Log To Console "Verify User System Admin Role..." + Run Keyword Verify User System Admin Role ${data} + Log To Console "Verify Endpoint..." + Run Keyword Verify Endpoint ${data} + Log To Console "Verify Replicationrule..." + Run Keyword Verify Replicationrule ${data} + Log To Console "Verify Project Setting..." + Run Keyword Verify Project Setting ${data} + Log To Console "Verify System Setting..." + Run Keyword Verify System Setting ${data} + Log To Console "Verify Image Tag..." + Run Keyword Verify Image Tag ${data} + +Test Case - Upgrade Verify + [Tags] 1.9-latest ${data}= Load Json From File ${CURDIR}${/}data.json Run Keyword Verify User ${data} Run Keyword Verify Project ${data} Run Keyword Verify Member Exist ${data} + #Run Keyword Verify Robot Account Exist ${data} + #Run Keyword Verify Project-level Whitelist ${data} + #Run Keyword Verify Webhook ${data} Run Keyword Verify User System Admin Role ${data} Run Keyword Verify Endpoint ${data} Run Keyword Verify Replicationrule ${data} Run Keyword Verify Project Setting ${data} Run Keyword Verify System Setting ${data} + #Run Keyword Verify System Setting Whitelist ${data} Run Keyword Verify Image Tag ${data} \ No newline at end of file