Merge pull request #10527 from danfengliu/cherry-pick-add-nightly-test-robot-account

[Cherry-Pick 1.10.0] Add nightly test case for robot account

tests/resources/Docker-Util.robot

@@ -23,17 +23,20 @@ Run Docker Info
     Wait Unitl Command Success  docker ${docker-params} info
 
 Pull image
-    [Arguments]  ${ip}  ${user}  ${pwd}  ${project}  ${image}  ${tag}=${null}
+    [Arguments]  ${ip}  ${user}  ${pwd}  ${project}  ${image}  ${tag}=${null}  ${is_robot}=${false}
     Log To Console  \nRunning docker pull ${image}...
     ${image_with_tag}=  Set Variable If  '${tag}'=='${null}'  ${image}  ${image}:${tag}
-    Wait Unitl Command Success  docker login -u ${user} -p ${pwd} ${ip}
+    Run Keyword If  ${is_robot}==${false}  Wait Unitl Command Success  docker login -u ${user} -p ${pwd} ${ip}
-    ${output}=  Wait Unitl Command Success  docker pull ${ip}/${project}/${image_with_tag}
+    ...  ELSE  Wait Unitl Command Success  docker login -u robot\\\$${user} -p ${pwd} ${ip}
+    ${output}=  Docker Pull  ${ip}/${project}/${image_with_tag}
+    Log  ${output}
+    Log To Console  ${output}
     Should Contain  ${output}  Digest:
     Should Contain  ${output}  Status:
     Should Not Contain  ${output}  No such image:
 
 Push image
-    [Arguments]  ${ip}  ${user}  ${pwd}  ${project}  ${image}  ${sha256}=${null}
+    [Arguments]  ${ip}  ${user}  ${pwd}  ${project}  ${image}  ${sha256}=${null}  ${is_robot}=${false}
     ${image_with_sha256}=  Set Variable If  '${sha256}'=='${null}'  ${image}  ${image}@sha256:${sha256}
     ${image_with_tag}=  Set Variable If  '${sha256}'=='${null}'  ${image}  ${image}:${sha256}
     Log To Console  \nRunning docker push ${image}...
@@ -139,7 +142,10 @@ Docker Login
 
 Docker Pull
     [Arguments]  ${image}
-    Wait Unitl Command Success  docker pull ${image}
+    ${output}=  Retry Keyword When Error  Wait Unitl Command Success  docker pull ${image}
+    Log  ${output}
+    Log To Console  Docker Pull: \n ${output}
+    [Return]  ${output}
 
 Docker Tag
     [Arguments]  ${src_image}   ${dst_image}

tests/resources/Harbor-Pages/Administration-Users.robot

@@ -21,28 +21,28 @@ Resource  ../../resources/Util.robot
 *** Keywords ***
 Assign User Admin
     [Arguments]  ${user}
-    Click Element  xpath=//harbor-user//hbr-filter//clr-icon
+    Retry Element Click  xpath=//harbor-user//hbr-filter//clr-icon
     Input Text  xpath=//harbor-user//hbr-filter//input  ${user}
     Sleep  2
     #select checkbox
-    Click Element  //clr-dg-row[contains(.,'${user}')]//label
+    Retry Element Click  //clr-dg-row[contains(.,'${user}')]//label
     #click assign admin
-    Click Element  //*[@id='set-admin']
+    Retry Element Click  //*[@id='set-admin']
     Sleep  1
 
 Switch to User Tag
-    Click Element  xpath=${administration_user_tag_xpath}
+    Retry Element Click  xpath=${administration_user_tag_xpath}
     Sleep  1
 
 Administration Tag Should Display
-    Page Should Contain Element  xpath=${administration_tag_xpath}
+    Retry Wait Until Page Contains Element  xpath=${administration_tag_xpath}
 
 User Email Should Exist
     [Arguments]  ${email}
     Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
     Switch to User Tag
-    Page Should Contain Element  xpath=//clr-dg-cell[contains(., '${email}')]
+    Retry Wait Until Page Contains Element  xpath=//clr-dg-cell[contains(., '${email}')]
 
 Add User Button Should Be Disabled
     Sleep  1
-    Page Should Contain Element  //button[contains(.,'New') and @disabled='']
+    Retry Wait Until Page Contains Element  //button[contains(.,'New') and @disabled='']

tests/resources/Harbor-Pages/Configuration.robot

@@ -140,10 +140,10 @@ Self Reg Should Be Enabled
     Checkbox Should Be Selected  xpath=${self_reg_xpath}
 
 Project Creation Should Display
-    Page Should Contain Element  xpath=${project_create_xpath}
+    Retry Wait Until Page Contains Element  xpath=${project_create_xpath}
 
 Project Creation Should Not Display
-    Page Should Not Contain Element  xpath=${project_create_xpath}
+    Retry Wait Until Page Not Contains Element  xpath=${project_create_xpath}
 
 ## System settings
 Switch To System Settings
@@ -153,6 +153,8 @@ Switch To System Settings
     Sleep  1
 
 Switch To Project Quotas
+    Sleep  1
+    Retry Element Click  xpath=${configuration_xpath}
     Sleep  1
     Retry Element Click  xpath=//clr-main-container//clr-vertical-nav//a[contains(.,'Project Quotas')]
     Sleep  1

tests/resources/Harbor-Pages/HomePage.robot

@@ -20,7 +20,7 @@ Resource  ../../resources/Util.robot
 
 *** Keywords ***
 Sign In Harbor
-    [Arguments]  ${url}  ${user}  ${pw}
+    [Arguments]  ${url}  ${user}  ${pw}  ${is_close_scan_plugin_mesg}=${false}
     Go To    ${url}
     Retry Wait Element  ${harbor_span_title}
     Retry Wait Element  ${login_name}
@@ -31,13 +31,14 @@ Sign In Harbor
     Retry Button Click  ${login_btn}
     Log To Console  ${user}
     Retry Wait Element  xpath=//span[contains(., '${user}')]
+    Run Keyword If  ${is_close_scan_plugin_mesg}==${true}  Run Keyword And Ignore Error  Retry Element Click  ${close_scan_plugin_mesg}
 
 Capture Screenshot And Source
     Capture Page Screenshot
     Log Source
 
 Sign Up Should Not Display
-    Page Should Not Contain Element  xpath=${sign_up_button_xpath}
+    Retry Wait Until Page Not Contains Element  xpath=${sign_up_button_xpath}
 
 Create An New User
     [Arguments]  ${url}  ${username}  ${email}  ${realname}  ${newPassword}  ${comment}

tests/resources/Harbor-Pages/HomePage_Elements.robot

@@ -30,4 +30,5 @@ ${login_btn}  //*[@id='log_in']
 ${harbor_span_title}  //span[contains(., 'Harbor')]
 ${login_name}  //*[@id='login_username']
 ${login_pwd}  //*[@id='login_password']
+${close_scan_plugin_mesg}  //clr-icon[@class='close-icon']
 

tests/resources/Harbor-Pages/LDAP-Mode.robot

@@ -21,8 +21,8 @@ Resource  ../../resources/Util.robot
 *** Keywords ***
 
 Ldap User Should Not See Change Password
-    Click Element  //clr-header//clr-dropdown[2]//button
+    Retry Element Click  //clr-header//clr-dropdown[2]//button
-    Sleep  1
+    Sleep  2
     Page Should Not Contain  Password
 
 

tests/resources/Harbor-Pages/Project-Config.robot

@@ -42,7 +42,7 @@ Public Should Be Selected
 
 Project Should Be Public
     [Arguments]  ${projectName}
-    Page Should Contain Element  //clr-dg-row[contains(.,'${projectName}')]//clr-dg-cell[contains(.,'Public')]
+    Retry Wait Until Page Contains Element  //clr-dg-row[contains(.,'${projectName}')]//clr-dg-cell[contains(.,'Public')]
 
 Content Trust Should Be Selected
     Checkbox Should Be Selected  //hbr-project-policy-config//input[@name='content-trust']

tests/resources/Harbor-Pages/Project-Members.robot

@@ -21,28 +21,24 @@ Resource  ../../resources/Util.robot
 *** Keywords ***
 Go Into Project
     [Arguments]  ${project}  ${has_image}=${true}
-    Sleep  2
-    Retry Wait Element  ${search_input}
-    Input Text  ${search_input}  ${project}
-    Retry Wait Until Page Contains  ${project}
-    Retry Element Click  xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a
-    #To prevent waiting for a fixed-period of time for page loading and failure caused by exception, we add loop to re-run <Wait Until Element Is Visible And Enabled> when
-    #    exception was caught.
     :For  ${n}  IN RANGE  1  5
-    \    ${out}  Run Keyword If  ${has_image}==${false}  Run Keyword And Ignore Error  Wait Until Element Is Visible And Enabled  xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any repositories!\")]
+    \    Sleep  2
-    \    ...  ELSE  Run Keyword And Ignore Error  Wait Until Element Is Visible And Enabled  xpath=//clr-dg-cell[contains(.,'${project}/')]
+    \    Retry Wait Element  ${search_input}
-    \    Log To Console  ${out[0]}
+    \    Retry Clear Element Text  ${search_input}
-    \    ${result}  Set Variable If  '${out[0]}'=='PASS'  ${true}  ${false}
+    \    Input Text  ${search_input}  ${project}
-    \    Run Keyword If  ${result} == ${true}  Exit For Loop
+    \    ${out}  Run Keyword If  ${has_image}==${false}  Retry Double Keywords When Error  Retry Element Click  xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a  Wait Until Element Is Visible And Enabled  xpath=//clr-dg-placeholder[contains(.,\"We couldn\'t find any repositories!\")]  DoAssert=${false}
+    \    ...  ELSE  Retry Double Keywords When Error  Retry Element Click  xpath=//*[@id='project-results']//clr-dg-cell[contains(.,'${project}')]/a  Wait Until Element Is Visible And Enabled  xpath=//project-detail//hbr-repository-gridview//clr-dg-cell[contains(.,'${project}/')]  DoAssert=${false}
+    \    Log To Console  ${out}
+    \    Run Keyword If  ${out} == 'PASS'  Exit For Loop
     \    Sleep  1
-    Should Be Equal  ${result}  ${true}
+    Should Be Equal  ${out}  'PASS'
     Sleep  1
 
 Add User To Project Admin
     [Arguments]  ${project}  ${user}
     # *** this keyword has not been used ***
     Go Into Project
-    Retry Element Click  xpath=${project_member_tag_xpath}
+    Retry Element Click  xpath=${project_member_xpath}
     Retry Element Click  xpath=${project_member_add_button_xpath}
     Retry Text Input  xpath=${project_member_add_username_xpath}  ${user}
     Retry Element Click  xpath=${project_member_add_admin_xpath}
@@ -61,7 +57,7 @@ Search Project Member
 Change Project Member Role
     [Arguments]  ${project}  ${user}  ${role}
     Retry Element Click  xpath=//clr-dg-cell//a[contains(.,'${project}')]
-    Retry Element Click  xpath=${project_member_tag_xpath}
+    Retry Element Click  xpath=${project_member_xpath}
     Retry Element Click  xpath=//project-detail//clr-dg-row[contains(.,'${user}')]//clr-checkbox-wrapper
     #change role
     Retry Element Click  ${project_member_action_xpath}
@@ -74,13 +70,13 @@ User Can Change Role
      [arguments]  ${username}
      Retry Element Click  xpath=//clr-dg-row[contains(.,'${username}')]//input/../label
      Retry Element Click  xpath=//*[@id='member-action']
-     Page Should Not Contain Element  xpath=//button[@disabled='' and contains(.,'Admin')]
+     Retry Wait Until Page Not Contains Element  xpath=//button[@disabled='' and contains(.,'Admin')]
 
 User Can Not Change Role
      [arguments]  ${username}
      Retry Element Click  xpath=//clr-dg-row[contains(.,'${username}')]//input/../label
      Retry Element Click  xpath=//*[@id='member-action']
-     Page Should Contain Element  xpath=//button[@disabled='' and contains(.,'Admin')]
+     Retry Wait Until Page Contains Element  xpath=//button[@disabled='' and contains(.,'Admin')]
 
 #this keyworkd seems will not use any more, will delete in the future
 Non-admin View Member Account
@@ -88,11 +84,11 @@ Non-admin View Member Account
     Xpath Should Match X Times  //clr-dg-row-master  ${times}
 
 User Can Not Add Member
-    Page Should Contain Element  xpath=//button[@disabled='' and contains(.,'User')]
+    Retry Wait Until Page Contains Element  xpath=//button[@disabled='' and contains(.,'User')]
 
 Add Guest Member To Project
     [arguments]  ${member}
-    Retry Element Click  xpath=${project_member_add_button_xpath}
+    Retry Double Keywords When Error  Retry Element Click  xpath=${project_member_add_button_xpath}  Retry Wait Until Page Contains Element  xpath=${project_member_add_button_xpath}
     Retry Text Input  xpath=${project_member_add_username_xpath}  ${member}
     #select guest
     Mouse Down  xpath=${project_member_guest_radio_checkbox}
@@ -102,10 +98,9 @@ Add Guest Member To Project
 Delete Project Member
     [arguments]  ${member}
     Retry Element Click  xpath=//clr-dg-row[contains(.,'${member}')]//input/../label
-    Retry Element Click  ${member_action_xpath}
+    Retry Double Keywords When Error  Retry Element Click  ${member_action_xpath}  Retry Wait Until Page Contains Element  ${delete_action_xpath}
-    Retry Element Click  ${delete_action_xpath}
+    Retry Double Keywords When Error  Retry Element Click  ${delete_action_xpath}  Retry Wait Until Page Contains Element  ${repo_delete_on_card_view_btn}
-    Retry Element Click  ${repo_delete_on_card_view_btn}
+    Retry Double Keywords When Error  Retry Element Click  ${repo_delete_on_card_view_btn}  Retry Wait Element  xpath=${project_member_xpath}
-    Retry Wait Element  xpath=${project_member_xpath}
     Sleep  1
 
 User Should Be Owner Of Project
@@ -157,7 +152,7 @@ User Should Be Guest
     Go Into Project  ${project}
     Switch To Member
     User Can Not Add Member
-    Page Should Contain Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Guest')]
+    Retry Wait Until Page Contains Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Guest')]
     Logout Harbor
     Pull image  ${ip}  ${user}  ${password}  ${project}  hello-world
     Cannot Push image  ${ip}  ${user}  ${password}  ${project}  hello-world
@@ -172,7 +167,7 @@ User Should Be Developer
     Go Into Project  ${project}
     Switch To Member
     User Can Not Add Member
-    Page Should Contain Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Developer')]
+    Retry Wait Until Page Contains Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Developer')]
     Logout Harbor
     Push Image With Tag  ${ip}  ${user}  ${password}  ${project}  hello-world  v1
 
@@ -187,7 +182,7 @@ User Should Be Admin
     Switch To Member
     Add Guest Member To Project  ${guest}
     User Can Change Role  ${guest}
-    Page Should Contain Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Admin')]
+    Retry Wait Until Page Contains Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Admin')]
     Logout Harbor
     Push Image With Tag  ${ip}  ${user}  ${password}  ${project}  hello-world  v2
 
@@ -201,7 +196,7 @@ User Should Be Master
     Go Into Project  ${project}
     Delete Repo  ${project}
     Switch To Member
-    Page Should Contain Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Master')]
+    Retry Wait Until Page Contains Element  xpath=//clr-dg-row[contains(.,'${user}')]//clr-dg-cell[contains(.,'Master')]
     Logout Harbor
     Push Image With Tag  ${ip}  ${user}  ${password}  ${project}  hello-world  v3
 
@@ -210,5 +205,5 @@ Project Should Have Member
     Sign In Harbor  ${HARBOR_URL}  %{HARBOR_ADMIN}  %{HARBOR_PASSWORD}
     Go Into Project  ${project}
     Switch To Member
-    Page Should Contain Element  xpath=//clr-dg-cell[contains(., '${user}')]
+    Retry Wait Until Page Contains Element  xpath=//clr-dg-cell[contains(., '${user}')]
     Logout Harbor

tests/resources/Harbor-Pages/Project-Members_Elements.robot

@@ -16,7 +16,7 @@
 Documentation  This resource provides any keywords related to the Harbor private registry appliance
 
 *** Variables ***
-${project_member_tag_xpath}  //clr-main-container//project-detail/nav/ul//a[contains(.,'Members')]
+${project_member_tag_xpath}  //clr-main-container//project-detail/clr-tabs//a[contains(.,'Members')]
 ${project_member_add_button_xpath}  //project-detail//button[contains(.,'User')]
 ${project_member_add_username_xpath}  //*[@id='member_name']
 ${project_member_add_admin_xpath}  /html/body/harbor-app/harbor-shell/clr-main-container/div/div/project-detail/ng-component/div/div[1]/div/div[1]/add-member/clr-modal/div/div[1]/div/div[1]/div/div[2]/form/section/div[2]/div[1]/label

tests/resources/Harbor-Pages/Project-Repository_Elements.robot

@@ -21,3 +21,6 @@ ${first_cve_xpath}  //clr-dg-row[1]//clr-dg-cell//a
 ${view_log_xpath}  //clr-dg-row[1]//clr-dg-cell[4]//a
 ${build_history_btn}  //button[contains(.,'Build History')]
 ${build_history_data}  //clr-dg-row
+${push_image_command_btn}  //hbr-push-image-button//button
+
+

tests/resources/Harbor-Pages/Project-Tag-Retention.robot

@@ -27,11 +27,27 @@ Add A Tag Retention Rule
     Retry Element Click  xpath=${project_tag_retention_save_add_button_xpath}
     Retry Wait Until Page Contains Element   xpath=${project_tag_retention_rule_name_xpath}
 
+Add A Tag Immutability Rule
+    [Arguments]  ${scope}  ${tag}
+    Retry Double Keywords When Error  Retry Element Click  xpath=${project_tag_retention_add_rule_xpath}  Retry Wait Until Page Contains Element  xpath=${project_tag_immutability_save_add_button_xpath}
+    Retry Clear Element Text  ${project_tag_immutability_scope_input_xpath}
+    Retry Text Input  ${project_tag_immutability_scope_input_xpath}  ${scope}
+    Retry Clear Element Text  ${project_tag_immutability_tag_input_xpath}
+    Retry Text Input  ${project_tag_immutability_tag_input_xpath}  ${tag}
+    Retry Double Keywords When Error  Retry Element Click  xpath=${project_tag_immutability_save_add_button_xpath}  Retry Wait Until Page Contains Element  xpath=${project_tag_retention_rule_name_xpath}
+    Retry Wait Until Page Contains  ${scope}
+    Retry Wait Until Page Contains  ${tag}
+
 Delete A Tag Retention Rule
     Retry Element Click  xpath=${project_tag_retention_action_button_xpath}
     Retry Element Click  xpath=${project_tag_retention_delete_button_xpath}
     Retry Wait Until Page Not Contains Element   xpath=${project_tag_retention_rule_name_xpath}
 
+Delete A Tag Immutability Rule
+    Retry Element Click  xpath=${project_tag_retention_action_button_xpath}
+    Retry Element Click  xpath=${project_tag_retention_delete_button_xpath}
+    Retry Wait Until Page Not Contains Element   xpath=${project_tag_retention_rule_name_xpath}
+
 Edit A Tag Retention Rule
     [Arguments]  ${repos}   ${tags}
     Retry Element Click  xpath=${project_tag_retention_action_button_xpath}
@@ -42,6 +58,18 @@ Edit A Tag Retention Rule
     Retry Element Click  xpath=${project_tag_retention_save_add_button_xpath}
     Retry Wait Until Page Contains Element   xpath=//span[contains(@class, 'rule-name')]//span[contains(.,'${tags}')]
 
+Edit A Tag Immutability Rule
+    [Arguments]  ${scope}  ${tag}
+    Retry Element Click  xpath=${project_tag_retention_action_button_xpath}
+    Retry Element Click  xpath=${project_tag_retention_edit_button_xpath}
+    Retry Clear Element Text  ${project_tag_immutability_scope_input_xpath}
+    Retry Text Input  ${project_tag_immutability_scope_input_xpath}  ${scope}
+    Retry Clear Element Text  ${project_tag_immutability_tag_input_xpath}
+    Retry Text Input  ${project_tag_immutability_tag_input_xpath}  ${tag}
+    Retry Double Keywords When Error  Retry Element Click  xpath=${project_tag_immutability_save_add_button_xpath}  Retry Wait Until Page Contains Element  xpath=${project_tag_retention_rule_name_xpath}
+    Retry Wait Until Page Contains  ${scope}
+    Retry Wait Until Page Contains  ${tag}
+
 Set Daily Schedule
     Retry Element Click  xpath=${project_tag_retention_edit_schedule_xpath}
     Retry Element Click  xpath=${project_tag_retention_select_policy_xpath}

tests/resources/Harbor-Pages/Project-Tag-Retention_Elements.robot

@@ -39,3 +39,7 @@ ${project_tag_retention_action_button_xpath}  //button[contains(.,'ACTION')]
 ${project_tag_retention_delete_button_xpath}  //div[contains(@class,'dropdown-menu')]//button[contains(.,'Delete')]
 ${project_tag_retention_edit_button_xpath}  //div[contains(@class,'dropdown-menu')]//button[contains(.,'Edit')]
 ${project_tag_retention_modal_title_edit_xpath}  //h3[contains(.,'Edit Tag Retention Rule')]
+
+${project_tag_immutability_scope_input_xpath}  //*[@id='scope-input']
+${project_tag_immutability_tag_input_xpath}  //*[@id='tag-input']
+${project_tag_immutability_save_add_button_xpath}  //*[@id='add-edit-btn']

tests/resources/Harbor-Pages/Project-Webhooks.robot

@@ -6,6 +6,7 @@ Resource  ../../resources/Util.robot
 
 *** Keywords ***
 Switch To Project Webhooks
+    #Switch To Project Tab Overflow
     Retry Element Click  xpath=//project-detail//a[contains(.,'Webhooks')]
     Sleep  1
 

tests/resources/Harbor-Pages/Project.robot

@@ -27,7 +27,7 @@ Create An New Project
     Capture Page Screenshot
     Retry Text Input  xpath=${project_name_xpath}  ${projectname}
     ${element_project_public}=  Set Variable  xpath=${project_public_xpath}
-    Run Keyword If  '${public}' == 'true'  Run Keywords  Wait Until Element Is Visible And Enabled  ${element_project_public}  AND  Click Element  ${element_project_public}
+    Run Keyword If  '${public}' == 'true'  Run Keywords  Wait Until Element Is Visible And Enabled  ${element_project_public}  AND  Retry Element Click  ${element_project_public}
     Run Keyword If  '${count_quota}'!='${null}'  Input Count Quota  ${count_quota}
     Run Keyword If  '${storage_quota}'!='${null}'  Input Storage Quota  ${storage_quota}  ${storage_quota_unit}
     Capture Page Screenshot
@@ -45,6 +45,7 @@ Create An New Project With New User
 
 #It's the log of project.
 Go To Project Log
+    #Switch To Project Tab Overflow
     Retry Element Click  xpath=${project_log_xpath}
     Sleep  2
 
@@ -66,7 +67,18 @@ Switch To Project Configuration
     Sleep  1
 
 Switch To Tag Retention
-    Retry Element Click  xpath=${project_tag_retention_xpath}
+    #Switch To Project Tab Overflow
+    Retry Element Click  xpath=${project_tag_strategy_xpath}
+    Sleep  1
+
+Switch To Tag Immutability
+    #Switch To Project Tab Overflow
+    Retry Double Keywords When Error  Retry Element Click  xpath=${project_tag_strategy_xpath}  Retry Wait Until Page Contains Element  ${project_tag_immutability_switch}
+    Retry Double Keywords When Error  Retry Element Click  xpath=${project_tag_immutability_switch}  Retry Wait Until Page Contains  Immutability rules
+    Sleep  1
+
+Switch To Project Tab Overflow
+    Retry Element Click  xpath=${project_tab_overflow_btn}
     Sleep  1
 
 Navigate To Projects
@@ -172,11 +184,13 @@ Do Log Advanced Search
     Retry Text Input  xpath=//audit-log//hbr-filter//input  harbor
     Sleep  1
     Capture Page Screenshot  LogAdvancedSearch2.png
-    ${rc} =  Get Matching Xpath Count  //audit-log//clr-dg-row
+    ${rc} =  Get Element Count  //audit-log//clr-dg-row
     Should Be Equal As Integers  ${rc}  0
 
 Go Into Repo
     [Arguments]  ${repoName}
+    Sleep  2
+    Retry Wait Until Page Not Contains Element  ${repo_list_spinner}
     ${repo_name_element}=  Set Variable  xpath=//clr-dg-cell[contains(.,'${repoName}')]/a
     Retry Element Click  ${repo_search_icon}
     :For  ${n}  IN RANGE  1  10
@@ -184,12 +198,12 @@ Go Into Repo
     \    Retry Text Input  ${repo_search_input}  ${repoName}
     \    ${out}  Run Keyword And Ignore Error  Retry Wait Until Page Contains Element  ${repo_name_element}
     \    Exit For Loop If  '${out[0]}'=='PASS'
-    \    Capture Page Screenshot  gointo_${repoName}.png
     \    Sleep  2
+    Capture Page Screenshot
     Retry Double Keywords When Error  Retry Element Click  ${repo_name_element}  Retry Wait Until Page Not Contains Element  ${repo_name_element}
     Retry Wait Element  ${tag_table_column_pull_command}
     Retry Wait Element  ${tag_images_btn}
-    Capture Page Screenshot  gointo_${repoName}.png
+    Capture Page Screenshot
 
 Switch To CardView
     Retry Element Click  xpath=//hbr-repository-gridview//span[@class='card-btn']/clr-icon

tests/resources/Harbor-Pages/Project_Elements.robot

@@ -23,10 +23,13 @@ ${project_save_css}  html body.no-scrolling harbor-app harbor-shell clr-main-con
 ${log_xpath}  //clr-main-container//clr-vertical-nav//a[contains(.,'Logs')]
 ${projects_xpath}  //clr-main-container//clr-vertical-nav//a[contains(.,'Projects')]
 ${project_replication_xpath}  //project-detail//a[contains(.,'Replication')]
-${project_log_xpath}  //project-detail//li[contains(.,'Logs')]
+${project_log_xpath}  //project-detail//a[contains(.,'Logs')]
-${project_member_xpath}  //project-detail//li[contains(.,'Members')]
+${project_member_xpath}  //project-detail//a[contains(.,'Members')]
 ${project_config_tabsheet}  xpath=//project-detail//a[contains(.,'Configuration')]
-${project_tag_retention_xpath}  //nav//li//a[contains(.,'Tag')]
+${project_tag_strategy_xpath}  //project-detail//a[contains(.,'Tag Retention')]
+${project_tab_overflow_btn}  //clr-tabs//li//button[contains(@class,"dropdown-toggle")]
+
+${project_tag_immutability_switch}  //project-detail//a[contains(.,'Tag Immutability')]
 
 ${create_project_CANCEL_button_xpath}  xpath=//button[contains(.,'CANCEL')]
 ${create_project_OK_button_xpath}  xpath=//button[contains(.,'OK')]
@@ -40,10 +43,11 @@ ${repo_delete_btn}  xpath=//repository//button[contains(.,'Delete')]
 ${project_delete_btn}  xpath=//list-project//clr-datagrid//button[contains(.,'Delete')]
 ${tag_delete_btn}  xpath=//tag-repository//clr-datagrid//button[contains(.,'Delete')]
 ${user_delete_btn}  xpath=/clr-dropdown-menu//button[contains(.,'Delete')]
-#${repo_search_icon}  xpath=//hbr-filter//clr-icon
+${repo_search_icon}  xpath=//hbr-filter//clr-icon
-#${repo_search_input}  xpath=//hbr-filter//input
+${repo_search_input}  xpath=//hbr-filter//input
-${repo_search_icon}  xpath=//hbr-repository-gridview//clr-datagrid//clr-dg-column[contains(.,'Name')]//clr-dg-string-filter//button//clr-icon
+${repo_list_spinner}  xpath=//clr-datagrid//clr-spinner
-${repo_search_input}  xpath=//div[@class[contains(.,'datagrid-filter')]]//input
+#${repo_search_icon}  xpath=//hbr-repository-gridview//clr-datagrid//clr-dg-column[contains(.,'Name')]//clr-dg-string-filter//button//clr-icon
+#${repo_search_input}  xpath=//div[@class[contains(.,'datagrid-filter')]]//input
 ${repo_tag_1st_checkbox}  xpath=//clr-datagrid//clr-dg-row//clr-checkbox-wrapper
 ${tag_table_column_pull_command}  xpath=//clr-dg-column//span[contains(.,'Pull Command')]
 ${tag_table_column_tag}  xpath=//clr-dg-column//span[contains(.,'Tag')]

tests/resources/Harbor-Pages/Project_Robot_Account.robot

@@ -0,0 +1,23 @@
+*** Settings ***
+
+Resource  ../../resources/Util.robot
+
+*** Variables ***
+
+*** Keywords ***
+Switch To Project Robot Account
+    #Switch To Project Tab Overflow
+    Retry Element Click  ${project_robot_account_tabpage}
+    Retry Wait Until Page Contains Element  ${project_robot_account_create_btn}
+
+Create A Robot Account And Return Token
+    [Arguments]    ${projectname}    ${robot_account_name}    ${project_has_image}=${false}
+    Go Into Project    ${projectname}    has_image=${project_has_image}
+    Switch To Project Robot Account
+    Retry Element Click    ${project_robot_account_create_btn}
+    Retry Text Input    ${project_robot_account_create_name_input}    ${robot_account_name}
+    Retry Double Keywords When Error    Retry Element Click    ${project_robot_account_create_save_btn}    Retry Wait Until Page Not Contains Element    ${project_robot_account_create_save_btn}
+    ${token}=    Get Value    ${project_robot_account_token_input}
+    [Return]    ${token}
+
+

tests/resources/Harbor-Pages/Project_Robot_Account_Elements.robot

@@ -0,0 +1,9 @@
+*** Settings ***
+Documentation  This resource provides any keywords related to the Harbor robot-account feature
+
+*** Variables ***
+${project_robot_account_tabpage}  xpath=//project-detail//a[contains(.,'Robot Accounts')]
+${project_robot_account_create_btn}  xpath=//project-detail/app-robot-account//button
+${project_robot_account_token_input}  xpath=//app-robot-account//hbr-copy-input//input
+${project_robot_account_create_name_input}  //input[@id='robot_name']
+${project_robot_account_create_save_btn}  //add-robot//button[contains(.,'SAVE')]

tests/resources/Harbor-Pages/Replication.robot

@@ -185,11 +185,14 @@ Find Item And Click Delete Button
     Retry Select Object    ${name}
     Retry Element Click    ${action_bar_delete}
 
-Edit Replication Rule By Name
+Switch To Replication Manage Page
     [Arguments]    ${name}
     Switch To Registries
     Switch To Replication Manage
-    Find Item And Click Edit Button  ${name}
+
+Edit Replication Rule By Name
+    [Arguments]    ${name}
+    Retry Double Keywords When Error  Switch To Replication Manage Page  "NULL"  Find Item And Click Edit Button  ${name}
 
 Delete Replication Rule By Name
     [Arguments]    ${name}
@@ -226,6 +229,11 @@ Select Rule And Replicate
     Retry Element Click    ${replication_exec_id}
     Retry Double Keywords When Error    Retry Element Click    xpath=${dialog_replicate}    Retry Wait Until Page Not Contains Element    xpath=${dialog_replicate}
 
+Select Rule And Click Edit Button
+    [Arguments]  ${rule_name}
+    Retry Element Click  //clr-dg-row[contains(.,'${rule_name}')]//clr-radio-wrapper/label
+    Retry Element Click  ${edit_replication_rule_id}
+
 Delete Replication Rule
     [Arguments]  ${name}
     Retry Element Click  ${endpoint_filter_search}
@@ -235,3 +243,17 @@ Delete Replication Rule
     Retry Element Click  ${action_bar_delete}
     Wait Until Page Contains Element  ${dialog_delete}
     Retry Element Click  ${dialog_delete}
+
+Image Should Be Replicated To Project
+    [Arguments]  ${project}  ${image}  ${period}=60  ${times}=10
+    :For  ${n}  IN RANGE  1  ${times}
+    \    Sleep  ${period}
+    \    Go Into Project    ${project}
+    \    Switch To Project Repo
+    \    #In AWS-ECR, under repository a, there're only several images: httpd,alpine,hello-world.
+    \    ${out}  Run Keyword And Ignore Error  Retry Wait Until Page Contains  ${project}/${image}
+    \    Log To Console  Return value is ${out[0]}
+    \    Exit For Loop If  '${out[0]}'=='PASS'
+    \    Sleep  5
+    Run Keyword If  '${out[0]}'=='FAIL'  Capture Page Screenshot
+    Should Be Equal As Strings  '${out[0]}'  'PASS'

tests/resources/Harbor-Pages/ToolKit.robot

@@ -39,9 +39,17 @@ Filter Object
     Retry Element Click  xpath=//hbr-filter//clr-icon
     ${element}=  Set Variable  xpath=//hbr-filter//input
     Wait Until Element Is Visible And Enabled  ${element}
-    Input Text   ${element}  ${kw}
+    Retry Clear Element Text  ${element}
+    Retry Text Input   ${element}  ${kw}
     Sleep  3
 
+Filter Project
+#Filter project repo user tag.
+    [Arguments]    ${kw}
+    Retry Element Click  ${log_xpath} 
+    Retry Element Click  ${projects_xpath}
+    Filter Object  ${kw}
+
 Select Object
 #select single element such as user project repo tag
     [Arguments]    ${obj}

tests/resources/Harbor-Pages/UserProfile.robot

@@ -43,4 +43,4 @@ Logout Harbor
     Retry Link Click  Log Out
     Capture Page Screenshot  Logout.png
     Sleep  2
-    Wait Until Keyword Succeeds  5x  1  Page Should Contain Element  ${sign_in_title_xpath}
+    Wait Until Keyword Succeeds  5x  1  Retry Wait Until Page Contains Element  ${sign_in_title_xpath}

tests/resources/Harbor-Pages/Verify.robot

@@ -26,7 +26,7 @@ Verify Project
     Init Chrome Driver
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     :FOR    ${project}    IN    @{project}
-    \    Page Should Contain    ${project}
+    \    Retry Wait Until Page Contains    ${project}
     Verify Project Metadata  ${json}
     Close Browser
 
@@ -56,10 +56,10 @@ Verify Project Metadata
     \    Switch To Project Configuration
     \    Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.public  ${project_config_public_checkbox}
     \    Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.enable_content_trust  ${project_config_content_trust_checkbox}
-    \    Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.automatically_scan_images_on_push  ${project_config_scan_images_on_push_checkbox}
+    \    Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.auto_scan  ${project_config_scan_images_on_push_checkbox}
-    \    Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.prevent_vulnerable_images_from_running  ${project_config_prevent_vulnerable_images_from_running_checkbox}
+    \    Verify Checkbox  ${json}  $.projects[?(@.name=${project})].configuration.prevent_vul  ${project_config_prevent_vulnerable_images_from_running_checkbox}
     \    ${ret}    Get Selected List Value    ${project_config_severity_select}
-    \    @{severity}=    Get Value From Json    ${json}    $.projects[?(@.name=${project})].configuration.prevent_vlunerable_images_from_running_severity
+    \    @{severity}=    Get Value From Json    ${json}    $.projects[?(@.name=${project})].configuration.severity
     \    Should Contain    ${ret}    @{severity}[0]
     \    Navigate To Projects
     Close Browser
@@ -145,12 +145,40 @@ Verify Endpoint
 
 Verify Replicationrule
     [Arguments]    ${json}
-    @{replicationrule}=  Get Value From Json  ${json}  $.replicationrule..name
+    @{replicationrules}=    Get Value From Json    ${json}    $.replicationrule.[*].rulename
-    Init Chrome Driver
+    @{endpoints}=    Get Value From Json    ${json}    $.endpoint.[*].name
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    : FOR    ${replicationrule}    IN    @{replicationrules}
-    Switch To Replication Manage
+    \    Init Chrome Driver
-    :For    ${replicationrule}    IN    @{replicationrule}
+    \    Log To Console    -----replicationrule-----"${replicationrule}"------------
-    \    Page Should Contain    ${replicationrule}
+    \    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    \    Switch To Replication Manage
+    \    Select Rule And Click Edit Button    ${replicationrule}
+    \    @{is_src_registry}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].is_src_registry
+    \    @{trigger_type}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].trigger_type
+    \    @{name_filters}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].name_filters
+    \    @{tag_filters}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].tag_filters
+    \    @{dest_namespace}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].dest_namespace
+    \    @{cron}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].cron
+    \    @{is_src_registry}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].is_src_registry
+    \    Log To Console    -----is_src_registry-----@{is_src_registry}[0]------------
+    \    @{endpoint}=    Get Value From Json    ${json}    $.replicationrule[?(@.rulename=${replicationrule})].endpoint
+    \    Log To Console    -----endpoint-----@{endpoint}------------
+    \    ${endpoint0}=   Set Variable    @{endpoint}[0]
+    \    Log To Console    -----endpoint0-----${endpoint0}------------
+    \    @{endpoint_type}=    Get Value From Json    ${json}    $.endpoint[?(@.name=${endpoint0})].type
+    \    Retry Textfield Value Should Be    ${source_project}    @{name_filters}[0]
+    \    Retry Textfield Value Should Be    ${filter_tag}    @{tag_filters}[0]
+    \    Retry Textfield Value Should Be    ${rule_name_input}    ${replicationrule}
+    \    Retry Textfield Value Should Be    ${dest_namespace_xpath}    @{dest_namespace}[0]
+    \    Log To Console    -----endpoint_type-----@{endpoint_type}[0]------------
+    \    ${registry}=    Set Variable If    "@{endpoint_type}[0]"=="harbor"    ${endpoint0}-https://${IP}    ${endpoint0}-https://hub.docker.com
+    \    Log To Console    -------registry---${registry}------------
+    \    Run Keyword If    '@{is_src_registry}[0]' == '${true}'    Retry List Selection Should Be    ${src_registry_dropdown_list}    ${registry}
+    \    ...    ELSE    Retry List Selection Should Be    ${dest_registry_dropdown_list}    ${registry}
+    \    #\    Retry List Selection Should Be    ${rule_resource_selector}    ${resource_type}
+    \    Retry List Selection Should Be    ${rule_trigger_select}    @{trigger_type}[0]
+    \    Run Keyword If    '@{trigger_type}[0]' == 'scheduled'    Log To Console    ----------@{trigger_type}[0]------------
+    \    Run Keyword If    '@{trigger_type}[0]' == 'scheduled'    Retry Textfield Value Should Be    ${targetCron_id}    @{cron}[0]
     Close Browser
 
 Verify Project Setting

tests/resources/Harbor-Util.robot

@@ -80,9 +80,7 @@ Switch To LDAP
     Config Harbor cfg  auth=ldap_auth  http_proxy=https
     Prepare
     Up Harbor
-    ${rc}=  Run And Return Rc  docker pull osixia/openldap:1.1.7
+    Docker Pull  osixia/openldap:1.1.7
-    Log  ${rc}
-    Should Be Equal As Integers  ${rc}  0
     ${rc}  ${output}=  Run And Return Rc And Output  cd tests && ./ldapprepare.sh
     Log  ${rc}
     Log  ${output}
@@ -95,28 +93,13 @@ Switch To LDAP
 Enable Notary Client
     ${rc}  ${output}=  Run And Return Rc And Output  rm -rf ~/.docker/
     Log  ${rc}
-    Should Be Equal As Integers  ${rc}  0
+    ${rc}  ${output}=  Run And Return Rc and Output  curl -o /notary_ca.crt -s -k -X GET --header 'Accept: application/json' -u 'admin:Harbor12345' 'https://${ip}/api/systeminfo/getcert'
-    Log  ${ip}
-    Log To Console  ${ip}
-    ${rc}=  Run And Return Rc  mkdir -p /etc/docker/certs.d/${ip}/
-    Should Be Equal As Integers  ${rc}  0
-    Log To Console  ${notaryServerEndpointNoSubDir}
-    ${rc}=  Run And Return Rc  mkdir -p ~/.docker/tls/${notaryServerEndpointNoSubDir}/
-    Should Be Equal As Integers  ${rc}  0
-    ${rc}  ${output}=  Run And Return Rc And Output  cp ./harbor_ca.crt /etc/docker/certs.d/${ip}/
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
-    ${rc}  ${output}=  Run And Return Rc And Output  cp ./harbor_ca.crt ~/.docker/tls/${notaryServerEndpointNoSubDir}/
-    Log  ${output}
-    Should Be Equal As Integers  ${rc}  0
-    ${rc}  ${output}=  Run And Return Rc And Output  ls -la /etc/docker/certs.d/${ip}/
-    Log  ${output}
-    ${rc}  ${output}=  Run And Return Rc And Output  ls -la ~/.docker/tls/${notaryServerEndpointNoSubDir}/
-    Log  ${output}
 
 Remove Notary Signature
     [Arguments]  ${ip}  ${image}
-    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/notary-remove-image-signature.expect ${ip} library ${image}
+    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/notary-remove-image-signature.expect ${ip} library ${image} ${notaryServerEndpoint}
     Log To Console  ${output}
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0

tests/resources/Helm-Util.robot

@@ -28,7 +28,7 @@ Helm Repo Add
     Wait Unitl Command Success  helm repo add --ca-file /helm_ca/server.crt --username=${user} --password=${pwd} ${helm_repo_name} ${harbor_url}/chartrepo/${project_name}
 
 Helm Repo Push
-    [Arguments]  ${harbor_url}  ${user}  ${pwd}  ${chart_filename}  ${project_name}=library  ${helm_repo_name}=myrepo
+    [Arguments]  ${user}  ${pwd}  ${chart_filename}  ${helm_repo_name}=myrepo
     ${current_dir}=  Run  pwd
     Run  cd ${current_dir}
     Run  wget ${harbor_chart_file_url}

tests/resources/TestCaseBody.robot

@@ -98,7 +98,7 @@ Body Of List Helm Charts
     Close Browser
 
 Body Of Admin Push Signed Image
-    [Arguments]  ${image}  ${with_remove}=${false}
+    [Arguments]  ${image}=tomcat  ${with_remove}=${false}
     Enable Notary Client
 
     Docker Pull  ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}
@@ -150,7 +150,18 @@ Helm CLI Push Without Sign In Harbor
     ${d}=   Get Current Date    result_format=%m%s
     Create An New Project  project${d}
     Helm Repo Add  ${HARBOR_URL}  ${sign_in_user}  ${sign_in_pwd}  project_name=project${d}
-    Helm Repo Push  ${HARBOR_URL}  ${sign_in_user}  ${sign_in_pwd}  ${harbor_chart_filename}  project_name=project${d}
+    Helm Repo Push  ${sign_in_user}  ${sign_in_pwd}  ${harbor_chart_filename}
+    Go Into Project  project${d}  has_image=${false}
+    Switch To Project Charts
+    Go Into Chart Version  ${harbor_chart_name}
+    Retry Wait Until Page Contains  ${harbor_chart_version}
+    Capture Page Screenshot
+
+Helm3 CLI Push Without Sign In Harbor
+    [Arguments]  ${sign_in_user}  ${sign_in_pwd}
+    ${d}=   Get Current Date    result_format=%m%s
+    Create An New Project  project${d}
+    Helm Repo Push  ${sign_in_user}  ${sign_in_pwd}  ${harbor_chart_filename}  helm_repo_name=${HARBOR_URL}/chartrepo/project${d}
     Go Into Project  project${d}  has_image=${false}
     Switch To Project Charts
     Go Into Chart Version  ${harbor_chart_name}

tests/resources/Util.robot

@@ -47,6 +47,8 @@ Resource  Harbor-Pages/Project-Retag.robot
 Resource  Harbor-Pages/Project-Retag_Elements.robot
 Resource  Harbor-Pages/Project-Tag-Retention.robot
 Resource  Harbor-Pages/Project-Tag-Retention_Elements.robot
+Resource  Harbor-Pages/Project_Robot_Account.robot
+Resource  Harbor-Pages/Project_Robot_Account_Elements.robot
 Resource  Harbor-Pages/Replication.robot
 Resource  Harbor-Pages/Replication_Elements.robot
 Resource  Harbor-Pages/UserProfile.robot
@@ -66,7 +68,6 @@ Resource  Harbor-Pages/OIDC_Auth_Elements.robot
 Resource  Harbor-Pages/Verify.robot
 Resource  Docker-Util.robot
 Resource  Helm-Util.robot
-Resource  Admiral-Util.robot
 Resource  OVA-Util.robot
 Resource  Cert-Util.robot
 Resource  SeleniumUtil.robot
@@ -203,7 +204,7 @@ Clear Field Of Characters
     [Arguments]  ${field}  ${character count}
     [Documentation]  This keyword pushes the delete key (ascii: \8) a specified number of times in a specified field.
     : FOR  ${index}  IN RANGE  ${character count}
-    \    Press Key  ${field}  \\8
+    \    Press Keys  ${field}  \\8
 
 Wait Unitl Command Success
     [Arguments]  ${cmd}  ${times}=8
@@ -225,13 +226,14 @@ Command Should be Failed
 Retry Keyword When Error
     [Arguments]  ${keyword}  @{elements}
     :For  ${n}  IN RANGE  1  6
-    \    Log To Console  Trying ${keyword} ${n} times ...
+    \    Log To Console  Trying ${keyword} elements @{elements} ${n} times ...
     \    ${out}  Run Keyword And Ignore Error  ${keyword}  @{elements}
     \    Log To Console  Return value is ${out[0]}
     \    Exit For Loop If  '${out[0]}'=='PASS'
     \    Sleep  2
     Run Keyword If  '${out[0]}'=='FAIL'  Capture Page Screenshot
     Should Be Equal As Strings  '${out[0]}'  'PASS'
+    [Return]  ${out[1]}
 
 Retry Keyword When Return Value Mismatch
     [Arguments]  ${keyword}  ${expected_value}  ${count}  @{elements}
@@ -246,8 +248,8 @@ Retry Keyword When Return Value Mismatch
     Should Be Equal As Strings  ${status}  'PASS'
 
 Retry Double Keywords When Error
-    [Arguments]  ${keyword1}  ${element1}  ${keyword2}  ${element2}
+    [Arguments]  ${keyword1}  ${element1}  ${keyword2}  ${element2}  ${DoAssert}=${true}
-    :For  ${n}  IN RANGE  1  3
+    :For  ${n}  IN RANGE  1  5
     \    Log To Console  Trying ${keyword1} and ${keyword2} ${n} times ...
     \    ${out1}  Run Keyword And Ignore Error  ${keyword1}  ${element1}
     \    Capture Page Screenshot
@@ -257,6 +259,7 @@ Retry Double Keywords When Error
     \    Log To Console  Return value is ${out1[0]} ${out2[0]}
     \    Exit For Loop If  '${out2[0]}'=='PASS'
     \    Sleep  1
+    Return From Keyword If  ${DoAssert} == ${false}  '${out2[0]}'
     Should Be Equal As Strings  '${out2[0]}'  'PASS'
 
 Run Curl And Return Json

tests/robot-cases/Group0-Util/notary-push-image.sh

@@ -11,11 +11,14 @@ echo $IP
 mkdir -p /etc/docker/certs.d/$IP/
 mkdir -p ~/.docker/tls/$IP:4443/
 
-cp /notary_ca/ca.crt /etc/docker/certs.d/$IP/
+cp /notary_ca.crt /etc/docker/certs.d/$IP/
-cp /notary_ca/ca.crt ~/.docker/tls/$IP:4443/
+cp /notary_ca.crt ~/.docker/tls/$IP:4443/
+
+mkdir -p ~/.docker/tls/$notaryServerEndpoint/
+cp /notary_ca.crt ~/.docker/tls/$notaryServerEndpoint/
 
 export DOCKER_CONTENT_TRUST=1
-export DOCKER_CONTENT_TRUST_SERVER=https://$IP:4443
+export DOCKER_CONTENT_TRUST_SERVER=https://$notaryServerEndpoint
 
 export NOTARY_ROOT_PASSPHRASE=$PASSHRASE
 export NOTARY_TARGETS_PASSPHRASE=$PASSHRASE

tests/robot-cases/Group1-Nightly/Chartmuseum.robot

@@ -33,3 +33,10 @@ Test Case - Helm CLI Push
     ${pwd}=    Set Variable    Test1@34
     Sign In Harbor  ${HARBOR_URL}  ${user}  ${pwd}
     Helm CLI Push Without Sign In Harbor  ${user}  ${pwd}
+
+Test Case - Helm3 CLI Push
+    Init Chrome Driver
+    ${user}=    Set Variable    user004
+    ${pwd}=    Set Variable    Test1@34
+    Sign In Harbor  ${HARBOR_URL}  ${user}  ${pwd}
+    Helm3 CLI Push Without Sign In Harbor  ${user}  ${pwd}

tests/robot-cases/Group1-Nightly/Common.robot

@@ -42,20 +42,6 @@ Test Case - Delete A Project
     Delete A Project Without Sign In Harbor
     Close Browser
 
-Test Case - Read Only Mode
-    Init Chrome Driver
-    ${d}=   Get Current Date    result_format=%m%s
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Create An New Project  project${d}
-
-    Enable Read Only
-    Cannot Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  busybox:latest
-
-    Disable Read Only
-    Sleep  5
-    Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  busybox:latest
-    Close Browser
-
 Test Case - Repo Size
     Init Chrome Driver
     ${d}=  Get Current Date    result_format=%m%s
@@ -131,8 +117,7 @@ Test Case - Project Level Policy Public
     # Here logout and login to try avoid a bug only in autotest
     Logout Harbor
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Filter Object  project${d}
+    Retry Double Keywords When Error  Filter Project  project${d}  Project Should Be Public  project${d}
-    Project Should Be Public  project${d}
     Close Browser
 
 Test Case - Verify Download Ca Link
@@ -420,7 +405,7 @@ Test Case - Developer Operate Labels
     Sign In Harbor  ${HARBOR_URL}  user022  Test1@34
     Go Into Project  project${d}  has_image=${false}
     Sleep  3
-    Page Should Not Contain Element  xpath=//a[contains(.,'Labels')]
+    Retry Wait Until Page Not Contains Element  xpath=//a[contains(.,'Labels')]
     Close Browser
 
 Test Case - Retag A Image Tag
@@ -445,7 +430,7 @@ Test Case - Retag A Image Tag
     Page Should Contain  ${target_image_name}
     Go Into Repo  project${random_num1}${random_num2}/${target_image_name}
     Sleep  1
-    Page Should Contain Element  xpath=${tag_value_xpath}
+    Retry Wait Until Page Contains Element  xpath=${tag_value_xpath}
     Close Browser
 
 Test Case - Create An New Project With Quotas Set
@@ -501,8 +486,8 @@ Test Case - Project Storage Quotas Dispaly And Control
     ${d}=  Get Current Date  result_format=%m%s
     ${storage_quota}=  Set Variable  330
     ${storage_quota_unit}=  Set Variable  MB
-    ${image_a}=  Set Variable  ${LOCAL_REGISTRY}/harbor-ci/redis
+    ${image_a}=  Set Variable  redis
-    ${image_b}=  Set Variable  ${LOCAL_REGISTRY}/harbor-ci/logstash
+    ${image_b}=  Set Variable  logstash
     ${image_a_size}=    Set Variable    34.15MB
     ${image_b_size}=    Set Variable    321.03MB
     ${image_a_ver}=  Set Variable  donotremove5.0
@@ -564,37 +549,16 @@ Test Case - Project Quotas Control Under GC
     ${image_a_size}=    Set Variable    321.03MB
     ${image_a_ver}=  Set Variable  6.8.3
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Capture Page Screenshot
     Create An New Project  project${d}  storage_quota=${storage_quota}  storage_quota_unit=${storage_quota_unit}
-    Cannot Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  ${image_a}:${image_a_ver}  err_msg=Quota exceeded when processing the request of adding 82.5 MiB of storage resource, which when updated to current usage of 166.6 MiB will exceed the configured upper limit of 200.0 MiB
+    Capture Page Screenshot
+    Cannot Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  ${image_a}:${image_a_ver}  err_msg=will exceed the configured upper limit of 200.0 MiB
+    Capture Page Screenshot
     GC Now  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     @{param}  Create List  project${d}
     Retry Keyword When Return Value Mismatch  Get Project Storage Quota Text From Project Quotas List  0Byte of ${storage_quota}${storage_quota_unit}  60  @{param}
     Close Browser
 
-Test Case - Can Not Retag Image In ReadOnly Mode
-    Init Chrome Driver
-    ${random_num1}=   Get Current Date    result_format=%m%s
-    ${random_num2}=   Evaluate  str(random.randint(1000,9999))  modules=random
-
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Create An New Project  project${random_num1}
-    Create An New Project  project${random_num1}${random_num2}
-
-    Go Into Project  project${random_num1}  has_image=${false}
-    Sleep  1
-    Push Image With Tag  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${random_num1}  redis  ${image_tag}
-    Sleep  1
-    Enable Read Only
-    Go Into Repo  project${random_num1}/redis
-    Retag Image  ${image_tag}  project${random_num1}${random_num2}  ${target_image_name}  ${target_tag_value}
-    Retry Wait Element Not Visible  ${repo_retag_confirm_dlg}
-    Navigate To Projects
-    Go Into Project  project${random_num1}${random_num2}  has_image=${false}
-    Sleep  10
-    Go Into Project  project${random_num1}${random_num2}  has_image=${false}
-    Disable Read Only
-    Close Browser
-
 Test Case - Create New Webhook
     Init Chrome Driver
     ${d}=    Get Current Date    result_format=%m%s
@@ -646,3 +610,76 @@ Test Case - Tag Retention
     Execute Dry Run
     Execute Run
     Close Browser
+
+Test Case - Tag Immutability
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    ${d}=    Get Current Date    result_format=%m%s
+    Create An New Project  project${d}
+    Go Into Project  project${d}  has_image=${false}
+    Switch To Tag Immutability
+    Add A Tag Immutability Rule  1212  3434
+    Delete A Tag Immutability Rule
+    Add A Tag Immutability Rule  5566  7788
+    Edit A Tag Immutability Rule  hello-world  latest
+    Push Image With Tag  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  hello-world  latest
+    Push Image With Tag  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  busybox  latest
+    Go Into Project  project${d}
+    @{repo_list}  Create List  hello-world  busybox
+    Multi-delete Object  ${repo_delete_btn}  @{repo_list}
+    # Verify
+    Delete Fail  hello-world
+    Delete Success  busybox
+    Close Browser
+
+
+Test Case - Robot Account
+    Init Chrome Driver
+    ${d}=    Get Current Date    result_format=%m%s
+    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Create An New Project    project${d}
+    ${token}=    Create A Robot Account And Return Token    project${d}    robot${d}
+    Log To Console    ${token}
+    Log    ${token}
+    Push image  ${ip}  robot${d}  ${token}  project${d}  hello-world:latest  is_robot=${true}
+    Pull image  ${ip}  robot${d}  ${token}  project${d}  hello-world:latest  is_robot=${true}
+
+Test Case - Read Only Mode
+    Init Chrome Driver
+    ${d}=   Get Current Date    result_format=%m%s
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  project${d}
+
+    Enable Read Only
+    Cannot Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  busybox:latest
+
+    Disable Read Only
+    Sleep  5
+    Push image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  busybox:latest
+    Close Browser
+
+Test Case - Can Not Retag Image In ReadOnly Mode
+    Init Chrome Driver
+    ${random_num1}=   Get Current Date    result_format=%m%s
+    ${random_num2}=   Evaluate  str(random.randint(1000,9999))  modules=random
+
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Create An New Project  project${random_num1}
+    Create An New Project  project${random_num1}${random_num2}
+
+    Go Into Project  project${random_num1}  has_image=${false}
+    Sleep  1
+    Push Image With Tag  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${random_num1}  redis  ${image_tag}
+    Sleep  1
+    Enable Read Only
+    Go Into Repo  project${random_num1}/redis
+    Retag Image  ${image_tag}  project${random_num1}${random_num2}  ${target_image_name}  ${target_tag_value}
+    Retry Wait Element Not Visible  ${repo_retag_confirm_dlg}
+    Navigate To Projects
+    Go Into Project  project${random_num1}${random_num2}  has_image=${false}
+    Sleep  10
+    Go Into Project  project${random_num1}${random_num2}  has_image=${false}
+    Disable Read Only
+    Close Browser
+
+

tests/robot-cases/Group1-Nightly/LDAP.robot

@@ -44,6 +44,7 @@ Test Case - System Admin On-board New Member
     ${d}=    Get Current Date    result_format=%m%s
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     Switch To User Tag
+    Sleep  2
     Page Should Not Contain  mike02
     Navigate To Projects
     Create An New Project  project${d}
@@ -60,8 +61,10 @@ Test Case - LDAP User On-borad New Member
     Create An New Project  project${d}
     Go Into Project  project${d}  has_image=${false}
     Switch To Member
+    Sleep  2
     Page Should Not Contain  mike04
     Add Guest Member To Project  mike04
+    Sleep  2
     Page Should Contain  mike04
     Close Browser
 

tests/robot-cases/Group1-Nightly/Nightly.robot

@@ -393,7 +393,7 @@ TestCase - Developer Operate Labels
     Sign In Harbor  ${HARBOR_URL}  bob${d}  Test1@34
     Go Into Project  project${d}  has_image=${false}
     Sleep  3
-    Page Should Not Contain Element  xpath=//a[contains(.,'Labels')]
+    Retry Wait Until Page Not Contains Element  xpath=//a[contains(.,'Labels')]
     Close Browser
 
 Test Case - Scan A Tag In The Repo

tests/robot-cases/Group1-Nightly/Notary.robot

@@ -40,7 +40,7 @@ Test Case - Project Level Policy Content Trust
     Close Browser
 
 Test Case - Admin Push Signed Image
-    Body Of Admin Push Signed Image  tomcat
+    Body Of Admin Push Signed Image
 
 Test Case - Admin Push Signed Image And Remove Signature
     Body Of Admin Push Signed Image  image=mariadb  with_remove=${true}

tests/robot-cases/Group1-Nightly/Replication.robot

@@ -37,7 +37,7 @@ Test Case - Get Harbor Version
 
 Test Case - Pro Replication Rules Add
     Init Chrome Driver
-    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Switch To Replication Manage
     Check New Rule UI Without Endpoint
     Close Browser
@@ -156,21 +156,16 @@ Test Case - Replication Of Pull Images from DockerHub To Self
     Switch To Replication Manage
     Create A Rule With Existing Endpoint    rule${d}    pull    danfengliu/*    image    e${d}    project${d}
     Select Rule And Replicate  rule${d}
-    Sleep    30
-    Go Into Project    project${d}
-    Switch To Project Repo
     #In docker-hub, under repository danfengliu, there're only 2 images: centos,mariadb.
-    Retry Wait Until Page Contains    project${d}/centos
+    Image Should Be Replicated To Project  project${d}  centos
-    Go Into Project    project${d}
+    Image Should Be Replicated To Project  project${d}  mariadb
-    Switch To Project Repo
-    Retry Wait Until Page Contains    project${d}/mariadb
     Close Browser
 
 Test Case - Replication Of Push Images from Self To Harbor
     Init Chrome Driver
     ${d}=    Get Current Date    result_format=%m%s
     #login source
-    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Create An New Project    project${d}
     Push Image    ${ip}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}    project${d}    hello-world
     Push Image    ${ip}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}    project${d}    busybox:latest
@@ -181,27 +176,25 @@ Test Case - Replication Of Push Images from Self To Harbor
     Create A Rule With Existing Endpoint    rule${d}    push    project${d}/*    image    e${d}    project_dest${d}
     #logout and login target
     Logout Harbor
-    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Create An New Project    project_dest${d}
     #logout and login source
     Logout Harbor
-    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Switch To Replication Manage
     Select Rule And Replicate  rule${d}
-    Sleep    20
+    Sleep  20
     Logout Harbor
-    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
-    Go Into Project    project_dest${d}
+    Image Should Be Replicated To Project  project_dest${d}  hello-world
-    Switch To Project Repo
+    Image Should Be Replicated To Project  project_dest${d}  busybox
-    Retry Wait Until Page Contains    project_dest${d}/hello-world
-    Retry Wait Until Page Contains    project_dest${d}/busybox
     Close Browser
 
 Test Case - Replication Of Push Chart from Self To Harbor
     Init Chrome Driver
     ${d}=    Get Current Date    result_format=%m%s
     #login source
-    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Create An New Project    project${d}
     Go Into Project  project${d}  has_image=${false}
     Switch To Project Charts
@@ -212,16 +205,16 @@ Test Case - Replication Of Push Chart from Self To Harbor
     Create A Rule With Existing Endpoint    rule${d}    push    project${d}/*    chart    e${d}    project_dest${d}
     #logout and login target
     Logout Harbor
-    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Create An New Project    project_dest${d}
     #logout and login source
     Logout Harbor
-    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Switch To Replication Manage
     Select Rule And Replicate    rule${d}
     Sleep    20
     Logout Harbor
-    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Go Into Project    project_dest${d}    has_image=${false}
     Switch To Project Charts
     Go Into Chart Version    ${harbor_chart_name}
@@ -233,7 +226,7 @@ Test Case - Replication Of Push Images from Self To Harbor By Push Event
     Init Chrome Driver
     ${d}=    Get Current Date    result_format=%m%s
     #login source
-    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    ${HARBOR_URL}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Create An New Project    project${d}
     Switch To Registries
     Create A New Endpoint    harbor    e${d}    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
@@ -242,13 +235,10 @@ Test Case - Replication Of Push Images from Self To Harbor By Push Event
     ...    Event Based
     #logout and login target
     Logout Harbor
-    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}
+    Sign In Harbor    https://${ip1}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}  is_close_scan_plugin_mesg=${true}
     Create An New Project    project_dest${d}
     Push Image    ${ip}    ${HARBOR_ADMIN}    ${HARBOR_PASSWORD}    project${d}    centos
-    Sleep  10
+    Image Should Be Replicated To Project  project_dest${d}  centos
-    Go Into Project    project_dest${d}
-    Switch To Project Repo
-    Retry Wait Until Page Contains    project_dest${d}/centos
     Close Browser
 
 Test Case - Replication Of Pull Images from AWS-ECR To Self
@@ -262,15 +252,9 @@ Test Case - Replication Of Pull Images from AWS-ECR To Self
     Switch To Replication Manage
     Create A Rule With Existing Endpoint    rule${d}    pull    a/*    image    e${d}    project${d}
     Select Rule And Replicate  rule${d}
-    Sleep    60
+    Image Should Be Replicated To Project  project${d}  httpd
-    Go Into Project    project${d}
+    Image Should Be Replicated To Project  project${d}  alpine
-    Switch To Project Repo
+    Image Should Be Replicated To Project  project${d}  hello-world
-    #In AWS-ECR, under repository a, there're only several images: httpd,alpine,hello-world.
-    Retry Wait Until Page Contains    project${d}/httpd
-    Go Into Project    project${d}
-    Switch To Project Repo
-    Retry Wait Until Page Contains    project${d}/alpine
-    Retry Wait Until Page Contains    project${d}/hello-world
     Close Browser
 
 Test Case - Replication Of Pull Images from Google-GCR To Self
@@ -285,12 +269,6 @@ Test Case - Replication Of Pull Images from Google-GCR To Self
     Create A Rule With Existing Endpoint    rule${d}    pull    eminent-nation-87317/*    image    e${d}    project${d}
     Filter Replicatin Rule  rule${d}
     Select Rule And Replicate  rule${d}
-    Sleep    30
+    Image Should Be Replicated To Project  project${d}  httpd
-    Go Into Project    project${d}
+    Image Should Be Replicated To Project  project${d}  tomcat
-    Switch To Project Repo
-    #In Google-GCR, under repository a, there're only several images: httpd,tomcat.
-    Retry Wait Until Page Contains    project${d}/httpd
-    Go Into Project    project${d}
-    Switch To Project Repo
-    Retry Wait Until Page Contains    project${d}/tomcat
     Close Browser

tests/robot-cases/Group3-Upgrade/data.json

@@ -31,24 +31,53 @@
               "url":"http://url",
               "name":"endpoint1",
               "user":"admin",
-              "pass":"Harbor12345"
+              "pass":"Harbor12345",
-           }
+              "insecure":true,
+              "type":"harbor"
+           },
+           {
+               "url":"https://hub.docker.com",
+               "name":"endpoint2",
+               "user":"",
+               "pass":"",
+               "insecure":false,
+               "type":"docker-hub"
+            }
         ],
         "replicationrule":[
            {
               "project":"project1",
               "endpoint":"endpoint1",
               "trigger":"Manual",
-              "rulename":"rulename"
+              "rulename":"rulename",
+              "is_src_registry":false,
+              "dest_namespace":"rule1-namespace",
+              "trigger_type":"event_based",
+              "cron":"",
+              "deletion":true,
+              "enabled":true,
+              "override":true,
+              "name_filters":"namefilter1",
+              "tag_filters":"tagfilter1"
            },
            {
               "project":"library",
-              "endpoint":"endpoint1",
+              "endpoint":"endpoint2",
               "trigger":"Manual",
-              "rulename":"rulelibrary"
+              "rulename":"rulelibrary",
-           }
+              "endpoint_type":"harbor",
-        ],
+              "is_src_registry":true,
-        "configuration":{
+              "dest_namespace":"rule2-namespace",
+              "trigger_type":"scheduled",
+              "cron":"6 7 8 * * *",
+              "deletion":false,
+              "enabled":true,
+              "override":true,
+              "name_filters":"namefilter2",
+              "tag_filters":"tagfilter2"
+            }
+         ],
+         "configuration":{
            "authmode":"Database",
            "projectcreation":"everyone",
            "selfreg":false,
@@ -68,6 +97,25 @@
                  "daily":0
               }
            },
+           "deployment_security":{
+               "cve":[
+                  {
+                     "id":"CVE-2019-12904"
+                     },
+                  {
+                     "id":"CVE-2011-3389"
+                  },
+                  {
+                     "id":"CVE-2018-12886"
+                  },
+                  {
+                     "id":"CVE-2019-3844"
+                  }
+               ],
+              "never_expires":false,
+              "expires_at":"2572969600"
+           },
+           "webhooks_enabled":true,
            "syslabel":[
               {
                  "name":"label1"
@@ -121,6 +169,10 @@
                     "role":3
                  }
               ],
+              "count_limit":1234,
+              "storage_limit":53687091200,
+              "storage_limit_for_verify":50,
+              "storage_unit_for_verify":"GB",
               "replications":{
                  "rulename":"ruleproject1",
                  "endpointname":"endpoint1",
@@ -143,12 +195,50 @@
                     "operation":"create"
                  }
               ],
+              "robot_account":[
+                  {
+                     "name":"robot0",
+                     "access": [
+                        {
+                        "action": "pull"
+                        },
+                        {
+                        "action": "push"
+                        }
+                     ]
+                  }
+               ],
+               "webhook":{
+                  "address":"https://1.1.1.1",
+                  "skip_cert_verify":true,
+                  "auth_header":"aaa",
+                  "enabled":true
+               },
               "configuration":{
                  "public":"true",
                  "enable_content_trust":"true",
-                 "automatically_scan_images_on_push":"true",
+                 "auto_scan":"true",
-                 "prevent_vulnerable_images_from_running":"true",
+                 "prevent_vul":"true",
-                 "prevent_vlunerable_images_from_running_severity":"high"
+                 "severity":"high",
+                 "reuse_sys_cve_whitelist":"false",
+                 "deployment_security":{
+                  "cve":[
+                     {
+                        "id":"CVE-2019-111"
+                        },
+                     {
+                        "id":"CVE-2011-2222"
+                     },
+                     {
+                        "id":"CVE-2018-33333"
+                     },
+                     {
+                        "id":"CVE-2019-44444"
+                     }
+                  ],
+                 "never_expires":false,
+                 "expires_at":"2572969600"
+               }
               }
            },
            {
@@ -182,6 +272,10 @@
                     "role":3
                  }
               ],
+              "count_limit":-1,
+              "storage_limit":32985348833280,
+              "storage_limit_for_verify":30,
+              "storage_unit_for_verify":"TB",
               "replications":{
                  "rulename":"rulename1",
                  "endpointname":"endpoint1",
@@ -204,12 +298,52 @@
                     "operation":"create"
                  }
               ],
+              "robot_account":[
+                  {
+                     "name":"robot1",
+                     "access": [
+                        {
+                          "action": "pull"
+                        },
+                        {
+                          "action": "push"
+                        }
+                      ]
+                  },
+                  {
+                     "name":"robot2",
+                     "access": [
+                        {
+                          "action": "pull"
+                        }
+                      ]
+                  },
+                  {
+                     "name":"robot3",
+                     "access": [
+                        {
+                          "action": "push"
+                        }
+                      ]
+                  }
+               ],
+               "webhook":{
+                  "address":"https://1.1.1.1",
+                  "skip_cert_verify":true,
+                  "auth_header":"aaa",
+                  "enabled":true
+               },
               "configuration":{
                  "public":"false",
                  "enable_content_trust":"false",
-                 "automatically_scan_images_on_push":"false",
+                 "auto_scan":"false",
-                 "prevent_vulnerable_images_from_running":"true",
+                 "prevent_vul":"true",
-                 "prevent_vlunerable_images_from_running_severity":"medium"
+                 "severity":"medium",
+                 "reuse_sys_cve_whitelist":"true",
+                 "deployment_security":{
+                  "cve":[
+                  ]
+               }
               }
            }
         ]

tests/robot-cases/Group3-Upgrade/feature_map.json

@@ -0,0 +1,192 @@
+{
+    "create_project":[
+        {
+            "branch":1,
+            "version":"1.4"
+        },
+        {
+            "branch":1,
+            "version":"1.6"
+        },
+        {
+            "branch":1,
+            "version":"1.7"
+        },
+        {
+            "branch":1,
+            "version":"1.8"
+        },
+        {
+            "branch":2,
+            "version":"1.9"
+        },
+        {
+            "branch":2,
+            "version":"1.10"
+        }
+    ],
+    "add_member":[
+        {
+            "branch":1,
+            "version":"1.4"
+        },
+        {
+            "branch":2,
+            "version":"1.6"
+        },
+        {
+            "branch":2,
+            "version":"1.7"
+        },
+        {
+            "branch":2,
+            "version":"1.8"
+        },
+        {
+            "branch":2,
+            "version":"1.9"
+        },
+        {
+            "branch":2,
+            "version":"1.10"
+        }
+    ],
+    "set_user_admin":[
+        {
+            "branch":1,
+            "version":"1.4"
+        },
+        {
+            "branch":2,
+            "version":"1.6"
+        },
+        {
+            "branch":2,
+            "version":"1.7"
+        },
+        {
+            "branch":2,
+            "version":"1.8"
+        },
+        {
+            "branch":2,
+            "version":"1.9"
+        },
+        {
+            "branch":2,
+            "version":"1.10"
+        }
+    ],
+    "add_endpoint":[
+        {
+            "branch":1,
+            "version":"1.4"
+        },
+        {
+            "branch":1,
+            "version":"1.6"
+        },
+        {
+            "branch":1,
+            "version":"1.7"
+        },
+        {
+            "branch":2,
+            "version":"1.8"
+        },
+        {
+            "branch":2,
+            "version":"1.9"
+        },
+        {
+            "branch":2,
+            "version":"1.10"
+        }
+    ],
+    "add_replication_rule":[
+        {
+            "branch":1,
+            "version":"1.4"
+        },
+        {
+            "branch":1,
+            "version":"1.6"
+        },
+        {
+            "branch":1,
+            "version":"1.7"
+        },
+        {
+            "branch":2,
+            "version":"1.8"
+        },
+        {
+            "branch":2,
+            "version":"1.9"
+        },
+        {
+            "branch":2,
+            "version":"1.10"
+        }
+    ],
+    "add_sys_whitelist":[
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        }
+    ],
+    "update_project_setting_whitelist":[
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        }
+    ],
+    "add_project_robot_account":[
+        {
+            "branch":1,
+            "version":"1.8"
+        },
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        }
+    ],
+    "add_tag_retention_rule":[
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        }
+    ],
+    "add_tag_immutability_rule":[
+        {
+            "branch":1,
+            "version":"1.10"
+        }
+    ],
+    "add_webhook":[
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        }
+    ]
+ }

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -1,8 +1,9 @@
 import os
+import sys
 import json
 import argparse
 import requests
-
+from functools import wraps
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
@@ -15,9 +16,43 @@ url = "https://"+args.endpoint+"/api/"
 endpoint_url = "https://"+args.endpoint
 print url
 
+with open("feature_map.json") as f:
+    feature_map = json.load(f)
+
+def get_branch(func_name, version):
+    has_feature = False
+    for node in feature_map[func_name]:
+        has_feature = True
+        if node["version"] == version:
+            return node["branch"]
+    if has_feature is False:
+        return "No Restriction"
+    else:
+        return "Not Supported"
+
+def get_feature_branch(func):
+    @wraps(func)
+    def inner_func(*args,**kwargs):
+        branch=get_branch(inner_func.__name__, kwargs.get("version"))
+        if branch == "No Restriction":
+            func(*args,**kwargs)
+        elif branch == "Not Supported":
+            print("Feature {} is not supported in version {}".format(inner_func.__name__, kwargs.get("version")))
+        else:
+            kwargs["branch"] = branch
+            func(*args,**kwargs)
+        return
+    return inner_func
+
 class HarborAPI:
-    def create_project(self, project_name):
+    @get_feature_branch
-        body=dict(body={"project_name": ""+project_name+"", "metadata": {"public": "true"}})
+    def create_project(self, project, **kwargs):
+        if kwargs["branch"] == 1:
+            body=dict(body={"project_name": ""+project["name"]+"", "metadata": {"public": "true"}})
+        elif kwargs["branch"] == 2:
+            body=dict(body={"project_name": ""+project["name"]+"", "metadata": {"public": "true"},"count_limit":project["count_limit"],"storage_limit":project["storage_limit"]})
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
         request(url+"projects", 'post', **body)
 
     def create_user(self, username):
@@ -25,65 +60,135 @@ class HarborAPI:
         body=dict(body=payload)
         request(url+"users", 'post', **body)
 
-    def set_user_admin(self, user):
+    @get_feature_branch
+    def set_user_admin(self, user, **kwargs):
         r = request(url+"users?username="+user+"", 'get')
         userid = str(r.json()[0]['user_id'])
-        if args.version == "1.6":
+
+        if kwargs["branch"] == 1:
+            body=dict(body={"has_admin_role": 1})
+        elif kwargs["branch"] == 2:
             body=dict(body={"has_admin_role": True})
         else:
-            body=dict(body={"has_admin_role": 1})
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
         request(url+"users/"+userid+"/sysadmin", 'put', **body)
 
-    def add_member(self, project, user, role):
+    @get_feature_branch
+    def add_member(self, project, user, role, **kwargs):
         r = request(url+"projects?name="+project+"", 'get')
         projectid = str(r.json()[0]['project_id'])
-        if args.version == "1.6":
+
+        if kwargs["branch"] == 1:
+            payload = {"roles": [role], "username":""+user+""}
+        elif kwargs["branch"] == 2:
             payload = {"member_user":{ "username": ""+user+""},"role_id": role}
         else:
-            payload = {"roles": [role], "username":""+user+""}
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
-
         body=dict(body=payload)
         request(url+"projects/"+projectid+"/members", 'post', **body)
 
-    def add_endpoint(self, endpointurl, endpointname, username, password, insecure):
+    @get_feature_branch
-        payload = {
+    def add_endpoint(self, endpointurl, endpointname, username, password, insecure, registry_type, **kwargs):
-            "credential":{
+        if kwargs["branch"] == 1:
-                "access_key":""+username+"",
+            payload = {"endpoint": ""+endpointurl+"", "name": ""+endpointname+"", "username": ""+username+"", "password": ""+password+"", "insecure": insecure}
-                "access_secret":""+password+"",
+            body=dict(body=payload)
-                "type":"basic"
+            request(url+"targets", 'post', **body)
-            },
+        elif kwargs["branch"] == 2:
-            "insecure":insecure,
+            if registry_type == "harbor":
-            "name":""+endpointname+"",
+                endpointurl = endpoint_url
-            "type":"harbor",
+            payload = {
-            "url":""+endpoint_url+""
+                "credential":{
-        }
+                    "access_key":""+username+"",
-        body=dict(body=payload)
+                    "access_secret":""+password+"",
-        print  body
+                    "type":"basic"
-        request(url+"/registries", 'post', **body)
+                },
+                "insecure":insecure,
+                "name":""+endpointname+"",
+                "type":""+registry_type+"",
+                "url":""+endpointurl+""
+            }
+            body=dict(body=payload)
+            print  body
+            request(url+"/registries", 'post', **body)
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
 
-    def add_replication_rule(self, project, target, trigger, rulename):
+    @get_feature_branch
-        r = request(url+"registries?name="+target+"", 'get')
+    def add_replication_rule(self, replicationrule, **kwargs):
-        targetid = r.json()[0]['id']
+        if kwargs["branch"] == 1:
-        payload = {"name": ""+rulename+"", "deletion": False, "enabled": True, "description": "string", "dest_registry": {"id": targetid},"trigger": {"type": "manual"}}
+            r = request(url+"projects?name="+replicationrule["project"]+"", 'get')
-        body=dict(body=payload)
+            projectid = r.json()[0]['project_id']
-        request(url+"replication/policies", 'post', **body)
+            r = request(url+"targets?name="+replicationrule["endpoint"]+"", 'get')
+            targetid = r.json()[0]['id']
+            payload = {"name": ""+replicationrule["rulename"]+"", "description": "string", "projects": [{"project_id": projectid,}], "targets": [{"id": targetid,}], "trigger": {"kind": ""+replicationrule["trigger"]+"", "schedule_param": {"type": "weekly", "weekday": 1, "offtime": 0}}}
+            body=dict(body=payload)
+            request(url+"policies/replication", 'post', **body)
+        elif kwargs["branch"] == 2:
+            r = request(url+"registries?name="+replicationrule["endpoint"]+"", 'get')
+            targetid = r.json()[0]['id']
+            if replicationrule["is_src_registry"] is True:
+                registry = r'"src_registry": { "id": '+str(targetid)+r'},'
+            else:
+                registry = r'"dest_registry": { "id": '+str(targetid)+r'},'
 
-    def update_project_setting(self, project, public, contenttrust, preventrunning, preventseverity, scanonpush):
+            body=dict(body=json.loads(r'{"name":"'+replicationrule["rulename"].encode('utf-8')+r'","dest_namespace":"'+replicationrule["dest_namespace"].encode('utf-8')+r'","deletion": '+str(replicationrule["deletion"]).lower()+r',"enabled": '+str(replicationrule["enabled"]).lower()+r',"override": '+str(replicationrule["override"]).lower()+r',"description": "string",'+ registry + r'"trigger":{"type": "'+replicationrule["trigger_type"]+r'", "trigger_settings":{"cron":"'+replicationrule["cron"]+r'"}},"filters":[ {"type":"name","value":"'+replicationrule["name_filters"]+r'"},{"type":"tag","value":"'+replicationrule["tag_filters"]+r'"}]}'))
+            print body
+            request(url+"replication/policies", 'post', **body)
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
+
+    #@get_feature_branch
+    def update_project_setting_metadata(self, project, public, contenttrust, preventrunning, preventseverity, scanonpush):
         r = request(url+"projects?name="+project+"", 'get')
         projectid = str(r.json()[0]['project_id'])
         payload = {
-            "project_name": ""+project+"",
             "metadata": {
                 "public": public,
                 "enable_content_trust": contenttrust,
-                "prevent_vulnerable_images_from_running": preventrunning,
+                "prevent_vul": preventrunning,
-                "prevent_vulnerable_images_from_running_severity": preventseverity,
+                "severity": preventseverity,
-                "automatically_scan_images_on_push": scanonpush
+                "auto_scan": scanonpush
             }
         }
         body=dict(body=payload)
+        print body
         request(url+"projects/"+projectid+"", 'put', **body)
 
+    @get_feature_branch
+    def add_sys_whitelist(self, cve_id_list, **kwargs):
+        cve_id_str = ""
+        if kwargs["branch"] == 1:
+            for index, cve_id in enumerate(cve_id_list["cve"]):
+                cve_id_str = cve_id_str + '{"cve_id":"' +cve_id["id"] + '"}'
+                if index != len(cve_id_list["cve"]) - 1:
+                    cve_id_str = cve_id_str + ","
+            body=dict(body=json.loads(r'{"items":['+cve_id_str.encode('utf-8')+r'],"expires_at":'+cve_id_list["expires_at"]+'}'))
+            request(url+"system/CVEWhitelist", 'put', **body)
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
+
+    @get_feature_branch
+    def update_project_setting_whitelist(self, project, reuse_sys_cve_whitelist, cve_id_list, **kwargs):
+        r = request(url+"projects?name="+project+"", 'get')
+        projectid = str(r.json()[0]['project_id'])
+        cve_id_str = ""
+        if kwargs["branch"] == 1:
+            for index, cve_id in enumerate(cve_id_list["cve"]):
+                cve_id_str = cve_id_str + '{"cve_id":"' +cve_id["id"] + '"}'
+                if index != len(cve_id_list["cve"]) - 1:
+                    cve_id_str = cve_id_str + ","
+            print cve_id_str
+            if reuse_sys_cve_whitelist == "true":
+                payload = r'{"metadata":{"reuse_sys_cve_whitelist":"true"}}'
+            else:
+                payload = r'{"metadata":{"reuse_sys_cve_whitelist":"false"},"cve_whitelist":{"project_id":'+projectid+',"items":['+cve_id_str.encode('utf-8')+r'],"expires_at":'+cve_id_list["expires_at"]+'}}'
+            print payload
+            body=dict(body=json.loads(payload))
+            request(url+"projects/"+projectid+"", 'put', **body)
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
+
+
     def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token):
         payload = {
             "auth_mode": "db_auth",
@@ -108,6 +213,78 @@ class HarborAPI:
         body=dict(body=payload)
         request(url+"configurations", 'put', **body)
 
+    @get_feature_branch
+    def add_project_robot_account(self, project, robot_account, **kwargs):
+        r = request(url+"projects?name="+project+"", 'get')
+        projectid = str(r.json()[0]['project_id'])
+
+        if kwargs["branch"] == 1:
+            if len(robot_account["access"]) == 1:
+                robot_account_ac = robot_account["access"][0]
+                payload = {
+                    "name": robot_account["name"],
+                    "access": [
+                        {
+                            "resource": "/project/"+projectid+"/repository",
+                            "action": robot_account_ac["action"]
+                        }
+                    ]
+                }
+            elif len(robot_account["access"]) == 2:
+                payload = {
+                    "name": robot_account["name"],
+                    "access": [
+                        {
+                            "resource": "/project/"+projectid+"/repository",
+                            "action": "pull"
+                        },
+                        {
+                            "resource": "/project/"+projectid+"/repository",
+                            "action": "push"
+                        }
+                    ]
+                }
+            else:
+                raise Exception(r"Error: Robot account count {} is not legal!".format(len(robot_account["access"])))
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
+        print payload
+        body=dict(body=payload)
+        request(url+"projects/"+projectid+"/robots", 'post', **body)
+
+    @get_feature_branch
+    def add_tag_retention_rule(self, project, robot_account, **kwargs):
+        return
+
+    @get_feature_branch
+    def add_webhook(self, webhook, **kwargs):
+        if kwargs["branch"] == 1:
+            payload = {
+                "targets":[
+                    {
+                        "type":"http",
+                        "address":webhook["address"],
+                        "skip_cert_verify":webhook["skip_cert_verify"],
+                        "auth_header":webhook["auth_header"]
+                    }
+                ],
+                "event_types":[
+                    "downloadChart",
+                    "deleteChart",
+                    "uploadChart",
+                    "deleteImage",
+                    "pullImage",
+                    "pushImage",
+                    "scanningFailed",
+                    "scanningCompleted"
+                ],
+                "enabled":+webhook["enabled"]
+            }
+            body=dict(body=payload)
+            request(url+"system/CVEWhitelist", 'put', **body)
+        else:
+            raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"]))
+
     def update_repoinfo(self, reponame):
         payload = {"description": "testdescription"}
         body=dict(body=payload)
@@ -125,6 +302,7 @@ class HarborAPI:
             try:
                 os.makedirs(ca_path)
             except Exception, e:
+                print str(e)
                 pass
         open(target, 'wb').write(ca_content)
 
@@ -155,7 +333,7 @@ def pull_image(*image):
 
 def push_image(image, project):
     os.system("docker tag "+image+" "+args.endpoint+"/"+project+"/"+image)
-    os.system("docker login "+args.endpoint+" -u Admin"+" -p Harbor12345")
+    os.system("docker login "+args.endpoint+" -u admin"+" -p Harbor12345")
     os.system("docker push "+args.endpoint+"/"+project+"/"+image)
 
 def push_signed_image(image, project, tag):
@@ -169,30 +347,39 @@ def do_data_creation():
         harborAPI.create_user(user["name"])
 
     for user in data["admin"]:
-        harborAPI.set_user_admin(user["name"])
+        harborAPI.set_user_admin(user["name"], version=args.version)
 
     for project in data["projects"]:
-        harborAPI.create_project(project["name"])
+        harborAPI.create_project(project, version=args.version)
         for member in project["member"]:
-            harborAPI.add_member(project["name"], member["name"], member["role"])
+            harborAPI.add_member(project["name"], member["name"], member["role"], version=args.version)
+        for robot_account in project["robot_account"]:
+            harborAPI.add_project_robot_account(project["name"], robot_account, version=args.version)
+        harborAPI.add_webhook(project["webhook"], version=args.version)
 
     pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2")
     push_image("busybox", data["projects"][0]["name"])
     push_signed_image("alpine", data["projects"][0]["name"], "latest")
 
     for endpoint in data["endpoint"]:
-        harborAPI.add_endpoint(endpoint["url"], endpoint["name"], endpoint["user"], endpoint["pass"], True)
+        harborAPI.add_endpoint(endpoint["url"], endpoint["name"], endpoint["user"], endpoint["pass"], endpoint["insecure"], endpoint["type"], version=args.version)
+
     for replicationrule in data["replicationrule"]:
-        harborAPI.add_replication_rule(replicationrule["project"],
+        harborAPI.add_replication_rule(replicationrule, version=args.version)
-                                       replicationrule["endpoint"], replicationrule["trigger"],
+
-                                       replicationrule["rulename"])
     for project in data["projects"]:
-        harborAPI.update_project_setting(project["name"],
+        harborAPI.update_project_setting_metadata(project["name"],
                                         project["configuration"]["public"],
                                         project["configuration"]["enable_content_trust"],
-                                        project["configuration"]["prevent_vulnerable_images_from_running"],
+                                        project["configuration"]["prevent_vul"],
-                                        project["configuration"]["prevent_vlunerable_images_from_running_severity"],
+                                        project["configuration"]["severity"],
-                                        project["configuration"]["automatically_scan_images_on_push"])
+                                        project["configuration"]["auto_scan"])
+
+    for project in data["projects"]:
+        harborAPI.update_project_setting_whitelist(project["name"],
+                                    project["configuration"]["reuse_sys_cve_whitelist"],
+                                    project["configuration"]["deployment_security"],version=args.version)
+
     harborAPI.update_systemsetting(data["configuration"]["emailsetting"]["emailfrom"],
                                    data["configuration"]["emailsetting"]["emailserver"],
                                    float(data["configuration"]["emailsetting"]["emailport"]),
@@ -200,4 +387,7 @@ def do_data_creation():
                                    data["configuration"]["projectcreation"],
                                    data["configuration"]["selfreg"],
                                    float(data["configuration"]["token"]))
+
+    harborAPI.add_sys_whitelist(data["configuration"]["deployment_security"],version=args.version)
+
 do_data_creation()

tests/robot-cases/Group3-Upgrade/verify.robot

@@ -9,13 +9,41 @@ ${HARBOR_ADMIN}  admin
 
 *** Test Cases ***
 Test Case - Upgrade Verify
+    [Tags]  1.8-latest
+    ${data}=  Load Json From File  ${CURDIR}${/}data.json
+    Log To Console  "Verify User..."
+    Run Keyword  Verify User  ${data}
+    Log To Console  "Verify Project..."
+    Run Keyword  Verify Project  ${data}
+    Log To Console  "Verify Member Exist..."
+    Run Keyword  Verify Member Exist  ${data}
+    #Run Keyword  Verify Robot Account Exist  ${data}
+    Log To Console  "Verify User System Admin Role..."
+    Run Keyword  Verify User System Admin Role  ${data}
+    Log To Console  "Verify Endpoint..."
+    Run Keyword  Verify Endpoint  ${data}
+    Log To Console  "Verify Replicationrule..."
+    Run Keyword  Verify Replicationrule  ${data}
+    Log To Console  "Verify Project Setting..."
+    Run Keyword  Verify Project Setting  ${data}
+    Log To Console  "Verify System Setting..."
+    Run Keyword  Verify System Setting  ${data}
+    Log To Console  "Verify Image Tag..."
+    Run Keyword  Verify Image Tag  ${data}
+
+Test Case - Upgrade Verify
+    [Tags]  1.9-latest
     ${data}=  Load Json From File  ${CURDIR}${/}data.json
     Run Keyword  Verify User  ${data}
     Run Keyword  Verify Project  ${data}
     Run Keyword  Verify Member Exist  ${data}
+    #Run Keyword  Verify Robot Account Exist  ${data}
+    #Run Keyword  Verify Project-level Whitelist  ${data}
+    #Run Keyword  Verify Webhook  ${data}
     Run Keyword  Verify User System Admin Role  ${data}
     Run Keyword  Verify Endpoint  ${data}
     Run Keyword  Verify Replicationrule  ${data}
     Run Keyword  Verify Project Setting  ${data}
     Run Keyword  Verify System Setting  ${data}
+    #Run Keyword  Verify System Setting Whitelist  ${data}
     Run Keyword  Verify Image Tag  ${data}