Add vulneraibility case and add a testcase doc. (#4378)

Add disable schedule 
vulnerability data not ready 
can as an unprivileged user 
scan image with empty vul 
manual scan all
project level image serverity policy 
scan on push

tests/resources/Harbor-Pages/Project.robot

@@ -195,13 +195,6 @@ Expand Repo
     Click Element  //repository//clr-dg-row[contains(.,'${projectname}')]//button/clr-icon
     Sleep  1
 
-Scan Repo
-    [Arguments]  ${tagname}
-    #select one tag
-    Click Element  //clr-dg-row[contains(.,"${tagname}")]//label
-    Click Element  //button[contains(.,'Scan')]
-    Sleep  15
-
 Edit Repo Info
     Click Element  //*[@id="repo-info"]
     Sleep  1
@@ -222,6 +215,4 @@ Edit Repo Info
     Page Should Contain  test_description_info
     Capture Page Screenshot  RepoInfo.png
 
-Summary Chart Should Display
+
-    [Arguments]  ${tagname}
-    Page Should Contain Element  //clr-dg-row[contains(.,'${tagname}')]//hbr-vulnerability-bar//hbr-vulnerability-summary-chart

tests/resources/Harbor-Pages/ToolKit.robot

@@ -33,14 +33,16 @@ Partly Success
     Sleep  1
 
 Filter Object
+#Filter project repo user tag.
     [Arguments]    ${kw}
     Click Element  xpath=//hbr-filter//clr-icon
     Input Text   xpath=//hbr-filter//input  ${kw}
     Sleep  1
 
 Select Object
+#select single element such as user project repo tag
     [Arguments]    ${obj}
-    Click Element  //clr-dg-cell[contains(.,'${obj}')]//label
+    Click Element  //clr-dg-row[contains(.,'${obj}')]//label
 
 Multi-delete Object
     [Arguments]    @{obj}

tests/resources/Harbor-Pages/Vulnerability.robot

@@ -0,0 +1,64 @@
+*** Settings ***
+Documentation  This resource provides any keywords related to the Harbor private registry appliance
+Resource  ../../resources/Util.robot
+
+*** Variables ***
+ ${HARBOR_VERSION}  v1.1.1
+ 
+*** Keywords ***
+Disable Scan Schedule
+     Click Element  //vulnerability-config//select[@id="scanAllPolicy"]
+     Click Element  //vulnerability-config//select[@id="scanAllPolicy"]//option[contains(.,'None')]
+     Click Element  //button[contains(.,'SAVE')]
+
+Go To Vulnerability Config
+    Click Element  //config//button[contains(.,'Vulnerability')]
+
+Trigger Scan Now
+    Click Element  //config//button[contains(.,'NOW')]
+    Sleep  10
+
+Set Vulnerabilty Serverity
+#0 is high 1 is medium 2 is low 3 is negligible
+    [Arguments]  ${level}
+    Goto Project Config
+    #enable first
+    Click Element  //project-detail//clr-checkbox[@name="prevent-vulenrability-image"]//label
+    Checkbox Should Be Selected  //project-detail//clr-checkbox//input[@name="prevent-vulenrability-image"]
+    Click Element  //project-detail//select
+    #wait for dropdown popup
+    Sleep  1
+    Select From List By Index  //project-detail//select  ${level}
+    Click Element  //hbr-project-policy-config//button[contains(.,'SAVE')]
+
+Scan Is Disabled
+    Page Should Contain Element  //button[contains(.,'Scan') and @disabled='']
+
+Move To Summary Chart
+    Mouse Over  //hbr-vulnerability-summary-chart
+    Sleep  1
+
+Scan Repo
+#use fail for image clair can not scan, otherwise use success
+    [Arguments]  ${tagname}  ${status}
+    #select one tag
+    Click Element  //clr-dg-row[contains(.,"${tagname}")]//label
+    Click Element  //button[contains(.,'Scan')]
+    Run Keyword If  '${status}' == 'Succeed'  Wait Until Page Contains Element  //hbr-vulnerability-bar//hbr-vulnerability-summary-chart
+    Run Keyword If  '${status}' == 'Fail'  Wait Until Page Contains Element  //hbr-vulnerability-bar//a
+
+
+Summary Chart Should Display
+    [Arguments]  ${tagname}
+    Page Should Contain Element  //clr-dg-row[contains(.,'${tagname}')]//hbr-vulnerability-bar//hbr-vulnerability-summary-chart
+
+Enable Scan On Push
+    Checkbox Should Not Be Selected  //clr-checkbox[@name="scan-image-on-push"]//input
+    Click Element  //clr-checkbox[@name="scan-image-on-push"]//label
+    Checkbox Should Be Selected  //clr-checkbox[@name="scan-image-on-push"]//input
+    Click Element  //hbr-project-policy-config//button[contains(.,'SAVE')]
+
+Vulnerability Not Ready Project Hint
+    Page Should Contain Element  //span[@class="db-status-warning"]
+Vulnerability Not Ready Config Hint
+    Page Should Contain Element  //vulnerability-config//clr-tooltip

tests/resources/Util.robot

@@ -43,6 +43,7 @@ Resource  Harbor-Pages/Administration-Users_Elements.robot
 Resource  Harbor-Pages/Configuration.robot
 Resource  Harbor-Pages/Configuration_Elements.robot
 Resource  Harbor-Pages/ToolKit.robot
+Resource  Harbor-Pages/Vulnerability.robot
 Resource  Docker-Util.robot
 Resource  Admiral-Util.robot
 Resource  OVA-Util.robot

tests/robot-cases/Group0-BAT/BAT.robot

@@ -42,7 +42,7 @@ Test Case - Clair Basic Verfication
     Push Image  ${ip}  tester${d}  Test1@34  project${d}  hello-world
     Go Into Project  project${d}
     Go Into Repo  project${d}/hello-world
-    Scan Repo  latest
+    Scan Repo  latest  Succeed
     Summary Chart Should Display  latest
     #Edit Repo Info
     Close Browser
@@ -71,4 +71,4 @@ Test Case - Ldap Basic Verfication
     Init LDAP
     Logout Harbor
     Sign In Harbor  ${HARBOR_URL}  mike  zhu88jie
-    Close Browser
+    Close Browser

tests/robot-cases/Group11-Nightly/Nightly.robot

@@ -25,6 +25,16 @@ ${SSH_USER}  root
 ${HARBOR_ADMIN}  admin
 
 *** Test Cases ***
+Test Case - Vulnerability Data Not Ready
+#This case must run before vulnerability db ready
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Vulnerability Not Ready Project Hint
+    Switch To Configure
+    Go To Vulnerability Config
+    Vulnerability Not Ready Config Hint
+
 Test Case - Create An New User
     Init Chrome Driver    
     ${d}=    Get Current Date    result_format=%m%s
@@ -261,11 +271,86 @@ Test Case - Scan A Tag In The Repo
     Push Image  ${ip}  tester${d}  Test1@34  project${d}  hello-world
     Go Into Project  project${d}
     Go Into Repo  project${d}/hello-world
-    Scan Repo  latest
+    Scan Repo  latest  Succeed
     Summary Chart Should Display  latest
+    Pull Image  ${ip}  tester${d}  Test1@34  project${d}  hello-world
     # Edit Repo Info
     Close Browser
 
+Test Case - Scan As An Unprivileged User
+    Init Chrome Driver
+    ${d}=    get current date    result_format=%m%s
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  hello-world
+    Create An New User  ${HARBOR_URL}  user${d}  user${d}@vmware.com  user${d}  Test1@34  harbor
+    Go Into Project  library
+    Go Into Repo  hello-world
+    Select Object  latest
+    Scan Is Disabled
+    Close Browser
+##
+Test Case - Scan Image With Empty Vul
+    Init Chrome Driver
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  hello-world
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Go Into Repo  hello-world
+    Scan Repo  latest  Succeed
+    Move To Summary Chart
+    Wait Until Page Contains  Unknow
+    Close Browser
+###
+Test Case - Disable Scan Schedule
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Configure
+    Go To Vulnerability Config
+    Disable Scan Schedule
+    Logout Harbor
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Configure
+    Go To Vulnerability Config
+    Page Should Contain  None
+    Close Browser
+###
+Test Case - Manual Scan All
+    Init Chrome Driver
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  redis
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Configure
+    Go To Vulnerability Config
+    Trigger Scan Now
+    Back To Projects
+    Go Into Project  library
+    Go Into Repo  redis
+    Summary Chart Should Display  latest
+    Close Browser
+#
+Test Case - Project Level Image Serverity Policy 
+    Init Chrome Driver
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  haproxy
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Go Into Repo  haproxy
+    Scan Repo  latest  Succeed
+    Back To Projects
+    Go Into Project  library
+    Set Vulnerabilty Serverity  0
+    Cannot pull image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  haproxy
+    Close Browser
+
+Test Case - Scan Image On Push
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Goto Project Config
+    Enable Scan On Push
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  memcached 
+    Back To Projects
+    Go Into Project  library
+    Go Into Repo  memcached
+    Summary Chart Should Display  latest
+    Close Browser
+
 Test Case - Manage Project Member
     Init Chrome Driver
     ${d}=    Get current Date  result_format=%m%s
@@ -421,7 +506,7 @@ Test Case - View Scan Results
     Push Image  ${ip}  tester${d}  Test1@34  project${d}  tomcat
     Go Into Project  project${d}
     Go Into Repo  project${d}/tomcat
-    Scan Repo  latest
+    Scan Repo  latest  Succeed
     Summary Chart Should Display  latest
     View Repo Scan Details
     Close Browser
@@ -433,7 +518,7 @@ Test Case - View Scan Error
     Push Image  ${ip}  tester${d}  Test1@34  project${d}  vmware/photon:1.0
     Go Into Project  project${d}
     Go Into Repo  project${d}/vmware/photon
-    Scan Repo  1.0
+    Scan Repo  1.0  Fail
     View Scan Error Log
     Close Browser
 

tests/testcases/Group10-Vulnerability/Test-10-12-Scan-image-on-push.md

@@ -0,0 +1,25 @@
+Test 10-12  Scan Image on Push   
+=======
+  
+# Purpose:    
+To verify that clair can automatic scan image when image is pushed.  
+
+# References:    
+User guide  
+
+# Environment:  
+* This test requires that a Harbor instance is running and available.  
+* Harbor is installed with clair enable.  
+* A linux host with Docker CLI installed.  
+* Clair has been updated to the latest.    
+
+# Test Steps:  
+1. Login Harbor as admin.  
+2. Create a new project.  
+3. Goto project configuration page.  
+4. Enable automatically scan.  
+5. Push an image not scanned before to the project.  
+
+# Expected Outcome:    
+* Step5 image should be scaned automatically.  
+