Merge pull request #11857 from danfengliu/Enhance-verificaton-for-tag-retention-pytest

Enhance verificaton for tag retention pytest

tests/apitests/python/library/retention.py

@@ -110,7 +110,7 @@ class Retention(base.Base):
         _, status_code, _ = client.retentions_id_put_with_http_info(retention_id, policy)
         base._assert_status_code(expect_status_code, status_code)
 
-    def update_retention_add_rule(self, retention_id, selector_repository="**", selector_tag="**", expect_status_code = 200, **kwargs):
+    def update_retention_add_rule(self, retention_id, selector_repository="**", selector_tag="**", with_untag="True", expect_status_code = 200, **kwargs):
         client = self._get_client(**kwargs)
         policy, status_code, _ = client.retentions_id_get_with_http_info(retention_id)
         base._assert_status_code(200, status_code)
@@ -134,7 +134,7 @@ class Retention(base.Base):
                                                     {
                                                         "kind": "doublestar",
                                                         "decoration": "matches",
-                                                        "extras":'["untagged":True]',
+                                                        "extras":'["untagged":'+with_untag+']',
                                                         "pattern": selector_tag
                                                     }
                                                 ]

tests/apitests/python/test_retention.py

@@ -41,6 +41,7 @@ class TestProjects(unittest.TestCase):
         self.retention = Retention()
         self.artifact = Artifact()
         self.repo_name_1 = "test1"
+        self.repo_name_2 = "test2"
 
     def testTagRetention(self):
         user_ra_password = "Aa123456"
@@ -57,15 +58,21 @@ class TestProjects(unittest.TestCase):
         push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['1.0'])
         push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['2.0'])
         push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['3.0','latest'])
-        push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['1.0'])
+        push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_2, ['1.0'])
-        push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['latest'])
+        push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_2, ['latest'])
         push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test3", ['1.0'])
         push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test4", ['1.0'])
 
+        tag_data_artifact3_image1 = self.artifact.get_reference_info(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", **TestProjects.USER_RA_CLIENT)
+        print tag_data_artifact3_image1[0].digest
+
+        tag_data_artifact2_image2 = self.artifact.get_reference_info(TestProjects.project_src_repo_name, self.repo_name_2, "latest", **TestProjects.USER_RA_CLIENT)
+        print tag_data_artifact2_image2[0].digest
+
         tags = list_image_tags(harbor_server, TestProjects.project_src_repo_name+"/"+self.repo_name_1, user_ra_name, user_ra_password)
-        #Delete all tags of "artifact3" in repostory "image1";
+        #Delete all 2 tags of "artifact3" in repostory "image1";
-        self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "latest",**TestProjects.USER_RA_CLIENT)
+        self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "latest", **TestProjects.USER_RA_CLIENT)
-        self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "3.0",**TestProjects.USER_RA_CLIENT)
+        self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "3.0", **TestProjects.USER_RA_CLIENT)
         tags = list_image_tags(harbor_server, TestProjects.project_src_repo_name+"/"+self.repo_name_1, user_ra_name, user_ra_password)
 
         resp=self.repo.list_repositories(TestProjects.project_src_repo_name, **TestProjects.USER_RA_CLIENT)
@@ -104,18 +111,18 @@ class TestProjects(unittest.TestCase):
         self.assertEqual(len(resp), 4)
         resp=self.retention.get_retention_exec_task_log(retention_id,execution.id,resp[0].id, **TestProjects.USER_RA_CLIENT)
         print(resp)
-        # TODO As the repository isn't deleted when no tags left anymore
-        # TODO we should check the artifact/tag count here
-        # resp=self.repo.list_repositories(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT)
-        # self.assertEqual(len(resp), 3)
 
-        #List artifacts successfully;
+        #List artifacts successfully, and untagged artifact in test1 should be the only one retained;
-        artifacts = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_1, **TestProjects.USER_RA_CLIENT)
+        artifacts_1 = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_1, **TestProjects.USER_RA_CLIENT)
-        print artifacts
+        print artifacts_1[0].digest
-        # 'test1' has 3 artifacts, artifact1 with tag '1.0' and artifact2 with tag '2.0' should be deleted because they doesn't match 'latest'
+        self.assertTrue(len(artifacts_1)==1)
-        # artifact3 should be retained because it has no tag, so count of artifacts should be 1.
+        self.assertEqual(artifacts_1[0].digest, tag_data_artifact3_image1[0].digest)
-        # TODO: This verfication should be enhanced by verify sha256 at the same time;
+
-        self.assertTrue(len(artifacts)==1)
+        #List artifacts successfully, and artifact with latest tag in test2 should be the only one retained;
+        artifacts_2 = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_2, **TestProjects.USER_RA_CLIENT)
+        print artifacts_2[0].digest
+        self.assertTrue(len(artifacts_2)==1)
+        self.assertEqual(artifacts_2[0].digest, tag_data_artifact2_image2[0].digest)
 
     @classmethod
     def tearDownClass(self):

tests/resources/Docker-Util.robot

@@ -72,6 +72,7 @@ Cannot Pull Unsigned Image
     [Arguments]  ${ip}  ${user}  ${pass}  ${proj}  ${imagewithtag}
     Wait Unitl Command Success  docker login -u ${user} -p ${pass} ${ip}
     ${output}=  Command Should be Failed  docker pull ${ip}/${proj}/${imagewithtag}
+    Log To Console  ${output}
     Should Contain  ${output}  The image is not signed in Notary
 
 Cannot Push image

tests/resources/Harbor-Pages/Configuration.robot

@@ -169,6 +169,10 @@ Token Must Be Match
     [Arguments]  ${minutes}
     Textfield Value Should Be  xpath=//*[@id='tokenExpiration']  ${minutes}
 
+Robot Account Token Must Be Match
+    [Arguments]  ${days}
+    Textfield Value Should Be  xpath=//*[@id='robotTokenExpiration']  ${days}
+
 ## Replication
 Check Verify Remote Cert
     Mouse Down  xpath=//*[@id='clr-checkbox-verifyRemoteCert']

tests/resources/Harbor-Pages/Project-Repository.robot

@@ -18,12 +18,11 @@ Resource  ../../resources/Util.robot
 
 *** Keywords ***
 View Repo Scan Details
+    [Arguments]  @{vulnerabilities_level}
     Retry Element Click  xpath=${first_repo_xpath}
     Capture Page Screenshot
-    Retry Wait Until Page Contains  unknown
+    :FOR  ${item}  IN  @{vulnerabilities_level}
-    Retry Wait Until Page Contains  high
+    \    Retry Wait Until Page Contains Element  //hbr-artifact-vulnerabilities//clr-dg-row[contains(.,'${item}')]
-    Retry Wait Until Page Contains  medium
-    Retry Wait Until Page Contains  CVE
     Retry Element Click  xpath=${build_history_btn}
     Retry Wait Until Page Contains Element  xpath=${build_history_data}
 

tests/resources/Harbor-Pages/Verify.robot

@@ -309,6 +309,17 @@ Verify Project Setting
     \    Run Keyword If  ${scanonpush} == "true"  Checkbox Should Not Be Checked  //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input
     \   Close Browser
 
+Verify Interrogation Services
+    [Arguments]    ${json}
+    Log To Console  "Verify Interrogation Services..."
+    @{cron}=  Get Value From Json  ${json}  $.interrogation_services..cron
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Vulnerability Page
+    Page Should Contain  Custom
+    Page Should Contain  @{cron}[0]
+    Close Browser
+
 Verify System Setting
     [Arguments]    ${json}
     Log To Console  "Verify System Setting..."
@@ -320,6 +331,7 @@ Verify System Setting
     @{emailuser}=  Get Value From Json  ${json}  $.configuration..emailuser
     @{emailfrom}=  Get Value From Json  ${json}  $.configuration..emailfrom
     @{token}=  Get Value From Json  ${json}  $.configuration..token
+    @{robot_token}=  Get Value From Json  ${json}  $.configuration..robot_token
     @{scanschedule}=  Get Value From Json  ${json}  $.configuration..scanall
     @{cve_ids}=  Get Value From Json  ${json}  $.configuration..cve
     Init Chrome Driver
@@ -337,9 +349,7 @@ Verify System Setting
     ${ret}  Get Selected List Value  xpath=//select[@id='proCreation']
     Should Be Equal As Strings  ${ret}  @{creation}[0]
     Token Must Be Match  @{token}[0]
-    #ToDo:These 2 lines below should be uncommented right after issue 9211 was fixed
+    Robot Account Token Must Be Match  @{robot_token}[0]
-    #Switch To Vulnerability Page
-    #Page Should Contain  None
     Close Browser
 
 Verify Project-level Whitelist
@@ -383,4 +393,11 @@ Verify Clair Is Default Scanner
     Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
     Switch To Scanners Page
     Should Display The Default Clair Scanner
-    Close Browser
+    Close Browser
+
+Verify Trivy Is Default Scanner
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Scanners Page
+    Should Display The Default Trivy Scanner
+    Close Browser

tests/resources/Harbor-Pages/Vulnerability.robot

@@ -3,7 +3,7 @@ Documentation  This resource provides any keywords related to the Harbor private
 Resource  ../../resources/Util.robot
 
 *** Variables ***
- 
+
 *** Keywords ***
 Disable Scan Schedule
     Retry Double Keywords When Error  Retry Element Click  ${vulnerability_edit_btn}  Retry Wait Until Page Not Contains Element  ${vulnerability_edit_btn}
@@ -53,10 +53,10 @@ Scan Repo
     Run Keyword If  '${status}' == 'Succeed'  Wait Until Element Is Visible  //hbr-vulnerability-bar//hbr-result-tip-histogram  300
     Run Keyword If  '${status}' == 'Fail'  Wait Until Element Is Visible  //hbr-vulnerability-bar//a  300
 
-
+Scan Result Should Display In List Row
-Summary Chart Should Display
+    [Arguments]  ${tagname}  ${is_no_vulerabilty}=${false}
-    [Arguments]  ${tagname}
+    Run Keyword If  ${is_no_vulerabilty}==${true}  Retry Wait Until Page Contains Element  //artifact-list-tab//clr-dg-row[contains(.,'No vulnerability') and contains(.,'${tagname}')]//clr-dg-cell//clr-tooltip
-    Retry Wait Until Page Contains Element  //artifact-list-tab//clr-dg-row[contains(.,'${tagname}')]//clr-dg-cell//clr-tooltip
+    ...  ELSE  Retry Wait Until Page Contains Element  //artifact-list-tab//clr-dg-row[contains(.,'Total') and contains(.,'Fixable') and contains(.,'${tagname}')]//clr-dg-cell//clr-tooltip
 
 Enable Scan On Push
     Checkbox Should Not Be Selected  //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input
@@ -68,7 +68,7 @@ Enable Scan On Push
 Vulnerability Not Ready Project Hint
     Sleep  2
     ${element}=  Set Variable  xpath=//span[contains(@class, 'db-status-warning')]
-    Wait Until Element Is Visible And Enabled  ${element} 
+    Wait Until Element Is Visible And Enabled  ${element}
 
 Switch To Scanners Page
     Retry Element Click  xpath=//clr-main-container//clr-vertical-nav//a[contains(.,'Interrogation')]

tests/resources/TestCaseBody.robot

@@ -53,7 +53,7 @@ Body Of Manage project publicity
     Close Browser
 
 Body Of Scan A Tag In The Repo
-    [Arguments]  ${image_argument}  ${tag_argument}
+    [Arguments]  ${image_argument}  ${tag_argument}  ${is_no_vulerabilty}=${false}
     Init Chrome Driver
     ${d}=  get current date  result_format=%m%s
 
@@ -64,11 +64,68 @@ Body Of Scan A Tag In The Repo
     Go Into Project  project${d}
     Go Into Repo  project${d}/${image_argument}
     Scan Repo  ${tag_argument}  Succeed
-    Summary Chart Should Display  ${tag_argument}
+    Scan Result Should Display In List Row  ${tag_argument}  is_no_vulerabilty=${is_no_vulerabilty}
     Pull Image  ${ip}  user023  Test1@34  project${d}  ${image_argument}  ${tag_argument}
     # Edit Repo Info
     Close Browser
 
+Body Of Scan Image With Empty Vul
+    [Arguments]  ${image_argument}  ${tag_argument}
+    Init Chrome Driver
+    ${tag}=  Set Variable  ${tag_argument}
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  ${image_argument}:${tag_argument}
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Go Into Repo  ${image_argument}
+    Scan Repo  ${tag}  Succeed
+    Move To Summary Chart
+    Scan Result Should Display In List Row  ${tag}  is_no_vulerabilty=${true}
+    Close Browser
+
+Body Of Manual Scan All
+    [Arguments]  @{vulnerability_levels}
+    Init Chrome Driver
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  redis
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Switch To Vulnerability Page
+    Trigger Scan Now And Wait Until The Result Appears
+    Navigate To Projects
+    Go Into Project  library
+    Go Into Repo  redis
+    Scan Result Should Display In List Row  latest
+    View Repo Scan Details  @{vulnerability_levels}
+    Close Browser
+
+Body Of View Scan Results
+    [Arguments]  @{vulnerability_levels}
+    Init Chrome Driver
+    ${d}=  get current date  result_format=%m%s
+
+    Sign In Harbor  ${HARBOR_URL}  user025  Test1@34
+    Create An New Project  project${d}
+    Push Image  ${ip}  user025  Test1@34  project${d}  tomcat
+    Go Into Project  project${d}
+    Go Into Repo  project${d}/tomcat
+    Scan Repo  latest  Succeed
+    Scan Result Should Display In List Row  latest
+    View Repo Scan Details  @{vulnerability_levels}
+    Close Browser
+
+Body Of Scan Image On Push
+    [Arguments]  @{vulnerability_levels}
+    Init Chrome Driver
+    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
+    Go Into Project  library
+    Goto Project Config
+    Enable Scan On Push
+    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  memcached
+    Navigate To Projects
+    Go Into Project  library
+    Go Into Repo  memcached
+    Scan Result Should Display In List Row  latest
+    View Repo Scan Details  @{vulnerability_levels}
+    Close Browser
+
 Body Of List Helm Charts
     Init Chrome Driver
     ${d}=   Get Current Date    result_format=%m%s
@@ -99,15 +156,15 @@ Body Of List Helm Charts
     Close Browser
 
 Body Of Admin Push Signed Image
-    [Arguments]  ${image}=tomcat  ${with_remove}=${false}
+    [Arguments]  ${image}=tomcat  ${project}=library  ${with_remove}=${false}
     Enable Notary Client
 
     Docker Pull  ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}
-    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/notary-push-image.sh ${ip} library ${image} latest ${notaryServerEndpoint} ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}:latest
+    ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/notary-push-image.sh ${ip} ${project} ${image} latest ${notaryServerEndpoint} ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}:latest
     Log  ${output}
     Should Be Equal As Integers  ${rc}  0
 
-    ${rc}  ${output}=  Run And Return Rc And Output  curl -u admin:Harbor12345 -s --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/v2.0/projects/library/repositories/${image}/artifacts/latest?with_signature=true"
+    ${rc}  ${output}=  Run And Return Rc And Output  curl -u admin:Harbor12345 -s --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/v2.0/projects/${project}/repositories/${image}/artifacts/latest?with_signature=true"
 
     Log To Console  ${output}
     Should Be Equal As Integers  ${rc}  0

tests/robot-cases/Group1-Nightly/Clair.robot

@@ -43,7 +43,7 @@ Test Case - Disable Scan Schedule
     Close Browser
 
 Test Case - Scan A Tag In The Repo
-    Body Of Scan A Tag In The Repo  hello-world  latest
+    Body Of Scan A Tag In The Repo  hello-world  latest  is_no_vulerabilty=${true}
 
 Test Case - Scan As An Unprivileged User
     Init Chrome Driver
@@ -56,28 +56,12 @@ Test Case - Scan As An Unprivileged User
     Scan Is Disabled
     Close Browser
 
+# Chose a empty Vul repo
 Test Case - Scan Image With Empty Vul
-    Init Chrome Driver
+    Body Of Scan Image With Empty Vul  busybox  latest
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  busybox
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Go Into Project  library
-    Go Into Repo  busybox
-    Scan Repo  latest  Succeed
-    Move To Summary Chart
-    Wait Until Page Contains  No vulnerability
-    Close Browser
 
 Test Case - Manual Scan All
-    Init Chrome Driver
+    Body Of Manual Scan All  Low  High  Medium  Negligible
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  redis
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Switch To Vulnerability Page
-    Trigger Scan Now And Wait Until The Result Appears
-    Navigate To Projects
-    Go Into Project  library
-    Go Into Repo  redis
-    Summary Chart Should Display  latest
-    Close Browser
 
 Test Case - View Scan Error
     Init Chrome Driver
@@ -94,34 +78,11 @@ Test Case - View Scan Error
 
 Test Case - Scan Image On Push
     [Tags]  run-once
-    Init Chrome Driver
+    Body Of Scan Image On Push  Low  High  Medium  Negligible
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  hello-world
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Go Into Project  library
-    Goto Project Config
-    Enable Scan On Push
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  memcached
-    Navigate To Projects
-    Go Into Project  library
-    Go Into Repo  memcached
-    Summary Chart Should Display  latest
-    View Repo Scan Details
-    Close Browser
 
 Test Case - View Scan Results
     [Tags]  run-once
-    Init Chrome Driver
+    Body Of View Scan Results  Critical  High  Medium
-    ${d}=  get current date  result_format=%m%s
-
-    Sign In Harbor  ${HARBOR_URL}  user025  Test1@34
-    Create An New Project  project${d}
-    Push Image  ${ip}  user025  Test1@34  project${d}  tomcat
-    Go Into Project  project${d}
-    Go Into Repo  project${d}/tomcat
-    Scan Repo  latest  Succeed
-    Summary Chart Should Display  latest
-    View Repo Scan Details
-    Close Browser
 
 Test Case - Project Level Image Serverity Policy
     [Tags]  run-once

tests/robot-cases/Group1-Nightly/Notary.robot

@@ -35,8 +35,12 @@ Test Case - Project Level Policy Content Trust
     Click Content Trust
     Save Project Config
     # Verify
+    # Unsigned image can not be pulled
     Content Trust Should Be Selected
     Cannot Pull Unsigned Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  hello-world:latest
+    # Signed image can be pulled
+    Body Of Admin Push Signed Image  image=redis  project=project${d}
+    Pull image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  project${d}  redis  tag=latest
     Close Browser
 
 Test Case - Admin Push Signed Image

tests/robot-cases/Group1-Nightly/Trivy.robot

@@ -59,28 +59,14 @@ Test Case - Scan As An Unprivileged User
     Select Object  latest
     Scan Is Disabled
     Close Browser
-# chose a emptyVul repo
+
+# Chose a empty Vul repo
 Test Case - Scan Image With Empty Vul
-    Init Chrome Driver
+    Body Of Scan Image With Empty Vul  photon  2.0_scan
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  photon:2.0_scan
+
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Go Into Project  library
-    Go Into Repo  library/photon
-    Scan Repo  2.0  Succeed
-    Move To Summary Chart
-    Wait Until Page Contains  No vulnerability
-    Close Browser
 Test Case - Manual Scan All
-    Init Chrome Driver
+    Body Of Manual Scan All  Critical  High  Medium
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  redis
+
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Switch To Vulnerability Page
-    Trigger Scan Now And Wait Until The Result Appears
-    Navigate To Projects
-    Go Into Project  library
-    Go Into Repo  redis
-    Summary Chart Should Display  latest
-    Close Browser
 Test Case - View Scan Error
     Init Chrome Driver
     ${d}=  get current date  result_format=%m%s
@@ -96,34 +82,12 @@ Test Case - View Scan Error
 
 Test Case - Scan Image On Push
     [Tags]  run-once
-    Init Chrome Driver
+    Body Of Scan Image On Push  Critical  High  Medium
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  hello-world
-    Sign In Harbor  ${HARBOR_URL}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}
-    Go Into Project  library
-    Goto Project Config
-    Enable Scan On Push
-    Push Image  ${ip}  ${HARBOR_ADMIN}  ${HARBOR_PASSWORD}  library  memcached
-    Navigate To Projects
-    Go Into Project  library
-    Go Into Repo  memcached
-    Summary Chart Should Display  latest
-    View Repo Scan Details
-    Close Browser
 
 Test Case - View Scan Results
     [Tags]  run-once
-    Init Chrome Driver
+    Body Of View Scan Results  Critical  High
-    ${d}=  get current date  result_format=%m%s
 
-    Sign In Harbor  ${HARBOR_URL}  user025  Test1@34
-    Create An New Project  project${d}
-    Push Image  ${ip}  user025  Test1@34  project${d}  tomcat
-    Go Into Project  project${d}
-    Go Into Repo  project${d}/tomcat
-    Scan Repo  latest  Succeed
-    Summary Chart Should Display  latest
-    View Repo Scan Details
-    Close Browser
 Test Case - Project Level Image Serverity Policy
     [Tags]  run-once
     Init Chrome Driver

tests/robot-cases/Group1-Nightly/Upgrade.robot

@@ -27,7 +27,12 @@ Test Case - Manage project publicity
     Body Of Manage project publicity
 
 Test Case - Scan A Tag In The Repo
-    Body Of Scan A Tag In The Repo  hello-world  latest
+    [Tags]  clair
+    Body Of Scan A Tag In The Repo  hello-world  latest  is_no_vulerabilty=${true}
+
+Test Case - Scan A Tag In The Repo
+    [Tags]  trivy
+    Body Of Scan A Tag In The Repo  vmware/photon  1.0
 
 Test Case - List Helm Charts
     Body Of List Helm Charts

tests/robot-cases/Group3-Upgrade/data.json

@@ -77,6 +77,9 @@
               "tag_filters":"tagfilter2"
             }
          ],
+         "interrogation_services":{
+           "cron":"0 0 0 6 3 0"
+         },
          "configuration":{
            "authmode":"Database",
            "projectcreation":"everyone",
@@ -90,6 +93,7 @@
               "verifycert":false
            },
            "token":"40",
+           "robot_token":"31",
            "readonly":false,
            "scanall":{
               "type":"none",

tests/robot-cases/Group3-Upgrade/feature_map.json

@@ -188,5 +188,15 @@
             "branch":1,
             "version":"1.10"
         }
+    ],
+    "update_interrogation_services":[
+        {
+            "branch":1,
+            "version":"1.9"
+        },
+        {
+            "branch":1,
+            "version":"1.10"
+        }
     ]
  }

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -188,8 +188,14 @@ class HarborAPI:
         else:
             raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch))
 
+    @get_feature_branch
+    def update_interrogation_services(self, cron, **kwargs):
+        payload = {"schedule":{"type":"Custom","cron": cron}}
+        print payload
+        body=dict(body=payload)
+        request(url+"system/scanAll/schedule", 'post', **body)
 
-    def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token):
+    def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token, robot_token):
         payload = {
             "auth_mode": "db_auth",
             "email_from": emailfrom,
@@ -203,6 +209,7 @@ class HarborAPI:
             "read_only": False,
             "self_registration": selfreg,
             "token_expiration": token,
+            "robot_token_duration":robot_token,
             "scan_all_policy": {
                 "type": "none",
                 "parameter": {
@@ -470,7 +477,9 @@ def do_data_creation():
     for project in data["projects"]:
         harborAPI.update_project_setting_whitelist(project["name"],
                                     project["configuration"]["reuse_sys_cve_whitelist"],
-                                    project["configuration"]["deployment_security"],version=args.version)
+                                    project["configuration"]["deployment_security"], version=args.version)
+
+    harborAPI.update_interrogation_services(data["interrogation_services"]["cron"], version=args.version)
 
     harborAPI.update_systemsetting(data["configuration"]["emailsetting"]["emailfrom"],
                                    data["configuration"]["emailsetting"]["emailserver"],
@@ -478,8 +487,9 @@ def do_data_creation():
                                    data["configuration"]["emailsetting"]["emailuser"],
                                    data["configuration"]["projectcreation"],
                                    data["configuration"]["selfreg"],
-                                   float(data["configuration"]["token"]))
+                                   float(data["configuration"]["token"]),
+                                   float(data["configuration"]["robot_token"])*60*24)
 
-    harborAPI.add_sys_whitelist(data["configuration"]["deployment_security"],version=args.version)
+    harborAPI.add_sys_whitelist(data["configuration"]["deployment_security"], version=args.version)
 
 do_data_creation()

tests/robot-cases/Group3-Upgrade/verify.robot

@@ -36,10 +36,11 @@ Test Case - Upgrade Verify
     Run Keyword  Verify Endpoint  ${data}
     Run Keyword  Verify Replicationrule  ${data}
     Run Keyword  Verify Project Setting  ${data}
+    Run Keyword  Verify Interrogation Services  ${data}
     Run Keyword  Verify System Setting  ${data}
     Run Keyword  Verify System Setting Whitelist  ${data}
     Run Keyword  Verify Image Tag  ${data}
-    Run Keyword  Verify Clair Is Default Scanner
+    Run Keyword  Verify Trivy Is Default Scanner
 
 Test Case - Upgrade Verify
     [Tags]  1.10-latest
@@ -56,6 +57,7 @@ Test Case - Upgrade Verify
     Run Keyword  Verify Endpoint  ${data}
     Run Keyword  Verify Replicationrule  ${data}
     Run Keyword  Verify Project Setting  ${data}
+    Run Keyword  Verify Interrogation Services  ${data}
     Run Keyword  Verify System Setting  ${data}
     Run Keyword  Verify System Setting Whitelist  ${data}
     Run Keyword  Verify Image Tag  ${data}