diff --git a/tests/apitests/python/library/retention.py b/tests/apitests/python/library/retention.py index 8c3bb7eaa..4e3f83305 100644 --- a/tests/apitests/python/library/retention.py +++ b/tests/apitests/python/library/retention.py @@ -110,7 +110,7 @@ class Retention(base.Base): _, status_code, _ = client.retentions_id_put_with_http_info(retention_id, policy) base._assert_status_code(expect_status_code, status_code) - def update_retention_add_rule(self, retention_id, selector_repository="**", selector_tag="**", expect_status_code = 200, **kwargs): + def update_retention_add_rule(self, retention_id, selector_repository="**", selector_tag="**", with_untag="True", expect_status_code = 200, **kwargs): client = self._get_client(**kwargs) policy, status_code, _ = client.retentions_id_get_with_http_info(retention_id) base._assert_status_code(200, status_code) @@ -134,7 +134,7 @@ class Retention(base.Base): { "kind": "doublestar", "decoration": "matches", - "extras":'["untagged":True]', + "extras":'["untagged":'+with_untag+']', "pattern": selector_tag } ] diff --git a/tests/apitests/python/test_retention.py b/tests/apitests/python/test_retention.py index de452e1c2..7aad5c87f 100644 --- a/tests/apitests/python/test_retention.py +++ b/tests/apitests/python/test_retention.py @@ -41,6 +41,7 @@ class TestProjects(unittest.TestCase): self.retention = Retention() self.artifact = Artifact() self.repo_name_1 = "test1" + self.repo_name_2 = "test2" def testTagRetention(self): user_ra_password = "Aa123456" @@ -57,15 +58,21 @@ class TestProjects(unittest.TestCase): push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['1.0']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['2.0']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_1, ['3.0','latest']) - push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['1.0']) - push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test2", ['latest']) + push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_2, ['1.0']) + push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, self.repo_name_2, ['latest']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test3", ['1.0']) push_special_image_to_project(TestProjects.project_src_repo_name, harbor_server, user_ra_name, user_ra_password, "test4", ['1.0']) + tag_data_artifact3_image1 = self.artifact.get_reference_info(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", **TestProjects.USER_RA_CLIENT) + print tag_data_artifact3_image1[0].digest + + tag_data_artifact2_image2 = self.artifact.get_reference_info(TestProjects.project_src_repo_name, self.repo_name_2, "latest", **TestProjects.USER_RA_CLIENT) + print tag_data_artifact2_image2[0].digest + tags = list_image_tags(harbor_server, TestProjects.project_src_repo_name+"/"+self.repo_name_1, user_ra_name, user_ra_password) - #Delete all tags of "artifact3" in repostory "image1"; - self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "latest",**TestProjects.USER_RA_CLIENT) - self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "3.0",**TestProjects.USER_RA_CLIENT) + #Delete all 2 tags of "artifact3" in repostory "image1"; + self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "latest", **TestProjects.USER_RA_CLIENT) + self.artifact.delete_tag(TestProjects.project_src_repo_name, self.repo_name_1, "3.0", "3.0", **TestProjects.USER_RA_CLIENT) tags = list_image_tags(harbor_server, TestProjects.project_src_repo_name+"/"+self.repo_name_1, user_ra_name, user_ra_password) resp=self.repo.list_repositories(TestProjects.project_src_repo_name, **TestProjects.USER_RA_CLIENT) @@ -104,18 +111,18 @@ class TestProjects(unittest.TestCase): self.assertEqual(len(resp), 4) resp=self.retention.get_retention_exec_task_log(retention_id,execution.id,resp[0].id, **TestProjects.USER_RA_CLIENT) print(resp) - # TODO As the repository isn't deleted when no tags left anymore - # TODO we should check the artifact/tag count here - # resp=self.repo.list_repositories(TestProjects.project_src_repo_id, **TestProjects.USER_RA_CLIENT) - # self.assertEqual(len(resp), 3) - #List artifacts successfully; - artifacts = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_1, **TestProjects.USER_RA_CLIENT) - print artifacts - # 'test1' has 3 artifacts, artifact1 with tag '1.0' and artifact2 with tag '2.0' should be deleted because they doesn't match 'latest' - # artifact3 should be retained because it has no tag, so count of artifacts should be 1. - # TODO: This verfication should be enhanced by verify sha256 at the same time; - self.assertTrue(len(artifacts)==1) + #List artifacts successfully, and untagged artifact in test1 should be the only one retained; + artifacts_1 = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_1, **TestProjects.USER_RA_CLIENT) + print artifacts_1[0].digest + self.assertTrue(len(artifacts_1)==1) + self.assertEqual(artifacts_1[0].digest, tag_data_artifact3_image1[0].digest) + + #List artifacts successfully, and artifact with latest tag in test2 should be the only one retained; + artifacts_2 = self.artifact.list_artifacts(TestProjects.project_src_repo_name, self.repo_name_2, **TestProjects.USER_RA_CLIENT) + print artifacts_2[0].digest + self.assertTrue(len(artifacts_2)==1) + self.assertEqual(artifacts_2[0].digest, tag_data_artifact2_image2[0].digest) @classmethod def tearDownClass(self): diff --git a/tests/resources/Docker-Util.robot b/tests/resources/Docker-Util.robot index 9ddff1fdb..edaefe1ae 100644 --- a/tests/resources/Docker-Util.robot +++ b/tests/resources/Docker-Util.robot @@ -72,6 +72,7 @@ Cannot Pull Unsigned Image [Arguments] ${ip} ${user} ${pass} ${proj} ${imagewithtag} Wait Unitl Command Success docker login -u ${user} -p ${pass} ${ip} ${output}= Command Should be Failed docker pull ${ip}/${proj}/${imagewithtag} + Log To Console ${output} Should Contain ${output} The image is not signed in Notary Cannot Push image diff --git a/tests/resources/Harbor-Pages/Configuration.robot b/tests/resources/Harbor-Pages/Configuration.robot index 9f6c6e53a..49962969c 100644 --- a/tests/resources/Harbor-Pages/Configuration.robot +++ b/tests/resources/Harbor-Pages/Configuration.robot @@ -169,6 +169,10 @@ Token Must Be Match [Arguments] ${minutes} Textfield Value Should Be xpath=//*[@id='tokenExpiration'] ${minutes} +Robot Account Token Must Be Match + [Arguments] ${days} + Textfield Value Should Be xpath=//*[@id='robotTokenExpiration'] ${days} + ## Replication Check Verify Remote Cert Mouse Down xpath=//*[@id='clr-checkbox-verifyRemoteCert'] diff --git a/tests/resources/Harbor-Pages/Project-Repository.robot b/tests/resources/Harbor-Pages/Project-Repository.robot index 8a4a525ad..51717d6a4 100644 --- a/tests/resources/Harbor-Pages/Project-Repository.robot +++ b/tests/resources/Harbor-Pages/Project-Repository.robot @@ -18,12 +18,11 @@ Resource ../../resources/Util.robot *** Keywords *** View Repo Scan Details + [Arguments] @{vulnerabilities_level} Retry Element Click xpath=${first_repo_xpath} Capture Page Screenshot - Retry Wait Until Page Contains unknown - Retry Wait Until Page Contains high - Retry Wait Until Page Contains medium - Retry Wait Until Page Contains CVE + :FOR ${item} IN @{vulnerabilities_level} + \ Retry Wait Until Page Contains Element //hbr-artifact-vulnerabilities//clr-dg-row[contains(.,'${item}')] Retry Element Click xpath=${build_history_btn} Retry Wait Until Page Contains Element xpath=${build_history_data} diff --git a/tests/resources/Harbor-Pages/Verify.robot b/tests/resources/Harbor-Pages/Verify.robot index f888f3630..77f594663 100644 --- a/tests/resources/Harbor-Pages/Verify.robot +++ b/tests/resources/Harbor-Pages/Verify.robot @@ -309,6 +309,17 @@ Verify Project Setting \ Run Keyword If ${scanonpush} == "true" Checkbox Should Not Be Checked //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input \ Close Browser +Verify Interrogation Services + [Arguments] ${json} + Log To Console "Verify Interrogation Services..." + @{cron}= Get Value From Json ${json} $.interrogation_services..cron + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Vulnerability Page + Page Should Contain Custom + Page Should Contain @{cron}[0] + Close Browser + Verify System Setting [Arguments] ${json} Log To Console "Verify System Setting..." @@ -320,6 +331,7 @@ Verify System Setting @{emailuser}= Get Value From Json ${json} $.configuration..emailuser @{emailfrom}= Get Value From Json ${json} $.configuration..emailfrom @{token}= Get Value From Json ${json} $.configuration..token + @{robot_token}= Get Value From Json ${json} $.configuration..robot_token @{scanschedule}= Get Value From Json ${json} $.configuration..scanall @{cve_ids}= Get Value From Json ${json} $.configuration..cve Init Chrome Driver @@ -337,9 +349,7 @@ Verify System Setting ${ret} Get Selected List Value xpath=//select[@id='proCreation'] Should Be Equal As Strings ${ret} @{creation}[0] Token Must Be Match @{token}[0] - #ToDo:These 2 lines below should be uncommented right after issue 9211 was fixed - #Switch To Vulnerability Page - #Page Should Contain None + Robot Account Token Must Be Match @{robot_token}[0] Close Browser Verify Project-level Whitelist @@ -383,4 +393,11 @@ Verify Clair Is Default Scanner Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Scanners Page Should Display The Default Clair Scanner - Close Browser \ No newline at end of file + Close Browser + +Verify Trivy Is Default Scanner + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Scanners Page + Should Display The Default Trivy Scanner + Close Browser diff --git a/tests/resources/Harbor-Pages/Vulnerability.robot b/tests/resources/Harbor-Pages/Vulnerability.robot index f7148c374..5fde022ec 100644 --- a/tests/resources/Harbor-Pages/Vulnerability.robot +++ b/tests/resources/Harbor-Pages/Vulnerability.robot @@ -3,7 +3,7 @@ Documentation This resource provides any keywords related to the Harbor private Resource ../../resources/Util.robot *** Variables *** - + *** Keywords *** Disable Scan Schedule Retry Double Keywords When Error Retry Element Click ${vulnerability_edit_btn} Retry Wait Until Page Not Contains Element ${vulnerability_edit_btn} @@ -53,10 +53,10 @@ Scan Repo Run Keyword If '${status}' == 'Succeed' Wait Until Element Is Visible //hbr-vulnerability-bar//hbr-result-tip-histogram 300 Run Keyword If '${status}' == 'Fail' Wait Until Element Is Visible //hbr-vulnerability-bar//a 300 - -Summary Chart Should Display - [Arguments] ${tagname} - Retry Wait Until Page Contains Element //artifact-list-tab//clr-dg-row[contains(.,'${tagname}')]//clr-dg-cell//clr-tooltip +Scan Result Should Display In List Row + [Arguments] ${tagname} ${is_no_vulerabilty}=${false} + Run Keyword If ${is_no_vulerabilty}==${true} Retry Wait Until Page Contains Element //artifact-list-tab//clr-dg-row[contains(.,'No vulnerability') and contains(.,'${tagname}')]//clr-dg-cell//clr-tooltip + ... ELSE Retry Wait Until Page Contains Element //artifact-list-tab//clr-dg-row[contains(.,'Total') and contains(.,'Fixable') and contains(.,'${tagname}')]//clr-dg-cell//clr-tooltip Enable Scan On Push Checkbox Should Not Be Selected //clr-checkbox-wrapper[@id='scan-image-on-push-wrapper']//input @@ -68,7 +68,7 @@ Enable Scan On Push Vulnerability Not Ready Project Hint Sleep 2 ${element}= Set Variable xpath=//span[contains(@class, 'db-status-warning')] - Wait Until Element Is Visible And Enabled ${element} + Wait Until Element Is Visible And Enabled ${element} Switch To Scanners Page Retry Element Click xpath=//clr-main-container//clr-vertical-nav//a[contains(.,'Interrogation')] diff --git a/tests/resources/TestCaseBody.robot b/tests/resources/TestCaseBody.robot index 05802d4db..473afcafa 100644 --- a/tests/resources/TestCaseBody.robot +++ b/tests/resources/TestCaseBody.robot @@ -53,7 +53,7 @@ Body Of Manage project publicity Close Browser Body Of Scan A Tag In The Repo - [Arguments] ${image_argument} ${tag_argument} + [Arguments] ${image_argument} ${tag_argument} ${is_no_vulerabilty}=${false} Init Chrome Driver ${d}= get current date result_format=%m%s @@ -64,11 +64,68 @@ Body Of Scan A Tag In The Repo Go Into Project project${d} Go Into Repo project${d}/${image_argument} Scan Repo ${tag_argument} Succeed - Summary Chart Should Display ${tag_argument} + Scan Result Should Display In List Row ${tag_argument} is_no_vulerabilty=${is_no_vulerabilty} Pull Image ${ip} user023 Test1@34 project${d} ${image_argument} ${tag_argument} # Edit Repo Info Close Browser +Body Of Scan Image With Empty Vul + [Arguments] ${image_argument} ${tag_argument} + Init Chrome Driver + ${tag}= Set Variable ${tag_argument} + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library ${image_argument}:${tag_argument} + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Go Into Repo ${image_argument} + Scan Repo ${tag} Succeed + Move To Summary Chart + Scan Result Should Display In List Row ${tag} is_no_vulerabilty=${true} + Close Browser + +Body Of Manual Scan All + [Arguments] @{vulnerability_levels} + Init Chrome Driver + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library redis + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Switch To Vulnerability Page + Trigger Scan Now And Wait Until The Result Appears + Navigate To Projects + Go Into Project library + Go Into Repo redis + Scan Result Should Display In List Row latest + View Repo Scan Details @{vulnerability_levels} + Close Browser + +Body Of View Scan Results + [Arguments] @{vulnerability_levels} + Init Chrome Driver + ${d}= get current date result_format=%m%s + + Sign In Harbor ${HARBOR_URL} user025 Test1@34 + Create An New Project project${d} + Push Image ${ip} user025 Test1@34 project${d} tomcat + Go Into Project project${d} + Go Into Repo project${d}/tomcat + Scan Repo latest Succeed + Scan Result Should Display In List Row latest + View Repo Scan Details @{vulnerability_levels} + Close Browser + +Body Of Scan Image On Push + [Arguments] @{vulnerability_levels} + Init Chrome Driver + Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} + Go Into Project library + Goto Project Config + Enable Scan On Push + Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library memcached + Navigate To Projects + Go Into Project library + Go Into Repo memcached + Scan Result Should Display In List Row latest + View Repo Scan Details @{vulnerability_levels} + Close Browser + Body Of List Helm Charts Init Chrome Driver ${d}= Get Current Date result_format=%m%s @@ -99,15 +156,15 @@ Body Of List Helm Charts Close Browser Body Of Admin Push Signed Image - [Arguments] ${image}=tomcat ${with_remove}=${false} + [Arguments] ${image}=tomcat ${project}=library ${with_remove}=${false} Enable Notary Client Docker Pull ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image} - ${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/notary-push-image.sh ${ip} library ${image} latest ${notaryServerEndpoint} ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}:latest + ${rc} ${output}= Run And Return Rc And Output ./tests/robot-cases/Group0-Util/notary-push-image.sh ${ip} ${project} ${image} latest ${notaryServerEndpoint} ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image}:latest Log ${output} Should Be Equal As Integers ${rc} 0 - ${rc} ${output}= Run And Return Rc And Output curl -u admin:Harbor12345 -s --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/v2.0/projects/library/repositories/${image}/artifacts/latest?with_signature=true" + ${rc} ${output}= Run And Return Rc And Output curl -u admin:Harbor12345 -s --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/v2.0/projects/${project}/repositories/${image}/artifacts/latest?with_signature=true" Log To Console ${output} Should Be Equal As Integers ${rc} 0 diff --git a/tests/robot-cases/Group1-Nightly/Clair.robot b/tests/robot-cases/Group1-Nightly/Clair.robot index edd44fc70..1509fe473 100644 --- a/tests/robot-cases/Group1-Nightly/Clair.robot +++ b/tests/robot-cases/Group1-Nightly/Clair.robot @@ -43,7 +43,7 @@ Test Case - Disable Scan Schedule Close Browser Test Case - Scan A Tag In The Repo - Body Of Scan A Tag In The Repo hello-world latest + Body Of Scan A Tag In The Repo hello-world latest is_no_vulerabilty=${true} Test Case - Scan As An Unprivileged User Init Chrome Driver @@ -56,28 +56,12 @@ Test Case - Scan As An Unprivileged User Scan Is Disabled Close Browser +# Chose a empty Vul repo Test Case - Scan Image With Empty Vul - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library busybox - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project library - Go Into Repo busybox - Scan Repo latest Succeed - Move To Summary Chart - Wait Until Page Contains No vulnerability - Close Browser + Body Of Scan Image With Empty Vul busybox latest Test Case - Manual Scan All - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library redis - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Vulnerability Page - Trigger Scan Now And Wait Until The Result Appears - Navigate To Projects - Go Into Project library - Go Into Repo redis - Summary Chart Should Display latest - Close Browser + Body Of Manual Scan All Low High Medium Negligible Test Case - View Scan Error Init Chrome Driver @@ -94,34 +78,11 @@ Test Case - View Scan Error Test Case - Scan Image On Push [Tags] run-once - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project library - Goto Project Config - Enable Scan On Push - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library memcached - Navigate To Projects - Go Into Project library - Go Into Repo memcached - Summary Chart Should Display latest - View Repo Scan Details - Close Browser + Body Of Scan Image On Push Low High Medium Negligible Test Case - View Scan Results [Tags] run-once - Init Chrome Driver - ${d}= get current date result_format=%m%s - - Sign In Harbor ${HARBOR_URL} user025 Test1@34 - Create An New Project project${d} - Push Image ${ip} user025 Test1@34 project${d} tomcat - Go Into Project project${d} - Go Into Repo project${d}/tomcat - Scan Repo latest Succeed - Summary Chart Should Display latest - View Repo Scan Details - Close Browser + Body Of View Scan Results Critical High Medium Test Case - Project Level Image Serverity Policy [Tags] run-once diff --git a/tests/robot-cases/Group1-Nightly/Notary.robot b/tests/robot-cases/Group1-Nightly/Notary.robot index 377b153bc..189052460 100644 --- a/tests/robot-cases/Group1-Nightly/Notary.robot +++ b/tests/robot-cases/Group1-Nightly/Notary.robot @@ -35,8 +35,12 @@ Test Case - Project Level Policy Content Trust Click Content Trust Save Project Config # Verify + # Unsigned image can not be pulled Content Trust Should Be Selected Cannot Pull Unsigned Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest + # Signed image can be pulled + Body Of Admin Push Signed Image image=redis project=project${d} + Pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} redis tag=latest Close Browser Test Case - Admin Push Signed Image diff --git a/tests/robot-cases/Group1-Nightly/Trivy.robot b/tests/robot-cases/Group1-Nightly/Trivy.robot index 0bd5327c7..a4b936f62 100644 --- a/tests/robot-cases/Group1-Nightly/Trivy.robot +++ b/tests/robot-cases/Group1-Nightly/Trivy.robot @@ -59,28 +59,14 @@ Test Case - Scan As An Unprivileged User Select Object latest Scan Is Disabled Close Browser -# chose a emptyVul repo + +# Chose a empty Vul repo Test Case - Scan Image With Empty Vul - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library photon:2.0_scan - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project library - Go Into Repo library/photon - Scan Repo 2.0 Succeed - Move To Summary Chart - Wait Until Page Contains No vulnerability - Close Browser + Body Of Scan Image With Empty Vul photon 2.0_scan + Test Case - Manual Scan All - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library redis - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Switch To Vulnerability Page - Trigger Scan Now And Wait Until The Result Appears - Navigate To Projects - Go Into Project library - Go Into Repo redis - Summary Chart Should Display latest - Close Browser + Body Of Manual Scan All Critical High Medium + Test Case - View Scan Error Init Chrome Driver ${d}= get current date result_format=%m%s @@ -96,34 +82,12 @@ Test Case - View Scan Error Test Case - Scan Image On Push [Tags] run-once - Init Chrome Driver - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world - Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} - Go Into Project library - Goto Project Config - Enable Scan On Push - Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library memcached - Navigate To Projects - Go Into Project library - Go Into Repo memcached - Summary Chart Should Display latest - View Repo Scan Details - Close Browser + Body Of Scan Image On Push Critical High Medium Test Case - View Scan Results [Tags] run-once - Init Chrome Driver - ${d}= get current date result_format=%m%s + Body Of View Scan Results Critical High - Sign In Harbor ${HARBOR_URL} user025 Test1@34 - Create An New Project project${d} - Push Image ${ip} user025 Test1@34 project${d} tomcat - Go Into Project project${d} - Go Into Repo project${d}/tomcat - Scan Repo latest Succeed - Summary Chart Should Display latest - View Repo Scan Details - Close Browser Test Case - Project Level Image Serverity Policy [Tags] run-once Init Chrome Driver diff --git a/tests/robot-cases/Group1-Nightly/Upgrade.robot b/tests/robot-cases/Group1-Nightly/Upgrade.robot index f544378e5..f8b6593d0 100644 --- a/tests/robot-cases/Group1-Nightly/Upgrade.robot +++ b/tests/robot-cases/Group1-Nightly/Upgrade.robot @@ -27,7 +27,12 @@ Test Case - Manage project publicity Body Of Manage project publicity Test Case - Scan A Tag In The Repo - Body Of Scan A Tag In The Repo hello-world latest + [Tags] clair + Body Of Scan A Tag In The Repo hello-world latest is_no_vulerabilty=${true} + +Test Case - Scan A Tag In The Repo + [Tags] trivy + Body Of Scan A Tag In The Repo vmware/photon 1.0 Test Case - List Helm Charts Body Of List Helm Charts diff --git a/tests/robot-cases/Group3-Upgrade/data.json b/tests/robot-cases/Group3-Upgrade/data.json index 32af45fa6..ccfc20eb9 100644 --- a/tests/robot-cases/Group3-Upgrade/data.json +++ b/tests/robot-cases/Group3-Upgrade/data.json @@ -77,6 +77,9 @@ "tag_filters":"tagfilter2" } ], + "interrogation_services":{ + "cron":"0 0 0 6 3 0" + }, "configuration":{ "authmode":"Database", "projectcreation":"everyone", @@ -90,6 +93,7 @@ "verifycert":false }, "token":"40", + "robot_token":"31", "readonly":false, "scanall":{ "type":"none", diff --git a/tests/robot-cases/Group3-Upgrade/feature_map.json b/tests/robot-cases/Group3-Upgrade/feature_map.json index 6e891c9ea..b7bf3ae37 100644 --- a/tests/robot-cases/Group3-Upgrade/feature_map.json +++ b/tests/robot-cases/Group3-Upgrade/feature_map.json @@ -188,5 +188,15 @@ "branch":1, "version":"1.10" } + ], + "update_interrogation_services":[ + { + "branch":1, + "version":"1.9" + }, + { + "branch":1, + "version":"1.10" + } ] } \ No newline at end of file diff --git a/tests/robot-cases/Group3-Upgrade/prepare.py b/tests/robot-cases/Group3-Upgrade/prepare.py index 4cc04fc12..c8ca60e64 100644 --- a/tests/robot-cases/Group3-Upgrade/prepare.py +++ b/tests/robot-cases/Group3-Upgrade/prepare.py @@ -188,8 +188,14 @@ class HarborAPI: else: raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, branch)) + @get_feature_branch + def update_interrogation_services(self, cron, **kwargs): + payload = {"schedule":{"type":"Custom","cron": cron}} + print payload + body=dict(body=payload) + request(url+"system/scanAll/schedule", 'post', **body) - def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token): + def update_systemsetting(self, emailfrom, emailhost, emailport, emailuser, creation, selfreg, token, robot_token): payload = { "auth_mode": "db_auth", "email_from": emailfrom, @@ -203,6 +209,7 @@ class HarborAPI: "read_only": False, "self_registration": selfreg, "token_expiration": token, + "robot_token_duration":robot_token, "scan_all_policy": { "type": "none", "parameter": { @@ -470,7 +477,9 @@ def do_data_creation(): for project in data["projects"]: harborAPI.update_project_setting_whitelist(project["name"], project["configuration"]["reuse_sys_cve_whitelist"], - project["configuration"]["deployment_security"],version=args.version) + project["configuration"]["deployment_security"], version=args.version) + + harborAPI.update_interrogation_services(data["interrogation_services"]["cron"], version=args.version) harborAPI.update_systemsetting(data["configuration"]["emailsetting"]["emailfrom"], data["configuration"]["emailsetting"]["emailserver"], @@ -478,8 +487,9 @@ def do_data_creation(): data["configuration"]["emailsetting"]["emailuser"], data["configuration"]["projectcreation"], data["configuration"]["selfreg"], - float(data["configuration"]["token"])) + float(data["configuration"]["token"]), + float(data["configuration"]["robot_token"])*60*24) - harborAPI.add_sys_whitelist(data["configuration"]["deployment_security"],version=args.version) + harborAPI.add_sys_whitelist(data["configuration"]["deployment_security"], version=args.version) do_data_creation() \ No newline at end of file diff --git a/tests/robot-cases/Group3-Upgrade/verify.robot b/tests/robot-cases/Group3-Upgrade/verify.robot index 84e6af7c3..b8983ba29 100644 --- a/tests/robot-cases/Group3-Upgrade/verify.robot +++ b/tests/robot-cases/Group3-Upgrade/verify.robot @@ -36,10 +36,11 @@ Test Case - Upgrade Verify Run Keyword Verify Endpoint ${data} Run Keyword Verify Replicationrule ${data} Run Keyword Verify Project Setting ${data} + Run Keyword Verify Interrogation Services ${data} Run Keyword Verify System Setting ${data} Run Keyword Verify System Setting Whitelist ${data} Run Keyword Verify Image Tag ${data} - Run Keyword Verify Clair Is Default Scanner + Run Keyword Verify Trivy Is Default Scanner Test Case - Upgrade Verify [Tags] 1.10-latest @@ -56,6 +57,7 @@ Test Case - Upgrade Verify Run Keyword Verify Endpoint ${data} Run Keyword Verify Replicationrule ${data} Run Keyword Verify Project Setting ${data} + Run Keyword Verify Interrogation Services ${data} Run Keyword Verify System Setting ${data} Run Keyword Verify System Setting Whitelist ${data} Run Keyword Verify Image Tag ${data}