// Copyright (c) 2017 VMware, Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. *** Settings *** Documentation Harbor BATs Resource ../../resources/Util.robot Default Tags Nightly *** Variables *** ${HARBOR_URL} https://${ip} ${SSH_USER} root ${HARBOR_ADMIN} admin *** Test Cases *** Test Case - Sign With Admin Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Close Browser Test Case - Vulnerability Data Not Ready #This case must run before vulnerability db ready Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Go Into Project library has_image=${false} Vulnerability Not Ready Project Hint Switch To Vulnerability Page Vulnerability Not Ready Config Hint Test Case - Garbage Collection Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world Sleep 2 Go Into Project project${d} Delete Repo project${d} Switch To Garbage Collection Click GC Now Logout Harbor Sleep 2 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Garbage Collection Sleep 1 Switch To GC History Retry Wait Until Page Contains Finished ${rc} ${output}= Run And Return Rc And Output curl -u ${HARBOR_ADMIN}:${HARBOR_PASSWORD} -i --insecure -H "Content-Type: application/json" -X GET "https://${ip}/api/system/gc/1/log" Log To Console ${output} Should Be Equal As Integers ${rc} 0 Should Contain ${output} 3 blobs and 0 manifests eligible for deletion #Should Contain ${output} Deleting blob: Should Contain ${output} success to run gc in job. Close Browser Test Case - Create An New Project Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project test${d} Close Browser Test Case - Delete A Project Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Delete A Project Without Sign In Harbor Close Browser Test Case - Read Only Mode Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project project${d} Enable Read Only Cannot Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest Disable Read Only Sleep 5 Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} busybox:latest Close Browser Test Case - Repo Size Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library alpine 2.6 2.6 Go Into Project library Go Into Repo alpine Wait Until Page Contains 1.92MB Close Browser Test Case - Staticsinfo Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${element}= Set Variable ${project_statistics_private_repository_icon} Wait Until Element Is Visible ${element} ${privaterepocount1}= Get Statics Private Repo ${privateprojcount1}= Get Statics Private Project ${publicrepocount1}= Get Statics Public Repo ${publicprojcount1}= Get Statics Public Project ${totalrepocount1}= Get Statics Total Repo ${totalprojcount1}= Get Statics Total Project Create An New Project private${d} Create An New Project public${d} true Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} private${d} hello-world Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} public${d} hello-world Reload Page ${privateprojcount}= evaluate ${privateprojcount1}+1 ${privaterepocount}= evaluate ${privaterepocount1}+1 ${publicprojcount}= evaluate ${publicprojcount1}+1 ${publicrepocount}= evaluate ${publicrepocount1}+1 ${totalrepocount}= evaluate ${totalrepocount1}+2 ${totalprojcount}= evaluate ${totalprojcount1}+2 Navigate To Projects Wait Until Element Is Visible ${element} ${privaterepocountStr}= Convert To String ${privaterepocount} Wait Until Element Contains ${element} ${privaterepocountStr} ${privaterepocount2}= Get Statics Private Repo ${privateprojcount2}= get statics private project ${publicrepocount2}= get statics public repo ${publicprojcount2}= get statics public project ${totalrepocount2}= get statics total repo ${totalprojcount2}= get statics total project Should Be Equal As Integers ${privateprojcount2} ${privateprojcount} Should be equal as integers ${privaterepocount2} ${privaterepocount} Should be equal as integers ${publicprojcount2} ${publicprojcount} Should be equal as integers ${publicrepocount2} ${publicrepocount} Should be equal as integers ${totalprojcount2} ${totalprojcount} Should be equal as integers ${totalrepocount2} ${totalrepocount} Test Case - Push Image Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project test${d} Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} test${d} hello-world:latest Go Into Project test${d} Wait Until Page Contains test${d}/hello-world Test Case - Project Level Policy Public Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project project${d} Go Into Project project${d} has_image=${false} Goto Project Config Click Project Public Save Project Config # Verify Public Should Be Selected # Project${d} default should be private # Here logout and login to try avoid a bug only in autotest Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Filter Object project${d} Project Should Be Public project${d} Close Browser Test Case - Project Level Policy Content Trust Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Create An New Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest Go Into Project project${d} Goto Project Config Click Content Trust Save Project Config # Verify Content Trust Should Be Selected Cannot Pull Unsigned Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} hello-world:latest Close Browser Test Case - Verify Download Ca Link Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To System Settings Page Should Contain Registry Root Certificate Close Browser Test Case - Edit Email Settings Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Email Config Email Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Email Verify Email Close Browser Test Case - Edit Token Expire Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To System Settings Modify Token Expiration 20 Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To System Settings Token Must Be Match 20 #reset to default Modify Token Expiration 30 Close Browser Test Case - Create A New Labels Init Chrome Driver ${d}= Get Current Date Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To System Labels Create New Labels label_${d} Close Browser Test Case - Update Label Init Chrome Driver ${d}= Get Current Date Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To System Labels Create New Labels label_${d} Sleep 3 ${d1}= Get Current Date Update A Label label_${d} Close Browser Test Case - Delete Label Init Chrome Driver ${d}= Get Current Date Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To System Labels Create New Labels label_${d} Sleep 3 Delete A Label label_${d} Close Browser Test Case - Disable Scan Schedule Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Vulnerability Page Disable Scan Schedule Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Vulnerability Page Retry Wait Until Page Contains None Close Browser Test Case - User View Projects Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user001 Test1@34 Create An New Project test${d}1 Create An New Project test${d}2 Create An New Project test${d}3 Switch To Log Wait Until Page Contains test${d}1 Wait Until Page Contains test${d}2 Wait Until Page Contains test${d}3 Close Browser Test Case - User View Logs Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user002 Test1@34 Create An New Project project${d} Push image ${ip} user002 Test1@34 project${d} busybox:latest Pull image ${ip} user002 Test1@34 project${d} busybox:latest Go Into Project project${d} Delete Repo project${d} Sleep 3 Go To Project Log Advanced Search Should Display Do Log Advanced Search Close Browser Test Case - Manage Project Member Init Chrome Driver ${user}= Set Variable user004 ${pwd}= Set Variable Test1@34 Sign In Harbor ${HARBOR_URL} ${user} ${pwd} Manage Project Member Without Sign In Harbor ${user} ${pwd} Close Browser Test Case - Manage project publicity Body Of Manage project publicity Test Case - Assign Sys Admin Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user009 Test1@34 Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch to User Tag Assign User Admin user009 Logout Harbor Sign In Harbor ${HARBOR_URL} user009 Test1@34 Administration Tag Should Display Close Browser Test Case - Edit Project Creation # Create normal user and login Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user010 Test1@34 Project Creation Should Display Logout Harbor Sleep 3 Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Set Pro Create Admin Only Logout Harbor Sign In Harbor ${HARBOR_URL} user010 Test1@34 Project Creation Should Not Display Logout Harbor Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Set Pro Create Every One Close browser Test Case - Edit Repo Info Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user011 Test1@34 Create An New Project project${d} Push Image ${ip} user011 Test1@34 project${d} hello-world Go Into Project project${d} Go Into Repo project${d}/hello-world Edit Repo Info Close Browser Test Case - Delete Multi Project Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user012 Test1@34 Create An New Project projecta${d} Create An New Project projectb${d} Push Image ${ip} user012 Test1@34 projecta${d} hello-world Navigate To Projects Filter Object project Retry Wait Element Not Visible //clr-datagrid/div/div[2] @{project_list} Create List projecta projectb Multi-delete Object ${project_delete_btn} @{project_list} # Verify delete project with image should not be deleted directly Delete Fail projecta${d} Delete Success projectb${d} Close Browser Test Case - Delete Multi Repo Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user013 Test1@34 Create An New Project project${d} Push Image ${ip} user013 Test1@34 project${d} hello-world Push Image ${ip} user013 Test1@34 project${d} busybox Sleep 2 Go Into Project project${d} @{repo_list} Create List hello-world busybox Multi-delete Object ${repo_delete_btn} @{repo_list} # Verify Delete Success hello-world busybox Close Browser Test Case - Delete Multi Tag Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user014 Test1@34 Create An New Project project${d} Push Image With Tag ${ip} user014 Test1@34 project${d} redis 3.2.10-alpine 3.2.10-alpine Push Image With Tag ${ip} user014 Test1@34 project${d} redis 4.0.7-alpine 4.0.7-alpine Go Into Project project${d} Go Into Repo redis @{tag_list} Create List 3.2.10-alpine 4.0.7-alpine Multi-delete object ${tag_delete_btn} @{tag_list} # Verify Delete Success 3.2.10-alpine 4.0.7-alpine Close Browser Test Case - Delete Repo on CardView Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user015 Test1@34 Create An New Project project${d} Push Image ${ip} user015 Test1@34 project${d} hello-world Push Image ${ip} user015 Test1@34 project${d} busybox Go Into Project project${d} Switch To CardView Delete Repo on CardView busybox # Verify Delete Success busybox Close Browser Test Case - Delete Multi Member Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user016 Test1@34 Create An New Project project${d} Go Into Project project${d} has_image=${false} Switch To Member Add Guest Member To Project user017 Add Guest Member To Project user018 Multi-delete Member user017 user018 Delete Success user017 user018 Close Browser Test Case - Project Admin Operate Labels Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user019 Test1@34 Create An New Project project${d} Go Into Project project${d} has_image=${false} Sleep 2 # Add labels Switch To Project Label Create New Labels label_${d} Sleep 2 Update A Label label_${d} Sleep 2 Delete A Label label_${d} Close Browser Test Case - Project Admin Add Labels To Repo Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user020 Test1@34 Create An New Project project${d} Push Image With Tag ${ip} user020 Test1@34 project${d} redis 3.2.10-alpine 3.2.10-alpine Push Image With Tag ${ip} user020 Test1@34 project${d} redis 4.0.7-alpine 4.0.7-alpine Go Into Project project${d} Sleep 2 # Add labels Switch To Project Label Create New Labels label111 Create New Labels label22 Sleep 2 Switch To Project Repo Go Into Repo project${d}/redis Add Labels To Tag 3.2.10-alpine label111 Add Labels To Tag 4.0.7-alpine label22 Filter Labels In Tags label111 label22 Close Browser Test Case - Developer Operate Labels Init Chrome Driver ${d}= Get Current Date result_format=%m%s Sign In Harbor ${HARBOR_URL} user021 Test1@34 Create An New Project project${d} Logout Harbor Manage Project Member user021 Test1@34 project${d} user022 Add ${false} Change User Role In Project user021 Test1@34 project${d} user022 Developer Sign In Harbor ${HARBOR_URL} user022 Test1@34 Go Into Project project${d} has_image=${false} Sleep 3 Page Should Not Contain Element xpath=//a[contains(.,'Labels')] Close Browser Test Case - Scan A Tag In The Repo Body Of Scan A Tag In The Repo Test Case - Scan As An Unprivileged User Init Chrome Driver ${d}= get current date result_format=%m%s Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library hello-world Sign In Harbor ${HARBOR_URL} user024 Test1@34 Go Into Project library Go Into Repo hello-world Select Object latest Scan Is Disabled Close Browser Test Case - Scan Image With Empty Vul Init Chrome Driver Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library busybox Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Go Into Project library Go Into Repo busybox Scan Repo latest Succeed Move To Summary Chart Wait Until Page Contains Unknow Close Browser Test Case - Manual Scan All Init Chrome Driver Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library redis Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Switch To Vulnerability Page Trigger Scan Now Navigate To Projects Go Into Project library Go Into Repo redis Summary Chart Should Display latest Close Browser Test Case - View Scan Error Init Chrome Driver ${d}= get current date result_format=%m%s Sign In Harbor ${HARBOR_URL} user026 Test1@34 Create An New Project project${d} Push Image ${ip} user026 Test1@34 project${d} vmware/photon:1.0 Go Into Project project${d} Go Into Repo project${d}/vmware/photon Scan Repo 1.0 Fail View Scan Error Log Close Browser Test Case - List Helm Charts And Delete Chart Files Body Of List Helm Charts Test Case - Helm CLI Push Init Chrome Driver ${user}= Set Variable user004 ${pwd}= Set Variable Test1@34 Sign In Harbor ${HARBOR_URL} ${user} ${pwd} Helm CLI Push Without Sign In Harbor ${user} ${pwd} Test Case - Admin Push Signed Image Body Of Admin Push Signed Image Test Case - Retag A Image Tag Init Chrome Driver ${random_num1}= Get Current Date result_format=%m%s ${random_num2}= Evaluate str(random.randint(1000,9999)) modules=random Sign In Harbor ${HARBOR_URL} user028 Test1@34 Create An New Project project${random_num1} Create An New Project project${random_num2} Go Into Project project${random_num1} has_image=${false} Sleep 1 Push Image With Tag ${ip} user028 Test1@34 project${random_num1} redis ${image_tag} Sleep 1 Go Into Repo project${random_num1}/redis Retag Image ${image_tag} project${random_num2} ${target_image_name} ${target_tag_value} Retry Wait Element Not Visible ${repo_retag_confirm_dlg} Navigate To Projects Go Into Project project${random_num2} Sleep 1 Page Should Contain ${target_image_name} Go Into Repo project${random_num2}/${target_image_name} Sleep 1 Page Should Contain Element xpath=${tag_value_xpath} Close Browser Test Case - Scan Image On Push Wait Unitl Vul Data Ready ${HARBOR_URL} 7200 30 Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} Go Into Project library Goto Project Config Enable Scan On Push Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} library memcached Navigate To Projects Go Into Project library Go Into Repo memcached Summary Chart Should Display latest Close Browser Test Case - View Scan Results Init Chrome Driver ${d}= get current date result_format=%m%s Sign In Harbor ${HARBOR_URL} user025 Test1@34 Create An New Project project${d} Push Image ${ip} user025 Test1@34 project${d} tomcat Go Into Project project${d} Go Into Repo project${d}/tomcat Scan Repo latest Succeed Summary Chart Should Display latest View Repo Scan Details Close Browser Test Case - Project Level Image Serverity Policy Init Chrome Driver Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${d}= get current date result_format=%m%s Create An New Project project${d} Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} haproxy Go Into Project project${d} Go Into Repo haproxy Scan Repo latest Succeed Navigate To Projects Go Into Project project${d} Set Vulnerabilty Serverity 0 Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} haproxy Close Browser