Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-29 05:35:43 +01:00
Code Issues Projects Releases Wiki Activity

Add external scanner test case (#19682)

Browse Source 
Fix #19279

Signed-off-by: Yang Jiao <jiaoya@vmware.com>
This commit is contained in:
Yang Jiao 2023-12-07 11:20:11 +08:00 committed by GitHub
parent 323e11fefb
commit ed0a69cf80
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 228 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
tests/resources/Harbor-Pages/Project.robot
View File

@ -293,6 +293,9 @@ Switch To Project Label
Switch To Project Repo Switch To Project Repo
Retry Element Click xpath=//project-detail//a[contains(.,'Repositories')] Retry Element Click xpath=//project-detail//a[contains(.,'Repositories')]
Switch To Project Scanner
Retry Element Click xpath=//project-detail//a[contains(.,'Scanner')]
Add Labels To Tag Add Labels To Tag
[Arguments] ${tagName} ${labelName} [Arguments] ${tagName} ${labelName}
Retry Element Click xpath=//clr-dg-row[contains(.,'${tagName}')]//label[contains(@class,'clr-control-label')] Retry Element Click xpath=//clr-dg-row[contains(.,'${tagName}')]//label[contains(@class,'clr-control-label')]
@ -434,3 +437,11 @@ Download Latest CVE CSV File
Retry Double Keywords When Error Retry Element Click ${latest_download_cve_csv_file_xpath} Retry Wait Until Page Does Not Contains ${csv_file_name} Retry Double Keywords When Error Retry Element Click ${latest_download_cve_csv_file_xpath} Retry Wait Until Page Does Not Contains ${csv_file_name}
Retry File Should Exist ${csv_file_path} Retry File Should Exist ${csv_file_path}
[Return] ${csv_file_path} [Return] ${csv_file_path}
Select Project Scanner
[Arguments] ${scanner_name} ${scanner_count}=${null}
Retry Element Click //*[@id='edit-scanner']
Run Keyword If '${scanner_count}'!='${null}' Retry Wait Element Count //clr-dg-row ${scanner_count}
Retry Element Click //clr-dg-row[.//clr-dg-cell[text()='${scanner_name}']]//label[contains(@class,'clr-control-label')]
Retry Element Click //*[@id='save-scanner']
Retry Wait Element Visible //span[@id='scanner-name' and text()='${scanner_name}']

81
tests/resources/Harbor-Pages/Vulnerability.robot
View File

@ -105,3 +105,84 @@ Check Scan All Artifact Job Status Is Stopped
Wait Until Element Is Visible ${stopped_icon} Wait Until Element Is Visible ${stopped_icon}
${stopped_total}= Get Text ${stopped_icon} ${stopped_total}= Get Text ${stopped_icon}
Should Be True ${stopped_total} > 0 Should Be True ${stopped_total} > 0
Filter Scanner By Name
[Arguments] ${name}
Retry Button Click (//clr-dg-column//button)[2]
Retry Text Input //input[@name='search'] ${name}
Retry Button Click (//clr-dg-column//button)[2]
Filter Scanner By Endpoint
[Arguments] ${endpoint}
Retry Button Click (//clr-dg-column//button)[5]
Retry Text Input //input[@name='search'] ${endpoint}
Retry Button Click (//clr-dg-column//button)[5]
Select Scanner Auth Mode
[Arguments] ${auth}
Retry Element Click ${scanner_auth_select}
Retry Element Click ${scanner_auth_select}//option[contains(text(),'${auth}')]
Add A New Scanner
[Arguments] ${name} ${endpoint} ${auth} ${desc}=${null} ${skip_cert_cerification}=${false} ${internal_registry_address}=${false} ${username}=${null} ${password}=${null} ${token}=${null} ${api_key}=${null} ${apikey}=${null}
Retry Button Click ${add_scanner_btn}
Retry Text Input ${scanner_name_input} ${name}
Run Keyword If '${desc}'!='${null}' Retry Text Input ${scanner_desc_input} ${desc}
Retry Text Input ${scanner_endpoint_input} ${endpoint}
Select Scanner Auth Mode ${auth}
IF '${auth}'=='Basic'
Retry Text Input ${scanner_username_input} ${username}
Retry Text Input ${scanner_password_input} ${password}
ELSE IF '${auth}'=='Bearer'
Retry Text Input ${scanner_token_input} ${token}
ELSE IF '${auth}'=='APIKey'
Retry Text Input ${scanner_apikey_input} ${apikey}
END
Run Keyword If '${skip_cert_cerification}'==${true} Retry Element Click ${scanner_skip_cert_checkbox}
Run Keyword If '${internal_registry_address}'==${true} Retry Element Click ${add_scanner_internal_registry_checkbox}
Retry Double Keywords When Error Retry Button Click ${scanner_test_connection_btn} Wait Until Page Contains Test passed
Retry Double Keywords When Error Retry Element Click ${scanner_add_btn} Wait Until Page Does Not Contain Element ${scanner_add_btn}
Retry Wait Until Page Contains Element //clr-dg-row[.//span[text()='${name}'] and .//clr-dg-cell[text()='${endpoint}'] and .//span[text()='Healthy'] and .//span[text()='Enabled'] and .//clr-dg-cell[text()='${auth}']]
Update Scanner
[Arguments] ${origin_name} ${name} ${endpoint} ${auth} ${desc}=${null} ${skip_cert_cerification}=${false} ${internal_registry_address}=${false} ${username}=${null} ${password}=${null} ${token}=${null} ${api_key}=${null}
Filter Scanner By Name ${origin_name}
Retry Element Click //clr-dg-row[.//span[text()='${origin_name}']]//label[contains(@class,'clr-control-label')]
Retry Double Keywords When Error Retry Element Click ${scanner_action_xpath} Retry Element Click ${edit_scanner_action_xpath}
Retry Text Input ${scanner_name_input} ${name}
Run Keyword If '${desc}'!='${null}' Retry Text Input ${scanner_desc_input} ${desc}
Retry Text Input ${scanner_endpoint_input} ${endpoint}
Select Scanner Auth Mode ${auth}
IF '${auth}'=='Basic'
Retry Text Input ${scanner_username_input} ${username}
Retry Text Input ${scanner_password_input} ${password}
ELSE IF '${auth}'=='Bearer'
Retry Text Input ${scanner_token_input} ${token}
ELSE IF '${auth}'=='APIKey'
Retry Text Input ${scanner_apikey_input} ${api_key}
END
Run Keyword If '${skip_cert_cerification}'==${true} Retry Element Click ${scanner_skip_cert_checkbox}
Run Keyword If '${internal_registry_address}'==${true} Retry Element Click ${add_scanner_internal_registry_checkbox}
Retry Double Keywords When Error Retry Element Click ${scanner_save_btn} Wait Until Page Does Not Contain Element ${scanner_save_btn}
Retry Wait Until Page Contains Element //clr-dg-row[.//span[text()='${name}'] and .//clr-dg-cell[text()='${endpoint}'] and .//clr-dg-cell[text()='${auth}']]
Delete Scanner
[Arguments] ${name}
Filter Scanner By Name ${name}
Retry Double Keywords When Error Retry Element Click //clr-dg-row[.//span[text()='${name}']]//label[contains(@class,'clr-control-label')] Retry Element Click ${scanner_action_xpath}
Retry Double Keywords When Error Retry Element Click ${delete_scanner_action_xpath} Retry Element Click ${delete_scanner_confirm_btn}
Reload Page
Retry Double Keywords When Error Filter Scanner By Name ${name} Retry Wait Until Page Contains Can not find any scanner
Enable Or Deactivate Scanner
[Arguments] ${name} ${action}
Filter Scanner By Name ${name}
Retry Double Keywords When Error Retry Element Click //clr-dg-row[.//span[text()='${name}']]//label[contains(@class,'clr-control-label')] Retry Element Click ${scanner_action_xpath}
Retry Element Click //clr-dropdown-menu//span[text()='${action}']
${action}= Set Variable ${action[0].upper()}${action[1:].lower()}d
Retry Wait Until Page Contains Element //clr-dg-row[.//span[text()='${name}'] and .//span[text()='${action}']]
Set Scanner As Default
[Arguments] ${name}
Retry Element Click //clr-dg-row[.//span[text()='${name}']]//label[contains(@class,'clr-control-label')]
Retry Double Keywords When Error Retry Element Click ${scanner_set_default_btn} Retry Wait Element Visible //clr-dg-row[.//span[text()='${name}'] and .//span[text()='Default']]

15
tests/resources/Harbor-Pages/Vulnerability_Elements.robot
View File

@ -26,6 +26,7 @@ ${stop_scan_button} //vulnerability-config//button[contains(.,'Stop Scan')]
${vulnerability_page} //clr-vertical-nav-group-children/a[contains(.,'Vulnerability')] ${vulnerability_page} //clr-vertical-nav-group-children/a[contains(.,'Vulnerability')]
${set_default_scanner} //button[@id='set-default'] ${set_default_scanner} //button[@id='set-default']
${scanner_action_xpath} //span[@id='action-scanner'] ${scanner_action_xpath} //span[@id='action-scanner']
${edit_scanner_action_xpath} //span[text()='EDIT']
${delete_scanner_action_xpath} //span[@id='delete-scanner-action'] ${delete_scanner_action_xpath} //span[@id='delete-scanner-action']
${immutable_trivy_msg_xpath} //span[contains(.,'registration Trivy is not allowed to delete as it is immutable: scanner API: delete')] ${immutable_trivy_msg_xpath} //span[contains(.,'registration Trivy is not allowed to delete as it is immutable: scanner API: delete')]
${delete_scanner_confirm_btn} xpath=//clr-modal//button[contains(.,'DELETE')] ${delete_scanner_confirm_btn} xpath=//clr-modal//button[contains(.,'DELETE')]
@ -36,3 +37,17 @@ ${not_scanned_icon} xpath=//span[@class[contains(.,'not-scan')]]
${stopped_icon} //span[@class[contains(.,'badge abort')]] ${stopped_icon} //span[@class[contains(.,'badge abort')]]
${view_scanner_icon_xpath} //clr-signpost//cds-icon ${view_scanner_icon_xpath} //clr-signpost//cds-icon
${view_scanner_doc_xpath} //clr-signpost-content//a[contains(.,'view documentation')] ${view_scanner_doc_xpath} //clr-signpost-content//a[contains(.,'view documentation')]
${add_scanner_btn} //button[text()=' NEW SCANNER ']
${scanner_name_input} //*[@id='scanner-name']
${scanner_desc_input} //*[@id='description']
${scanner_endpoint_input} //*[@id='scanner-endpoint']
${scanner_skip_cert_checkbox} //label[contains(@class,'clr-control-label') and contains(@for,'scanner-skipCertVerify')]
${scanner_test_connection_btn} //*[@id='button-test']
${scanner_add_btn} //span[text()=' ADD ']
${scanner_save_btn} //span[text()=' SAVE ']
${scanner_auth_select} //*[@id='scanner-authorization']
${scanner_username_input} //*[@id='scanner-username']
${scanner_password_input} //*[@id='scanner-password']
${scanner_token_input} //*[@id='scanner-token']
${scanner_apikey_input} //*[@id='scanner-apiKey']
${scanner_set_default_btn} //*[@id='set-default']

120
tests/robot-cases/Group1-Nightly/Trivy.robot
View File

@ -163,3 +163,123 @@ Test Case - Verfiy Project Level CVE Allowlist By Quick Way of Add System
Test Case - Stop Scan And Stop Scan All Test Case - Stop Scan And Stop Scan All
[Tags] stop_scan_job [Tags] stop_scan_job
Body Of Stop Scan And Stop Scan All Body Of Stop Scan And Stop Scan All
Test Case - External Scanner CRUD
[Tags] external_scanner_crud need_scanner_endpoint
${SCANNER_ENDPOINT_VALUE}= Get Variable Value ${SCANNER_ENDPOINT} ${EMPTY}
Skip If '${SCANNER_ENDPOINT_VALUE}' == '${EMPTY}'
Init Chrome Driver
${d}= get current date result_format=%m%s
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Switch To Scanners Page
# Add a new scanner
Add A New Scanner scanner${d} ${SCANNER_ENDPOINT} Basic For testing ${true} ${true} scanner_name scanner_password
# Update this scanner
Update Scanner scanner${d} scanner${d}-edit1 ${SCANNER_ENDPOINT}1 Bearer For testing-edit1 ${true} ${true} token=scanner_token
Update Scanner scanner${d}-edit1 scanner${d}-edit2 ${SCANNER_ENDPOINT}2 APIKey For testing-edit2 ${true} ${true} api_key=scanner_api_key
Update Scanner scanner${d}-edit2 scanner${d} ${SCANNER_ENDPOINT} None For testing
# Filter this scanner
Filter Scanner By Name scanner${d}
Filter Scanner By Endpoint ${SCANNER_ENDPOINT}
Retry Wait Element Count //clr-dg-row 1
Retry Wait Until Page Contains Element //clr-dg-row[.//span[text()='scanner${d}'] and .//clr-dg-cell[text()='${SCANNER_ENDPOINT}'] and .//span[text()='Healthy'] and .//clr-dg-cell[text()='None']]
# Delete this scanner
Delete Scanner scanner${d}
Close Browser
Test Case - Set External Scanner As Default And Scan
[Tags] external_scanner_scan need_scanner_endpoint
${SCANNER_ENDPOINT_VALUE}= Get Variable Value ${SCANNER_ENDPOINT} ${EMPTY}
Skip If '${SCANNER_ENDPOINT_VALUE}' == '${EMPTY}'
Init Chrome Driver
${d}= get current date result_format=%m%s
${image}= Set Variable hello-world
${tag}= Set Variable latest
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Project And Go Into Project project${d}
Switch To Project Scanner
Retry Wait Element Visible //span[@id='scanner-name' and text()='Trivy']
Switch To Scanners Page
Retry Wait Element Visible //clr-dg-row[.//span[text()='Trivy'] and .//span[text()='Default']]
# Add a new scanner
Add A New Scanner scanner${d} ${SCANNER_ENDPOINT} None For testing
# Set this scanner to default
Set Scanner As Default scanner${d}
Go Into Project project${d} ${false}
Switch To Project Scanner
Retry Wait Element Visible //span[@id='scanner-name' and text()='scanner${d}']
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image}
Go Into Repo project${d} ${image}
Scan Repo ${tag} Succeed
Retry Wait Element Visible //hbr-result-tip-histogram//span[1][text()='10']
Switch To Scanners Page
Set Scanner As Default Trivy
Go Into Project project${d}
Switch To Project Scanner
Retry Wait Element Visible //span[@id='scanner-name' and text()='Trivy']
Go into Repo project${d} ${image}
Scan Repo ${tag} Succeed
Retry Wait Element Visible //hbr-result-tip-histogram//div[text()=' No vulnerability ']
Back Project Home project${d}
Switch To Project Scanner
Select Project Scanner scanner${d} 2
Go Into Repo project${d} ${image}
Retry Wait Element Visible //hbr-result-tip-histogram//span[1][text()='10']
Scan Repo ${tag} Succeed
Retry Wait Element Visible //hbr-result-tip-histogram//span[1][text()='10']
Back Project Home project${d}
Switch To Project Scanner
Select Project Scanner Trivy 2
Go Into Repo project${d} ${image}
Retry Wait Element Visible //hbr-result-tip-histogram//div[text()=' No vulnerability ']
Scan Repo ${tag} Succeed
Retry Wait Element Visible //hbr-result-tip-histogram//div[text()=' No vulnerability ']
Switch To Scanners Page
Delete Scanner scanner${d}
Go Into Project project${d}
Switch To Project Scanner
Retry Element Click //*[@id='edit-scanner']
Retry Wait Element Count //clr-dg-row 1
Close Browser
Test Case - Enable And Deactivate Scanner
[Tags] enable_deactivate_scanner need_scanner_endpoint
${SCANNER_ENDPOINT_VALUE}= Get Variable Value ${SCANNER_ENDPOINT} ${EMPTY}
Skip If '${SCANNER_ENDPOINT_VALUE}' == '${EMPTY}'
Init Chrome Driver
${d}= get current date result_format=%m%s
${image}= Set Variable hello-world
${tag}= Set Variable latest
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Project And Go Into Project project${d}
Push Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image}
Switch To Scanners Page
Add A New Scanner scanner${d} ${SCANNER_ENDPOINT} None For testing
Go Into Project project${d}
Switch To Project Scanner
Select Project Scanner scanner${d}
Go into Repo project${d} ${image}
Scan Repo ${tag} Succeed
# Deactivate this scanner
Switch To Scanners Page
Enable Or Deactivate Scanner scanner${d} DEACTIVATE
Go Into Project project${d}
Switch To Project Scanner
Retry Wait Element Visible //scanner//span[text()='Deactivated']
Go Into Repo project${d} ${image}
Retry Element Click //clr-dg-row[contains(.,'${tag}')]//label[contains(@class,'clr-control-label')]
Retry Wait Element Should Be Disabled //button[@id='scan-btn']
# Enable this scanner
Switch To Scanners Page
Enable Or Deactivate Scanner scanner${d} ENABLE
Go Into Project project${d}
Switch To Project Scanner
Retry Wait Element Not Visible //scanner//span[text()='Deactivated']
Go Into Repo project${d} ${image}
Scan Repo ${tag} Succeed
Switch To Scanners Page
Delete Scanner scanner${d}
Go Into Project project${d}
Switch To Project Scanner
Retry Wait Element Visible //span[@id='scanner-name' and text()='Trivy']
Close Browser