Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-11-24 11:15:24 +01:00
Code Issues Projects Releases Wiki Activity

Update the testcase for v2.12.0, changes include (#21113)

Browse Source 
pull image from registry.goharbor.io instead of dockerhub
   Update testcase to support Docker Image Can Be Pulled With Credential
   Change gitlab project name when user changed.
   Update permissions count and permission count total
   Change webhook_endpoint_ui

Signed-off-by: stonezdj <stone.zhang@broadcom.com>
Co-authored-by: Wang Yan <wangyan@vmware.com>
This commit is contained in:
stonezdj(Daojun Zhang) 2024-10-30 20:07:01 +08:00 committed by GitHub
parent 3dbfd4229b
commit 9e55afbb9a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 63 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
tests/apitests/python/library/cnab.py
View File

@ -38,9 +38,6 @@ def cnab_push_bundle(bundle_file, target):
def push_cnab_bundle(harbor_server, user, password, service_image, invocation_image, target, auto_update_bundle = True):
docker_api.docker_info_display()
#Add docker login command to avoid pull request access rate elimitation by docker hub
docker_api.docker_login_cmd("", DOCKER_USER, DOCKER_PWD, enable_manifest = False)
docker_api.docker_login_cmd(harbor_server, user, password, enable_manifest = False)
bundle_file = load_bundle(service_image, invocation_image)
fixed_bundle_file = cnab_fixup_bundle(bundle_file, target, auto_update_bundle = auto_update_bundle)

2
tests/apitests/python/test_push_cnab_bundle.py
View File

@ -71,7 +71,7 @@ class TestCNAB(unittest.TestCase):
#3. Push bundle to harbor as repository(RA);
target = harbor_server + "/" + TestCNAB.project_name + "/" + TestCNAB.cnab_repo_name + ":" + TestCNAB.cnab_tag
TestCNAB.reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, TestCNAB.user_name, TestCNAB.user_push_cnab_password, "goharbor/harbor-log:v1.10.0", "kong:latest", target)
TestCNAB.reference_sha256 = library.cnab.push_cnab_bundle(harbor_server, TestCNAB.user_name, TestCNAB.user_push_cnab_password, "registry.goharbor.io/nightly/goharbor/harbor-log:v1.10.0", "registry.goharbor.io/nightly/library/kong:latest", target)
#4. Get repository from Harbor successfully;
TestCNAB.cnab_bundle_data = TestCNAB.repo.get_repository(TestCNAB.project_name, TestCNAB.cnab_repo_name, **TestCNAB.USER_CLIENT)

27
tests/resources/Docker-Util.robot
View File

@ -261,6 +261,19 @@ Docker Image Can Not Be Pulled
Log To Console Cannot Pull Image From Docker - Pull Log: ${out[1]}
Should Be Equal As Strings '${out[0]}' 'PASS'
Docker Image Can Not Be Pulled With Credential
[Arguments] ${server} ${username} ${password} ${image}
FOR ${idx} IN RANGE 0 30
${out}= Run Keyword And Ignore Error Docker Login ${server} ${username} ${password}
Log To Console Return value is ${out}
${out}= Run Keyword And Ignore Error Command Should be Failed docker pull ${image}
Exit For Loop If '${out[0]}'=='PASS'
Log To Console Docker pull return value is ${out}
Sleep 3
END
Log To Console Cannot Pull Image From Docker - Pull Log: ${out[1]}
Should Be Equal As Strings '${out[0]}' 'PASS'
Docker Image Can Be Pulled
[Arguments] ${image} ${period}=60 ${times}=2
FOR ${n} IN RANGE 1 ${times}
@ -274,3 +287,17 @@ Docker Image Can Be Pulled
END
Run Keyword If '${out[0]}'=='FAIL' Capture Page Screenshot
Should Be Equal As Strings '${out[0]}' 'PASS'
Docker Image Can Be Pulled With Credential
[Arguments] ${server} ${username} ${password} ${image} ${period}=60 ${times}=2
FOR ${n} IN RANGE 1 ${times}
Sleep ${period}
${out}= Run Keyword And Ignore Error Docker Login ${server} ${username} ${password}
Log To Console Return value is ${out}
${out}= Run Keyword And Ignore Error Docker Pull ${image}
Log To Console Return value is ${out[0]}
Exit For Loop If '${out[0]}'=='PASS'
Sleep 5
END
Run Keyword If '${out[0]}'=='FAIL' Capture Page Screenshot
Should Be Equal As Strings '${out[0]}' 'PASS'

8
tests/resources/Harbor-Pages/Configuration.robot
View File

@ -334,13 +334,19 @@ Select Provider
Retry Element Click ${distribution_provider_select_id}
Retry Element Click ${distribution_provider_select_id}//option[contains(.,'${provider}')]
Set Authcode
[Arguments] ${auth_code}
Retry Element Click ${distribution_provider_authmode_id}
Retry Text Input ${distribution_provider_authcode_id} ${auth_code}
Create An New Distribution
[Arguments] ${provider} ${name} ${endpoint}
[Arguments] ${provider} ${name} ${endpoint} ${auth_token}
Switch To Distribution
Retry Element Click ${distribution_add_btn_id}
Select Provider ${provider}
Retry Text Input ${distribution_name_input_id} ${name}
Retry Text Input ${distribution_endpoint_id} ${endpoint}
Set Authcode ${auth_token}
Retry Double Keywords When Error Retry Element Click ${distribution_add_save_btn_id} Retry Wait Until Page Not Contains Element xpath=${distribution_add_save_btn_id}
Distribution Exist ${name} ${endpoint}

2
tests/resources/Harbor-Pages/Configuration_Elements.robot
View File

@ -45,6 +45,8 @@ ${cfg_auth_ldap_group_admin_dn} //*[@id='ldapGroupAdminDN']
${distribution_add_btn_id} //*[@id='new-instance']
${distribution_provider_select_id} //*[@id='provider']
${distribution_provider_authmode_id} //clr-main-container//clr-radio-container//label[contains(@class,'clr-control-label') and contains(.,'OAuth')]
${distribution_provider_authcode_id} //*[@id='auth_data_token']
${distribution_name_input_id} //*[@id='name']
${distribution_endpoint_id} //*[@id='endpoint']
${distribution_description_id} //*[@id='description']

4
tests/resources/Harbor-Pages/Project_Robot_Account.robot
View File

@ -24,8 +24,8 @@ Create A Project Robot Account
${permission_count}= Create Dictionary
${total}= Set Variable 0
IF '${first_resource}' == 'all'
Set To Dictionary ${permission_count} all=59
${total}= Set Variable 59
Set To Dictionary ${permission_count} all= 68
${total}= Set Variable 68
Retry Element Click //span[text()='Select all']
ELSE
FOR ${item} IN @{resources}

8
tests/resources/TestCaseBody.robot
View File

@ -302,11 +302,13 @@ Body Of Replication Of Push Images to Registry Triggered By Event
Select Rule rule${d}
${endpoint_body}= Fetch From Right ${endpoint} //
${dest_namespace}= Set Variable If '${provider}'=='gitlab' ${endpoint_body}/${dest_namespace} ${dest_namespace}
Run Keyword If '${provider}'=='docker-hub' or '${provider}'=='gitlab' Docker Image Can Be Pulled ${dest_namespace}/${image}:${tag1} times=3
Run Keyword If '${provider}'=='docker-hub' Docker Image Can Be Pulled ${dest_namespace}/${image}:${tag1} times=3
Run Keyword If '${provider}'=='gitlab' Docker Image Can Be Pulled With Credential registry.gitlab.com ${username} ${pwd} ${dest_namespace}/${image}:${tag1} times=3
Executions Result Count Should Be Succeeded event_based 1
Go Into Project project${d}
Delete Repo project${d} ${image}
Run Keyword If '${provider}'=='docker-hub' or '${provider}'=='gitlab' Docker Image Can Not Be Pulled ${dest_namespace}/${image}:${tag1}
Run Keyword If '${provider}'=='docker-hub' Docker Image Can Not Be Pulled ${dest_namespace}/${image}:${tag1}
Run Keyword If '${provider}'=='gitlab' Docker Image Can Not Be Pulled With Credential registry.gitlab.com ${username} ${pwd} ${dest_namespace}/${image}:${tag1}
Switch To Replication Manage
Filter Replication Rule rule${d}
Select Rule rule${d}
@ -669,7 +671,7 @@ Create Schedules For Job Service Dashboard Schedules
Add A Tag Retention Rule
Set Tag Retention Policy Schedule ${schedule_type} ${schedule_cron}
# Create a preheat policy triggered by schedule
Create An New Distribution Dragonfly ${distribution_name} ${distribution_endpoint}
Create An New Distribution Dragonfly ${distribution_name} ${distribution_endpoint} ${DRAGONFLY_AUTH_TOKEN}
Go Into Project ${project_name}
Create An New P2P Preheat Policy ${p2p_policy_name} ${distribution_name} ** ** Scheduled ${schedule_type} ${schedule_cron}
# Create a replication policy triggered by schedule

6
tests/robot-cases/Group1-Nightly/Common.robot
View File

@ -723,7 +723,7 @@ Test Case - System Robot Account Cover All Projects
Pull image ${ip} '${robot_account_name}' ${token} project${d} hello-world:latest
Retry Action Keyword Check System Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} all
Retry Action Keyword Check Project Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_id} ${project_name} hello-world latest all
Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 53 PERMISSION(S) '] and .//span[contains(.,'Never Expires')] and .//clr-dg-cell[text()='For testing'] ]
Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 72 PERMISSION(S) '] and .//span[contains(.,'Never Expires')] and .//clr-dg-cell[text()='For testing'] ]
System Robot Account Exist ${robot_account_name} all
Close Browser
@ -739,13 +739,13 @@ Test Case - System Robot Account
${project_id}= Set Variable ${words}[-2]
Switch To Robot Account
${robot_account_name} ${token}= Create A System Robot Account sys1${d} days days=100 description=For testing cover_all_system_resources=${true}
Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 53 PERMISSION(S) '] and .//span[contains(.,'99d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']]
Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//button[text()=' 72 PERMISSION(S) '] and .//span[contains(.,'99d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']]
Retry Action Keyword Check System Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} all
Retry Action Keyword Check Project Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_id} ${project_name} hello-world latest all 1
${robot_account_name} ${token}= Create A System Robot Account sys2${d} days days=2 description=For testing cover_all_project_resources=${true}
Push image ${ip} '${robot_account_name}' ${token} project${d} hello-world:latest
Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//span[text()='All projects with'] and .//button[text()=' 59 PERMISSION(S) '] and .//span[contains(.,'1d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']]
Retry Wait Element Visible //clr-dg-row[.//clr-dg-cell[contains(.,'${robot_account_name}')] and .//clr-icon[contains(@class, 'color-green')] and .//span[text()='All projects with'] and .//button[text()=' 68 PERMISSION(S) '] and .//span[contains(.,'1d 23h')] and .//clr-dg-cell[text()='For testing'] and .//clr-dg-cell//span[text()=' None ']]
Retry Action Keyword Check System Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} all 1
Retry Action Keyword Check Project Robot Account API Permission ${robot_account_name} ${token} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_id} ${project_name} hello-world latest all
Close Browser

8
tests/robot-cases/Group1-Nightly/P2P_Preheat.robot
View File

@ -29,7 +29,7 @@ Test Case - Distribution CRUD
${endpoint_new}= Set Variable https://10.65.65.42
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Distribution Dragonfly ${name} ${endpoint}
Create An New Distribution Dragonfly ${name} ${endpoint} ${DRAGONFLY_AUTH_TOKEN}
Edit A Distribution ${name} ${endpoint} new_endpoint=${endpoint_new}
Delete A Distribution ${name} ${endpoint_new}
Close Browser
@ -45,7 +45,7 @@ Test Case - P2P Preheat Policy CRUD
${tag}= Set Variable v1.0
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Distribution Dragonfly ${dist_name} ${endpoint}
Create An New Distribution Dragonfly ${dist_name} ${endpoint} ${DRAGONFLY_AUTH_TOKEN}
Create An New Project And Go Into Project ${project_name}
Create An New P2P Preheat Policy ${policy_name} ${dist_name} ${repo} ${tag}
Edit A P2P Preheat Policy ${policy_name} ${repo_new}
@ -67,7 +67,7 @@ Test Case - P2P Preheat By Manual
${tag2}= Set Variable stable
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Distribution Dragonfly ${dist_name} ${DISTRIBUTION_ENDPOINT}
Create An New Distribution Dragonfly ${dist_name} ${DISTRIBUTION_ENDPOINT} ${DRAGONFLY_AUTH_TOKEN}
Create An New Project And Go Into Project ${project_name}
Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_name} ${image1} ${tag1} ${tag1}
Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_name} ${image1} ${tag2} ${tag2}
@ -93,7 +93,7 @@ Test Case - P2P Preheat By Event
${label}= Set Variable p2p_preheat
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Distribution Dragonfly ${dist_name} ${DISTRIBUTION_ENDPOINT}
Create An New Distribution Dragonfly ${dist_name} ${DISTRIBUTION_ENDPOINT} ${DRAGONFLY_AUTH_TOKEN}
Create An New Project And Go Into Project ${project_name}
Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_name} ${image1} ${tag1} ${tag1}
Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_name} ${image1} ${tag2} ${tag2}

5
tests/robot-cases/Group1-Nightly/Replication.robot
View File

@ -299,10 +299,11 @@ Test Case - Replication Of Pull Images from Gitlab To Self
${image1}= Get From Dictionary ${image1_with_tag} image
${image2}= Get From Dictionary ${image2_with_tag} image
@{target_images}= Create List '&{image1_with_tag}' '&{image2_with_tag}'
Body Of Replication Of Pull Images from Registry To Self gitlab https://registry.gitlab.com ${gitlab_id} ${gitlab_key} dannylunsa/test_replication/{${image1},${image2}} ${null} N Flatten All Levels @{target_images}
#harbor424542/harbor-ci is the project created in gitlab by user stonezdj, change it when the gitlab user changed
Body Of Replication Of Pull Images from Registry To Self gitlab https://registry.gitlab.com ${gitlab_id} ${gitlab_key} harbor424542/harbor-ci/{${image1},${image2}} ${null} N Flatten All Levels @{target_images}
Test Case - Replication Of Push Images to Gitlab Triggered By Event
Body Of Replication Of Push Images to Registry Triggered By Event gitlab https://registry.gitlab.com ${gitlab_id} ${gitlab_key} dannylunsa/test_replication
Body Of Replication Of Push Images to Registry Triggered By Event gitlab https://registry.gitlab.com ${gitlab_id} ${gitlab_key} harbor424542/harbor-ci
Test Case - Replication Of Pull Manifest List and CNAB from Harbor To Self
&{image1_with_tag}= Create Dictionary image=busybox tag=1.32.0 total_artifact_count=9 archive_count=0

2
tests/robot-cases/Group1-Nightly/Schedule.robot
View File

@ -203,7 +203,7 @@ Test Case - P2P Preheat Schedule Job
${image}= Set Variable busybox
${tag}= Set Variable latest
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Distribution Dragonfly ${dist_name} ${DISTRIBUTION_ENDPOINT}
Create An New Distribution Dragonfly ${dist_name} ${DISTRIBUTION_ENDPOINT} ${DRAGONFLY_AUTH_TOKEN}
Create An New Project And Go Into Project ${project_name}
Push Image With Tag ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_name} ${image} ${tag}
Create An New P2P Preheat Policy ${policy_name} ${dist_name} ** **

16
tests/robot-cases/Group1-Nightly/Webhook.robot
View File

@ -43,7 +43,7 @@ Test Case - Artifact Event Type Webhook Functionality
${image}= Set Variable busybox
${tag}= Set Variable latest
${d}= Get Current Date result_format=%m%s
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -70,7 +70,7 @@ Test Case - Scan Event Type Webhook Functionality
${image2}= Set Variable goharbor/harbor-e2e-engine
${tag2}= Set Variable latest-api
${d}= Get Current Date result_format=%m%s
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -96,7 +96,7 @@ Test Case - Tag Retention And Replication Event Type Webhook Functionality
${tag1}= Set Variable latest
${tag2}= Set Variable stable
${d}= Get Current Date result_format=%m%s
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -124,7 +124,7 @@ Test Case - Tag Retention And Replication Event Type Webhook Functionality
Test Case - Tag Quota Event Type Webhook Functionality
[Tags] quota_webhook need_webhook_endpoint
Init Chrome Driver
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -141,7 +141,7 @@ Test Case - Artifact Event Type Webhook Functionality By CloudEvents Format
${tag}= Set Variable latest
${payload_format}= Set Variable CloudEvents
${d}= Get Current Date result_format=%m%s
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -169,7 +169,7 @@ Test Case - Scan Event Type Webhook Functionality By CloudEvents Format
${tag2}= Set Variable 5.0.0-api
${payload_format}= Set Variable CloudEvents
${d}= Get Current Date result_format=%m%s
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -196,7 +196,7 @@ Test Case - Tag Retention And Replication Event Type Webhook Functionality By Cl
${tag2}= Set Variable stable
${payload_format}= Set Variable CloudEvents
${d}= Get Current Date result_format=%m%s
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -225,7 +225,7 @@ Test Case - Tag Quota Event Type Webhook Functionality By CloudEvents Format
[Tags] quota_webhook_cloudevents need_webhook_endpoint
${payload_format}= Set Variable CloudEvents
Init Chrome Driver
Go To http://${WEBHOOK_ENDPOINT}
Go To http://${WEBHOOK_ENDPOINT_UI}
${webhook_endpoint_url}= Get Text //p//code
New Tab
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}