Upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor.git synced 2024-10-02 07:18:01 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #10499 from danfengliu/add-upgrade-verification

Browse Source 
Add robot-account verification for upgrade test
This commit is contained in:
danfengliu 2020-04-16 22:58:40 +08:00 committed by GitHub
commit 62c56d76b1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 309 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tests/resources/Harbor-Pages/Project-Webhooks_Elements.robot
View File

@ -16,6 +16,7 @@ ${dialog_delete_button} xpath=//clr-modal//button[contains(.,'DELETE')]
${create_webhooks_continue_button_xpath} xpath=//*[@id='new-webhook-continue']
${edit_webhooks_cancel_button_xpath} xpath=//*[@id='edit-webhook-cancel']
${edit_webhooks_save_button_xpath} xpath=//*[@id='edit-webhook-save']
${edit_webhooks_verify_remote_cert_chkb_xpath} xpath=//input[@id='verify_remote_cert']
${project_webhook_edit_id_xpath} xpath=//*[@id='edit-webhook']
${project_webhook_enable_id_xpath} xpath=//*[@id='enable-webhook-action']
${project_webhook_disable_id_xpath} xpath=//*[@id='disable-webhook-action']

2
tests/resources/Harbor-Pages/Project_Elements.robot
View File

@ -67,6 +67,8 @@ ${project_config_public_checkbox_label} xpath=//*[@id="clr-wrapper-public"]/div
${project_config_prevent_vulenrability_checkbox_label} xpath=//*[@id='prevent-vulenrability-image']//clr-checkbox-wrapper//label
${project_config_system_wl_radio_input} xpath=//clr-radio-wrapper//label[contains(.,'System whitelist')]
${project_config_project_wl_radio_input} xpath=//clr-radio-wrapper//label[contains(.,'Project whitelist')]
${project_config_system_wl_radio_input_id} systemWhitelistOrProjectWhitelist
${project_config_project_wl_radio_input_id} systemWhitelistOrProjectWhitelist
${project_config_project_wl_add_btn} xpath=//*[@id='show-add-modal']
${project_config_project_wl_add_confirm_btn} xpath=//*[@id='add-to-whitelist']
${project_config_save_btn} xpath=//hbr-project-policy-config//button[contains(.,'SAVE')]

153
tests/resources/Harbor-Pages/Verify.robot
View File

@ -6,6 +6,7 @@ Resource ../../resources/Util.robot
Verify User
[Arguments] ${json}
Log To Console "Verify User..."
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Switch To User Tag
@ -22,6 +23,7 @@ Verify User
Verify Project
[Arguments] ${json}
Log To Console "Verify Project..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -32,6 +34,7 @@ Verify Project
Verify Image Tag
[Arguments] ${json}
Log To Console "Verify Image Tag..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -65,9 +68,10 @@ Verify Project Metadata
Close Browser
Verify Checkbox
[Arguments] ${json} ${key} ${checkbox}
[Arguments] ${json} ${key} ${checkbox} ${is_opposite}=${false}
@{out}= Get Value From Json ${json} ${key}
Run Keyword If '@{out}[0]'=='true' Checkbox Should Be Selected ${checkbox}
${value}= Set Variable If '${is_opposite}'=='${true}' 'false' 'true'
Run Keyword If '@{out}[0]'==${value} Checkbox Should Be Selected ${checkbox}
... ELSE Checkbox Should Not Be Selected ${checkbox}
@ -78,6 +82,7 @@ Loop Image Repo
Verify Member Exist
[Arguments] ${json}
Log To Console "Verify Member Exist..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -91,13 +96,108 @@ Verify Member Exist
\ Navigate To Projects
Close Browser
Verify Webhook
[Arguments] ${json}
Log To Console "Verify Webhook..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
:For ${project} In @{project}
\ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image
\ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false}
\ Go Into Project ${project} has_image=${has_image}
\ Switch To Project Webhooks
\ @{enabled}= Get Value From Json ${json} $.projects[?(@.name=${project})].webhook.enabled
\ ${enable_count} Get Matching Xpath Count xpath=//span[contains(.,'Enabled')]
\ ${disable_count} Get Matching Xpath Count xpath=//span[contains(.,'Disabled')]
\ Log To Console '@{enabled}[0]'
\ Log To Console '${true}'
\ Run Keyword If '@{enabled}[0]' == '${true}' Page Should Contain Enabled
\ ... ELSE Page Should Contain Disabled
\ @{address}= Get Value From Json ${json} $.projects[?(@.name=${project})].webhook.address
\ Log To Console '@{address}[0]'
\ Page Should Contain @{address}[0]
\ Page Should Contain policy
\ Page Should Contain http
\ Navigate To Projects
Close Browser
Verify Tag Retention Rule
[Arguments] ${json}
Log To Console "Verify Tag Retention Rule..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
:For ${project} In @{project}
\ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image
\ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false}
\ Go Into Project ${project} has_image=${has_image}
\ Switch To Tag Retention
\ ${actions_count}= Set Variable 8
\ @{repository_patten}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_retention_rule.repository_patten
\ @{tag_decoration}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_retention_rule.tag_decoration
\ @{latestPushedK}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_retention_rule.latestPushedK
\ @{cron}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_retention_rule.cron
\ Log To Console '@{repository_patten}[0]'
\ Page Should Contain @{repository_patten}[0]
\ Page Should Contain @{tag_decoration}[0]
\ Page Should Contain @{latestPushedK}[0]
\ Page Should Contain @{cron}[0]
\ Navigate To Projects
Close Browser
Verify Tag Immutability Rule
[Arguments] ${json}
Log To Console "Verify Tag Immutability Rule..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
:For ${project} In @{project}
\ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image
\ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false}
\ Go Into Project ${project} has_image=${has_image}
\ Switch To Tag Immutability
\ @{repo_decoration}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_immutability_rule.repo_decoration
\ @{tag_decoration}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_immutability_rule.tag_decoration
\ @{repo_pattern}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_immutability_rule.repo_pattern
\ @{tag_pattern}= Get Value From Json ${json} $.projects[?(@.name=${project})].tag_immutability_rule.tag_pattern
\ Log To Console '@{repo_decoration}[0]'
#\ Page Should Contain @{repo_decoration}[0]
#\ Page Should Contain @{tag_decoration}[0]
\ Page Should Contain @{repo_pattern}[0]
\ Page Should Contain @{tag_pattern}[0]
\ Navigate To Projects
Close Browser
Loop Member
[Arguments] @{members}
:For ${member} In @{members}
\ Page Should Contain ${member}
Verify Robot Account Exist
[Arguments] ${json}
Log To Console "Verify Robot Account Exist..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
:For ${project} In @{project}
\ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image
\ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false}
\ Go Into Project ${project} has_image=${has_image}
\ Switch To Project Robot Account
\ @{robot_accounts}= Get Value From Json ${json} $.projects[?(@.name=${project})].robot_account..name
\ Loop Verify Robot Account @{robot_accounts}
\ Navigate To Projects
Close Browser
Loop Verify Robot Account
[Arguments] @{robot_accounts}
:For ${robot_account} In @{robot_accounts}
\ Page Should Contain ${robot_account}
Verify User System Admin Role
[Arguments] ${json}
Log To Console "Verify User System Admin Role..."
@{user}= Get Value From Json ${json} $.admin..name
Init Chrome Driver
:FOR ${user} IN @{user}
@ -108,6 +208,7 @@ Verify User System Admin Role
Verify System Label
[Arguments] ${json}
Log To Console "Verify System Label..."
@{label}= Get Value From Json ${json} $..syslabel..name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -119,6 +220,7 @@ Verify System Label
Verify Project Label
[Arguments] ${json}
Log To Console "Verify Project Label..."
@{project}= Get Value From Json ${json} $.peoject.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -135,6 +237,7 @@ Verify Project Label
Verify Endpoint
[Arguments] ${json}
Log To Console "Verify Endpoint..."
@{endpoint}= Get Value From Json ${json} $.endpoint..name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
@ -145,6 +248,7 @@ Verify Endpoint
Verify Replicationrule
[Arguments] ${json}
Log To Console "Verify Replicationrule..."
@{replicationrules}= Get Value From Json ${json} $.replicationrule.[*].rulename
@{endpoints}= Get Value From Json ${json} $.endpoint.[*].name
: FOR ${replicationrule} IN @{replicationrules}
@ -183,6 +287,7 @@ Verify Replicationrule
Verify Project Setting
[Arguments] ${json}
Log To Console "Verify Project Setting..."
@{projects}= Get Value From Json ${json} $.projects.[*].name
:For ${project} In @{Projects}
\ ${public}= Get Value From Json ${json} $.projects[?(@.name=${project})].accesslevel
@ -206,6 +311,7 @@ Verify Project Setting
Verify System Setting
[Arguments] ${json}
Log To Console "Verify System Setting..."
@{authtype}= Get Value From Json ${json} $.configuration.authmode
@{creation}= Get Value From Json ${json} $.configuration..projectcreation
@{selfreg}= Get Value From Json ${json} $.configuration..selfreg
@ -215,6 +321,7 @@ Verify System Setting
@{emailfrom}= Get Value From Json ${json} $.configuration..emailfrom
@{token}= Get Value From Json ${json} $.configuration..token
@{scanschedule}= Get Value From Json ${json} $.configuration..scanall
@{cve_ids}= Get Value From Json ${json} $.configuration..cve
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Switch To Configure
@ -235,3 +342,45 @@ Verify System Setting
#Page Should Contain None
Close Browser
Verify Project-level Whitelist
[Arguments] ${json}
Log To Console "Verify Project-level Whitelist..."
@{project}= Get Value From Json ${json} $.projects.[*].name
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
:FOR ${project} IN @{project}
\ @{out_has_image}= Get Value From Json ${json} $.projects[?(@.name=${project})].has_image
\ ${has_image} Set Variable If @{out_has_image}[0] == ${true} ${true} ${false}
\ Go Into Project ${project} has_image=${has_image}
\ Switch To Project Configuration
\ @{is_reuse_sys_cve_whitelist}= Get Value From Json ${json} $.projects[?(@.name=${project})].configuration.reuse_sys_cve_whitelist
\ Run Keyword If "@{is_reuse_sys_cve_whitelist}[0]" == "true" Retry Wait Element Should Be Disabled ${project_config_project_wl_add_btn}
\ ... ELSE Retry Wait Element ${project_config_project_wl_add_btn}
\ @{cve_ids}= Get Value From Json ${json} $.projects[?(@.name=${project})].configuration.cve
\ Loop Verifiy CVE_IDs @{cve_ids}
\ Navigate To Projects
Close Browser
Loop Verifiy CVE_IDs
[Arguments] @{cve_ids}
:For ${cve_id} In @{cve_ids}
\ Page Should Contain ${cve_id}
Verify System Setting Whitelist
[Arguments] ${json}
Log To Console "Verify Verify System Setting Whitelist..."
@{cve_ids}= Get Value From Json ${json} $.configuration..cve..id
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Switch To Configure
Switch To System Settings
Log To Console "@{cve_ids}"
Loop Verifiy CVE_IDs @{cve_ids}
Close Browser
Verify Clair Is Default Scanner
Init Chrome Driver
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Switch To Scanners Page
Should Display The Default Clair Scanner
Close Browser

32
tests/robot-cases/Group3-Upgrade/data.json
View File

@ -208,6 +208,18 @@
]
}
],
"tag_retention_rule":{
"repository_patten":"*photon*",
"tag_decoration":"*v1.*",
"latestPushedK":666,
"cron":"0 0 0 1 8 0"
},
"tag_immutability_rule":{
"repo_decoration":"repoMatches",
"tag_decoration":"excludes",
"repo_pattern":"*alpine*",
"tag_pattern":"*v1.2*"
},
"webhook":{
"address":"https://1.1.1.1",
"skip_cert_verify":true,
@ -327,11 +339,23 @@
]
}
],
"tag_retention_rule":{
"repository_patten":"*centos*",
"tag_decoration":"*latest",
"latestPushedK":999,
"cron":"0 0 0 1 6 0"
},
"tag_immutability_rule":{
"repo_decoration":"repoExcludes",
"tag_decoration":"matches",
"repo_pattern":"*buxybox*",
"tag_pattern":"*v3*"
},
"webhook":{
"address":"https://1.1.1.1",
"skip_cert_verify":true,
"auth_header":"aaa",
"enabled":true
"address":"https://2.2.2.2",
"skip_cert_verify":false,
"auth_header":"bbb",
"enabled":false
},
"configuration":{
"public":"false",

106
tests/robot-cases/Group3-Upgrade/prepare.py
View File

@ -210,6 +210,7 @@ class HarborAPI:
}
}
}
print payload
body=dict(body=payload)
request(url+"configurations", 'put', **body)
@ -253,11 +254,98 @@ class HarborAPI:
request(url+"projects/"+projectid+"/robots", 'post', **body)
@get_feature_branch
def add_tag_retention_rule(self, project, robot_account, **kwargs):
return
def add_tag_retention_rule(self, project, tag_retention_rule, **kwargs):
r = request(url+"projects?name="+project+"", 'get')
projectid = str(r.json()[0]['project_id'])
if kwargs["branch"] == 1:
payload = {
"algorithm":"or",
"rules":
[
{
"disabled":False,
"action":"retain",
"scope_selectors":
{
"repository":
[
{
"kind":"doublestar",
"decoration":"repoMatches",
"pattern":tag_retention_rule["repository_patten"]
}
]
},
"tag_selectors":
[
{
"kind":"doublestar",
"decoration":"matches","pattern":tag_retention_rule["tag_decoration"]
}
],
"params":{"latestPushedK":tag_retention_rule["latestPushedK"]},
"template":"latestPushedK"
}
],
"trigger":
{
"kind":"Schedule",
"references":{},
"settings":{"cron":tag_retention_rule["cron"]}
},
"scope":
{
"level":"project",
"ref":int(projectid)
}
}
print payload
body=dict(body=payload)
request(url+"retentions", 'post', **body)
else:
raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"]))
@get_feature_branch
def add_webhook(self, webhook, **kwargs):
def add_tag_immutability_rule(self, project, tag_immutability_rule, **kwargs):
r = request(url+"projects?name="+project+"", 'get')
projectid = str(r.json()[0]['project_id'])
if kwargs["branch"] == 1:
payload = {
"disabled":False,
"action":"immutable",
"scope_selectors":
{
"repository":
[
{
"kind":"doublestar",
"decoration":tag_immutability_rule["repo_decoration"],
"pattern":tag_immutability_rule["repo_pattern"]
}
]
},
"tag_selectors":
[
{
"kind":"doublestar",
"decoration":tag_immutability_rule["tag_decoration"],
"pattern":tag_immutability_rule["tag_pattern"]
}
],
"project_id":int(projectid),
"priority":0,
"template":"immutable_template"
}
print payload
body=dict(body=payload)
request(url+"projects/"+projectid+"/immutabletagrules", 'post', **body)
else:
raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"]))
@get_feature_branch
def add_webhook(self, project, webhook, **kwargs):
r = request(url+"projects?name="+project+"", 'get')
projectid = str(r.json()[0]['project_id'])
if kwargs["branch"] == 1:
payload = {
"targets":[
@ -278,15 +366,17 @@ class HarborAPI:
"scanningFailed",
"scanningCompleted"
],
"enabled":+webhook["enabled"]
"enabled":webhook["enabled"]
}
print payload
body=dict(body=payload)
request(url+"system/CVEWhitelist", 'put', **body)
request(url+"projects/"+projectid+"/webhook/policies", 'post', **body)
else:
raise Exception(r"Error: Feature {} has no branch {}.".format(sys._getframe().f_code.co_name, kwargs["branch"]))
def update_repoinfo(self, reponame):
payload = {"description": "testdescription"}
print payload
body=dict(body=payload)
request(url+"repositories/"+reponame+"", 'put', **body)
@ -318,7 +408,7 @@ def request(url, method, user = None, userp = None, **kwargs):
kwargs['headers']['Content-Type'] = 'application/json'
kwargs['data'] = json.dumps(kwargs['body'])
del kwargs['body']
print "url: ", url
resp = requests.request(method, url, verify=False, auth=(user, userp), **kwargs)
if resp.status_code >= 400:
raise Exception("[Exception Message] - {}".format(resp.text))
@ -355,7 +445,9 @@ def do_data_creation():
harborAPI.add_member(project["name"], member["name"], member["role"], version=args.version)
for robot_account in project["robot_account"]:
harborAPI.add_project_robot_account(project["name"], robot_account, version=args.version)
harborAPI.add_webhook(project["webhook"], version=args.version)
harborAPI.add_webhook(project["name"], project["webhook"], version=args.version)
harborAPI.add_tag_retention_rule(project["name"], project["tag_retention_rule"], version=args.version)
harborAPI.add_tag_immutability_rule(project["name"], project["tag_immutability_rule"], version=args.version)
pull_image("busybox", "redis", "haproxy", "alpine", "httpd:2")
push_image("busybox", data["projects"][0]["name"])

43
tests/robot-cases/Group3-Upgrade/verify.robot
View File

@ -11,24 +11,15 @@ ${HARBOR_ADMIN} admin
Test Case - Upgrade Verify
[Tags] 1.8-latest
${data}= Load Json From File ${CURDIR}${/}data.json
Log To Console "Verify User..."
Run Keyword Verify User ${data}
Log To Console "Verify Project..."
Run Keyword Verify Project ${data}
Log To Console "Verify Member Exist..."
Run Keyword Verify Member Exist ${data}
#Run Keyword Verify Robot Account Exist ${data}
Log To Console "Verify User System Admin Role..."
Run Keyword Verify Robot Account Exist ${data}
Run Keyword Verify User System Admin Role ${data}
Log To Console "Verify Endpoint..."
Run Keyword Verify Endpoint ${data}
Log To Console "Verify Replicationrule..."
Run Keyword Verify Replicationrule ${data}
Log To Console "Verify Project Setting..."
Run Keyword Verify Project Setting ${data}
Log To Console "Verify System Setting..."
Run Keyword Verify System Setting ${data}
Log To Console "Verify Image Tag..."
Run Keyword Verify Image Tag ${data}
Test Case - Upgrade Verify
@ -37,13 +28,35 @@ Test Case - Upgrade Verify
Run Keyword Verify User ${data}
Run Keyword Verify Project ${data}
Run Keyword Verify Member Exist ${data}
#Run Keyword Verify Robot Account Exist ${data}
#Run Keyword Verify Project-level Whitelist ${data}
#Run Keyword Verify Webhook ${data}
Run Keyword Verify Robot Account Exist ${data}
Run Keyword Verify Project-level Whitelist ${data}
Run Keyword Verify Webhook ${data}
Run Keyword Verify Tag Retention Rule ${data}
Run Keyword Verify User System Admin Role ${data}
Run Keyword Verify Endpoint ${data}
Run Keyword Verify Replicationrule ${data}
Run Keyword Verify Project Setting ${data}
Run Keyword Verify System Setting ${data}
#Run Keyword Verify System Setting Whitelist ${data}
Run Keyword Verify Image Tag ${data}
Run Keyword Verify System Setting Whitelist ${data}
Run Keyword Verify Image Tag ${data}
Run Keyword Verify Clair Is Default Scanner
Test Case - Upgrade Verify
[Tags] 1.10-latest
${data}= Load Json From File ${CURDIR}${/}data.json
Run Keyword Verify User ${data}
Run Keyword Verify Project ${data}
Run Keyword Verify Member Exist ${data}
Run Keyword Verify Robot Account Exist ${data}
Run Keyword Verify Project-level Whitelist ${data}
Run Keyword Verify Webhook ${data}
Run Keyword Verify Tag Retention Rule ${data}
Run Keyword Verify Tag Immutability Rule ${data}
Run Keyword Verify User System Admin Role ${data}
Run Keyword Verify Endpoint ${data}
Run Keyword Verify Replicationrule ${data}
Run Keyword Verify Project Setting ${data}
Run Keyword Verify System Setting ${data}
Run Keyword Verify System Setting Whitelist ${data}
Run Keyword Verify Image Tag ${data}
Run Keyword Verify Clair Is Default Scanner