Upstream
/
harbor
mirror of https://github.com/goharbor/harbor.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add replication schedule test 

1. Add replication schedule test;
2. Add some sleep after project creation before push image to avoid push image v2 error.
   It fixed issue# 12094: Scan image vmware/photon:1.0 return unsupported occasionally.
3. Add some sleep in test_user_view_logs.py, can't get operation logs sometimes.

Signed-off-by: danfengliu <danfengl@vmware.com>
This commit is contained in:
danfengliu 2020-05-29 16:17:10 +08:00
parent f862805244
commit ceaa0a57b3
12 changed files with 112 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -513,7 +513,7 @@ down:
swagger_client:
@echo "Generate swagger client"
wget -q https://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O swagger-codegen-cli.jar
wget https://repo1.maven.org/maven2/io/swagger/swagger-codegen-cli/2.3.1/swagger-codegen-cli-2.3.1.jar -O swagger-codegen-cli.jar
rm -rf harborclient
mkdir -p harborclient/harbor_client
mkdir -p harborclient/harbor_swagger_client

15
tests/apitests/python/test_user_view_logs.py
View File

@ -1,6 +1,7 @@
from __future__ import absolute_import
import unittest
import time
from testutils import ADMIN_CLIENT
from testutils import TEARDOWN
@ -68,23 +69,25 @@ class TestProjects(unittest.TestCase):
#2.1 Create a new project(PA) by user(UA);
TestProjects.project_user_view_logs_id, project_user_view_logs_name = self.project.create_project(metadata = {"public": "false"}, **TestProjects.USER_USER_VIEW_LOGS_CLIENT)
time.sleep(2)
#2.2 In project(PA), there should be 1 'create' log record;
operation = "create"
log_count = self.projectv2.filter_project_logs(project_user_view_logs_name, user_user_view_logs_name, project_user_view_logs_name, "project", operation, **TestProjects.USER_USER_VIEW_LOGS_CLIENT)
if log_count != 1:
self.test_result.add_test_result("1 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}".
format(user_user_view_logs_name, project_user_view_logs_name, "project", operation))
self.test_result.add_test_result("1 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.".
format(user_user_view_logs_name, project_user_view_logs_name, "project", operation, log_count))
#3.1 Push a new image(IA) in project(PA) by admin;
repo_name, tag = push_image_to_project(project_user_view_logs_name, harbor_server, admin_name, admin_password, "tomcat", "latest")
time.sleep(2)
#3.2 In project(PA), there should be 1 'push' log record;
operation = "create"
log_count = self.projectv2.filter_project_logs(project_user_view_logs_name, admin_name, r'{}:{}'.format(repo_name, tag), "artifact", operation, **TestProjects.USER_USER_VIEW_LOGS_CLIENT)
if log_count != 1:
self.test_result.add_test_result("2 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}".
format(user_user_view_logs_name, project_user_view_logs_name, "artifact", operation))
self.test_result.add_test_result("2 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.".
format(user_user_view_logs_name, project_user_view_logs_name, "artifact", operation, log_count))
#4.1 Delete repository(RA) by user(UA);
self.repo.delete_repoitory(project_user_view_logs_name, repo_name.split('/')[1], **TestProjects.USER_USER_VIEW_LOGS_CLIENT)
@ -92,8 +95,8 @@ class TestProjects(unittest.TestCase):
operation = "delete"
log_count = self.projectv2.filter_project_logs(project_user_view_logs_name, user_user_view_logs_name, repo_name, "repository", operation, **TestProjects.USER_USER_VIEW_LOGS_CLIENT)
if log_count != 1:
self.test_result.add_test_result("5 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}".
format(user_user_view_logs_name, project_user_view_logs_name, "repository", operation))
self.test_result.add_test_result("5 - Failed to get log with user:{}, resource:{}, resource_type:{} and operation:{}, expect count 1, but actual is {}.".
format(user_user_view_logs_name, project_user_view_logs_name, "repository", operation, log_count))
if __name__ == '__main__':
unittest.main()

13
tests/resources/Docker-Util.robot
View File

@ -39,6 +39,7 @@ Push image
[Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${sha256}=${null} ${is_robot}=${false}
${image_with_sha256}= Set Variable If '${sha256}'=='${null}' ${image} ${image}@sha256:${sha256}
${image_with_tag}= Set Variable If '${sha256}'=='${null}' ${image} ${image}:${sha256}
Sleep 3
Log To Console \nRunning docker push ${image}...
Docker Pull ${LOCAL_REGISTRY}/${LOCAL_REGISTRY_NAMESPACE}/${image_with_sha256}
Run Keyword If ${is_robot}==${false} Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip}
@ -62,11 +63,17 @@ Cannot Docker Login Harbor
[Arguments] ${ip} ${user} ${pwd}
Command Should be Failed docker login -u ${user} -p ${pwd} ${ip}
Cannot Pull image
[Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${tag}=${null}
Cannot Pull Image
[Arguments] ${ip} ${user} ${pwd} ${project} ${image} ${tag}=${null} ${err_msg}=${null}
${image_with_tag}= Set Variable If '${tag}'=='${null}' ${image} ${image}:${tag}
Wait Unitl Command Success docker login -u ${user} -p ${pwd} ${ip}
Command Should be Failed docker pull ${ip}/${project}/${image_with_tag}
:FOR ${idx} IN RANGE 0 4
\ ${out} Run Keyword And Ignore Error Command Should be Failed docker pull ${ip}/${project}/${image_with_tag}
\ Exit For Loop If '${out[0]}'=='PASS'
\ Sleep 3
Log To Console Cannot Pull Image - Pull Log: ${out[1]}
Should Be Equal As Strings '${out[0]}' 'PASS'
Run Keyword If '${err_msg}' != '${null}' Should Contain ${out[1]} ${err_msg}
Cannot Pull Unsigned Image
[Arguments] ${ip} ${user} ${pass} ${proj} ${imagewithtag}

4
tests/resources/Harbor-Pages/Project-Members.robot
View File

@ -126,8 +126,8 @@ User Should Not Be A Member Of Project
${password}= Set Variable If ${is_oidc_mode} == ${true} ${pwd_oidc} ${pwd}
Project Should Not Display ${project}
Logout Harbor
Cannot Pull image ${ip} ${user} ${password} ${project} ${ip}/${project}/hello-world
Cannot Push image ${ip} ${user} ${password} ${project} hello-world
Cannot Pull Image ${ip} ${user} ${password} ${project} ${ip}/${project}/hello-world
Cannot Pull Image ${ip} ${user} ${password} ${project} hello-world
Manage Project Member
[Arguments] ${admin} ${pwd} ${project} ${user} ${op} ${has_image}=${true} ${is_oidc_mode}=${false}

17
tests/resources/Harbor-Pages/Project-Tag-Retention.robot
View File

@ -79,13 +79,21 @@ Set Daily Schedule
Retry Element Click xpath=${project_tag_retention_schedule_ok_xpath}
Retry Wait Until Page Contains Element xpath=${project_tag_retention_span_daily_xpath}
Execute Result Should Be
[Arguments] ${result}
:FOR ${idx} IN RANGE 0 20
\ ${out} Run Keyword And Ignore Error Retry Wait Until Page Contains Element xpath=//clr-dg-cell[contains(., '${result}')]
\ Exit For Loop If '${out[0]}'=='PASS'
\ Sleep 6
Should Be Equal As Strings '${out[0]}' 'PASS'
Execute Dry Run
Retry Element Click xpath=${project_tag_retention_dry_run_xpath}
Retry Wait Until Page Contains Element xpath=${project_tag_retention_record_yes_xpath}
Sleep 5
Retry Element Click xpath=${project_tag_retention_record_yes_xpath}
# memcached:123 should be deleted and hello-world:latest should be retained
Retry Wait Until Page Contains Element xpath=//clr-dg-cell[contains(., '0/1')]
# memcached:123 should be deleted and hello-world:latest should be retained
Execute Result Should Be 0/1
Execute Run
@ -94,5 +102,6 @@ Execute Run
Retry Wait Until Page Contains Element xpath=${project_tag_retention_record_no_xpath}
Sleep 5
Retry Element Click xpath=${project_tag_retention_record_no_xpath}
# memcached:123 should be deleted and hello-world:latest should be retained
Retry Wait Until Page Contains Element xpath=//clr-dg-cell[contains(., '0/1')]
# memcached:123 should be deleted and hello-world:latest should be retained
Execute Result Should Be 0/1

4
tests/resources/Harbor-Pages/Project.robot
View File

@ -33,6 +33,7 @@ Create An New Project And Go Into Project
Capture Page Screenshot
Retry Double Keywords When Error Retry Element Click ${create_project_OK_button_xpath} Retry Wait Until Page Not Contains Element ${create_project_OK_button_xpath}
Capture Page Screenshot
Sleep 2
Go Into Project ${projectname} has_image=${false}
Create An New Project With New User
@ -43,6 +44,9 @@ Create An New Project With New User
Create An New Project And Go Into Project ${projectname} ${public}
Sleep 1
Artifact Exist
[Arguments] ${tag_name}
Retry Wait Until Page Contains Element //artifact-list-tab//clr-datagrid//clr-dg-row[contains(.,'sha256') and contains(.,'${tag_name}')]
#It's the log of project.
Go To Project Log
#Switch To Project Tab Overflow

4
tests/resources/Harbor-Pages/Replication.robot
View File

@ -73,7 +73,7 @@ Create A New Endpoint
Create A Rule With Existing Endpoint
[Arguments] ${name} ${replication_mode} ${project_name} ${resource_type} ${endpoint} ${dest_namespace}
... ${mode}=Manual
... ${mode}=Manual ${cron}="* */59 * * * *"
#click new
Retry Element Click ${new_name_xpath}
#input name
@ -88,7 +88,7 @@ Create A Rule With Existing Endpoint
Retry Text Input ${dest_namespace_xpath} ${dest_namespace}
#set trigger
Select Trigger ${mode}
Run Keyword If '${mode}' == 'Scheduled' Log To Console Scheduled
Run Keyword If '${mode}' == 'Scheduled' Retry Text Input ${targetCron_id} ${cron}
#click save
Retry Double Keywords When Error Retry Element Click ${rule_save_button} Retry Wait Until Page Not Contains Element ${rule_save_button}
Sleep 2

16
tests/resources/TestCaseBody.robot
View File

@ -41,7 +41,7 @@ Body Of Manage project publicity
Logout Harbor
Sign In Harbor ${HARBOR_URL} user008 Test1@34
Project Should Not Display project${d}
Cannot Pull image ${ip} user008 Test1@34 project${d} hello-world:latest
Cannot Pull Image ${ip} user008 Test1@34 project${d} hello-world:latest err_msg=unauthorized to access repository
Logout Harbor
Sign In Harbor ${HARBOR_URL} user007 Test1@34
@ -240,7 +240,7 @@ Body Of Verfiy System Level CVE Whitelist
Push Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} sha256=${sha256}
Go Into Project project${d}
Set Vulnerabilty Serverity 2
Cannot Pull image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256} err_msg=current image without vulnerability scanning cannot be pulled due to configured policy
Go Into Project project${d}
Go Into Repo project${d}/${image}
Scan Repo ${sha256} Succeed
@ -250,12 +250,12 @@ Body Of Verfiy System Level CVE Whitelist
Switch To Configuration System Setting
# Add Items To System CVE Whitelist CVE-2019-19317\nCVE-2019-19646 \nCVE-2019-5188 \nCVE-2019-20387 \nCVE-2019-17498 \nCVE-2019-20372 \nCVE-2019-19244 \nCVE-2019-19603 \nCVE-2019-19880 \nCVE-2019-19923 \nCVE-2019-19925 \nCVE-2019-19926 \nCVE-2019-19959 \nCVE-2019-20218 \nCVE-2019-19232 \nCVE-2019-19234 \nCVE-2019-19645
Add Items To System CVE Whitelist ${most_cve_list}
Cannot Pull image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256} err_msg=cannot be pulled due to configured policy
# Add Items To System CVE Whitelist CVE-2019-18276
Add Items To System CVE Whitelist ${single_cve}
Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Delete Top Item In System CVE Whitelist count=6
Cannot Pull image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256} err_msg=cannot be pulled due to configured policy
Close Browser
Body Of Verfiy Project Level CVE Whitelist
@ -273,17 +273,17 @@ Body Of Verfiy Project Level CVE Whitelist
Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Go Into Project project${d}
Set Vulnerabilty Serverity 2
Cannot Pull image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Go Into Project project${d}
Go Into Repo project${d}/${image}
Scan Repo ${sha256} Succeed
Go Into Project project${d}
Add Items to Project CVE Whitelist ${most_cve_list}
Cannot Pull image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Add Items to Project CVE Whitelist ${single_cve}
Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Delete Top Item In Project CVE Whitelist
Cannot Pull image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Close Browser
Body Of Verfiy Project Level CVE Whitelist By Quick Way of Add System
@ -311,7 +311,7 @@ Body Of Verfiy Project Level CVE Whitelist By Quick Way of Add System
Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Go Into Project project${d}
Set Project To Project Level CVE Whitelist
Cannot Pull image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Add System CVE Whitelist to Project CVE Whitelist By Add System Button Click
Pull Image ${ip} ${signin_user} ${signin_pwd} project${d} ${image} tag=${sha256}
Close Browser

2
tests/robot-cases/Group1-Nightly/Clair.robot
View File

@ -102,7 +102,7 @@ Test Case - Project Level Image Serverity Policy
Navigate To Projects
Go Into Project project${d}
Set Vulnerabilty Serverity 3
Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256} err_msg=To continue with pull, please contact your project administrator to exempt matched vulnerabilities through configuring the CVE whitelist
Close Browser
#Important Note: All CVE IDs in CVE Whitelist cases must unique!

2
tests/robot-cases/Group1-Nightly/Nightly.robot
View File

@ -195,7 +195,7 @@ Test Case - Manage project publicity
Logout Harbor
Sign In Harbor ${HARBOR_URL} userb${d} Test1@34
Project Should Not Display project${d}
Cannot Pull image ${ip} userb${d} Test1@34 project${d} hello-world:latest
Cannot Pull Image ${ip} userb${d} Test1@34 project${d} hello-world:latest
Logout Harbor
Sign In Harbor ${HARBOR_URL} usera${d} Test1@34

67
tests/robot-cases/Group1-Nightly/Schedule.robot
View File

@ -28,31 +28,84 @@ Test Case - Scan Schedule Job
Init Chrome Driver
${d}= Get Current Date result_format=%M
Log To Console ${d}
${project_name}= Set Variable scan_schedule_proj
${project_name}= Set Variable scan_schedule_proj${d}
${image}= Set Variable redis
${tag}= Set Variable latest
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Project And Go Into Project ${project_name}
Push image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} ${project_name} ${image}:${tag}
Sleep 50
Go Into Repo ${project_name}/${image}
Retry Wait Until Page Contains Element ${not_scanned_icon}
Switch To Vulnerability Page
${flag}= Set Variable ${false}
:FOR ${i} IN RANGE 999999
\ ${minite}= Get Current Date result_format=%M
\ ${left} = Evaluate ${minite}%10
\ ${d} = Convert To Integer ${left}
\ Log To Console ${i}/${d}
\ Run Keyword If ${d} <= 3 Run Keywords Set Scan Schedule custom value=* */10 * * * * AND Set Suite Variable ${flag} ${true}
\ ${minite_int} = Convert To Integer ${minite}
\ ${left} = Evaluate ${minite_int}%10
\ Log To Console ${i}/${left}
\ Sleep 55
\ Run Keyword If ${left} <= 3 and ${left} != 0 Run Keywords Set Scan Schedule custom value=* */10 * * * * AND Set Suite Variable ${flag} ${true}
\ Exit For Loop If '${flag}' == '${true}'
# After scan custom schedule is set, image should stay in unscanned status.
Sleep 360
Log To Console Sleep for 300 seconds......
Sleep 300
Go Into Project ${project_name}
Go Into Repo ${project_name}/${image}
Retry Wait Until Page Contains Element ${not_scanned_icon}
Sleep 360
Log To Console Sleep for 500 seconds......
Sleep 500
Go Into Project ${project_name}
Go Into Repo ${project_name}/${image}
Scan Result Should Display In List Row ${tag}
View Repo Scan Details Critical High Medium
Test Case - Replication Schedule Job
Init Chrome Driver
${d}= Get Current Date result_format=%M
Log To Console ${d}
${project_name}= Set Variable replication_schedule_proj${d}
${image_a}= Set Variable mariadb
${tag_a}= Set Variable 111
${image_b}= Set Variable centos
${tag_b}= Set Variable 222
Sign In Harbor ${HARBOR_URL} ${HARBOR_ADMIN} ${HARBOR_PASSWORD}
Create An New Project And Go Into Project ${project_name}
Switch To Registries
Create A New Endpoint docker-hub e${d} https://hub.docker.com/ danfengliu Aa123456 Y
Switch To Replication Manage
${flag}= Set Variable ${false}
:FOR ${i} IN RANGE 999999
\ ${minite}= Get Current Date result_format=%M
\ ${minite_int} = Convert To Integer ${minite}
\ ${left} = Evaluate ${minite_int}%10
\ Log To Console ${i}/${left}
\ Run Keyword If ${left} <= 3 and ${left} != 0 Run Keywords Create A Rule With Existing Endpoint rule${d} pull danfengliu/* image e${d} ${project_name} mode=Scheduled cron=* */10 * * * * AND Set Suite Variable ${flag} ${true}
\ Sleep 40
\ Exit For Loop If '${flag}' == '${true}'
# After replication schedule is set, project should contain 2 images.
Log To Console Sleep for 720 seconds......
Sleep 720
Go Into Project ${project_name}
Go Into Repo ${project_name}/${image_a}
Artifact Exist ${tag_a}
Go Into Project ${project_name}
Go Into Repo ${project_name}/${image_b}
Artifact Exist ${tag_b}
# Delete repository
Go Into Project ${project_name}
Delete Repo ${project_name}/${image_a}
Delete Repo ${project_name}/${image_b}
# After replication schedule is set, project should contain 2 images.
Log To Console Sleep for 600 seconds......
Sleep 600
Go Into Project ${project_name}
Go Into Repo ${project_name}/${image_a}
Artifact Exist ${tag_a}
Go Into Project ${project_name}
Go Into Repo ${project_name}/${image_b}
Artifact Exist ${tag_b}

2
tests/robot-cases/Group1-Nightly/Trivy.robot
View File

@ -106,7 +106,7 @@ Test Case - Project Level Image Serverity Policy
Navigate To Projects
Go Into Project project${d}
Set Vulnerabilty Serverity 3
Cannot pull image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256}
Cannot Pull Image ${ip} ${HARBOR_ADMIN} ${HARBOR_PASSWORD} project${d} ${image} tag=${sha256} err_msg=To continue with pull, please contact your project administrator to exempt matched vulnerabilities through configuring the CVE whitelist
Close Browser
#Important Note: All CVE IDs in CVE Whitelist cases must unique!